Fixes for viewport selection issues (#5494)

* improvements to editor selection in the viewport

Signed-off-by: Tom Hulton-Harrop <82228511+hultonha@users.noreply.github.com>

* fix issue with being able to select icons that are not showing for entities inside entity containers

Signed-off-by: Tom Hulton-Harrop <82228511+hultonha@users.noreply.github.com>

* update comment after review feedback

Signed-off-by: Tom Hulton-Harrop <82228511+hultonha@users.noreply.github.com>

* updates to viewport picking code to simplify the api

Signed-off-by: Tom Hulton-Harrop <82228511+hultonha@users.noreply.github.com>

* add test to replicate near clip intersection issue

Signed-off-by: Tom Hulton-Harrop <82228511+hultonha@users.noreply.github.com>

* small tidy-up changes

Signed-off-by: Tom Hulton-Harrop <82228511+hultonha@users.noreply.github.com>

* updates to how we perform world to screen and screen to world calculations, added test coverage and some tidy-up

Signed-off-by: Tom Hulton-Harrop <82228511+hultonha@users.noreply.github.com>

* add some more tests for ViewportInteractionImpl

Signed-off-by: Tom Hulton-Harrop <82228511+hultonha@users.noreply.github.com>

* minor tweaks before PR feedback

Signed-off-by: Tom Hulton-Harrop <82228511+hultonha@users.noreply.github.com>

* fix typo in fix

Signed-off-by: Tom Hulton-Harrop <82228511+hultonha@users.noreply.github.com>

* fix for manipulator test framework tests

Signed-off-by: Tom Hulton-Harrop <82228511+hultonha@users.noreply.github.com>

* updates to RPI::View and RenderPipeline after review feedback from VickyAtAZ

Signed-off-by: Tom Hulton-Harrop <82228511+hultonha@users.noreply.github.com>

* add constexpr to ScreenPoint, ScreenVector and ScreenSize initializing constructors

Signed-off-by: Tom Hulton-Harrop <82228511+hultonha@users.noreply.github.com>

* add PrintTo functions for Screen* types

Signed-off-by: Tom Hulton-Harrop <82228511+hultonha@users.noreply.github.com>

* downgrade error to warning temporarily

Signed-off-by: Tom Hulton-Harrop <82228511+hultonha@users.noreply.github.com>

* check incoming view is null

Signed-off-by: Tom Hulton-Harrop <82228511+hultonha@users.noreply.github.com>

* remove pragma optimize off

Signed-off-by: Tom Hulton-Harrop <82228511+hultonha@users.noreply.github.com>

Gems/Atom/Tools/AtomToolsFramework/Code/Include/AtomToolsFramework/Viewport/RenderViewportWidget.h

@@ -21,6 +21,7 @@
 #include <AzFramework/Windowing/WindowBus.h>
 #include <AzCore/Component/TickBus.h>
 #include <Atom/RPI.Public/AuxGeom/AuxGeomFeatureProcessorInterface.h>
+#include <AtomToolsFramework/Viewport/ViewportInteractionImpl.h>
 
 namespace AtomToolsFramework
 {
@@ -30,8 +31,8 @@ namespace AtomToolsFramework
     //! @see AZ::RPI::ViewportContext for Atom's API for setting up 
     class RenderViewportWidget
         : public QWidget
-        , public AzToolsFramework::ViewportInteraction::ViewportInteractionRequestBus::Handler
         , public AzToolsFramework::ViewportInteraction::ViewportMouseCursorRequestBus::Handler
+        , public AzToolsFramework::ViewportInteraction::ViewportInteractionRequests
         , public AzFramework::WindowRequestBus::Handler
         , protected AzFramework::InputChannelEventListener
         , protected AZ::TickBus::Handler
@@ -90,11 +91,11 @@ namespace AtomToolsFramework
         //! Input processing is enabled by default.
         void SetInputProcessingEnabled(bool enabled);
 
-        // AzToolsFramework::ViewportInteraction::ViewportInteractionRequestBus::Handler overrides ...
+        // ViewportInteractionRequests overrides ...
         AzFramework::CameraState GetCameraState() override;
         AzFramework::ScreenPoint ViewportWorldToScreen(const AZ::Vector3& worldPosition) override;
-        AZStd::optional<AZ::Vector3> ViewportScreenToWorld(const AzFramework::ScreenPoint& screenPosition, float depth) override;
-        AZStd::optional<AzToolsFramework::ViewportInteraction::ProjectedViewportRay> ViewportScreenToWorldRay(
+        AZ::Vector3 ViewportScreenToWorld(const AzFramework::ScreenPoint& screenPosition) override;
+        AzToolsFramework::ViewportInteraction::ProjectedViewportRay ViewportScreenToWorldRay(
             const AzFramework::ScreenPoint& screenPosition) override;
         float DeviceScalingFactor() override;
 
@@ -149,5 +150,7 @@ namespace AtomToolsFramework
         AZ::ScriptTimePoint m_time;
         // Maps our internal Qt events into AzFramework InputChannels for our ViewportControllerList.
         AzToolsFramework::QtEventToAzInputMapper* m_inputChannelMapper = nullptr;
+        // Implementation of ViewportInteractionRequests (handles viewport picking operations).
+        AZStd::unique_ptr<ViewportInteractionImpl> m_viewportInteractionImpl;
     };
 } //namespace AtomToolsFramework

Gems/Atom/Tools/AtomToolsFramework/Code/Include/AtomToolsFramework/Viewport/ViewportInteractionImpl.h

@@ -0,0 +1,42 @@
+/*
+ * Copyright (c) Contributors to the Open 3D Engine Project.
+ * For complete copyright and license terms please see the LICENSE at the root of this distribution.
+ *
+ * SPDX-License-Identifier: Apache-2.0 OR MIT
+ *
+ */
+
+#pragma once
+
+#include <Atom/RPI.Public/Base.h>
+#include <Atom/RPI.Public/View.h>
+#include <AzFramework/Viewport/CameraState.h>
+#include <AzToolsFramework/Viewport/ViewportMessages.h>
+
+namespace AtomToolsFramework
+{
+    //! A concrete implementation of the ViewportInteractionRequestBus.
+    //! Primarily concerned with picking (screen to world and world to screen transformations).
+    class ViewportInteractionImpl : public AzToolsFramework::ViewportInteraction::ViewportInteractionRequestBus::Handler
+    {
+    public:
+        explicit ViewportInteractionImpl(AZ::RPI::ViewPtr viewPtr);
+
+        void Connect(AzFramework::ViewportId viewportId);
+        void Disconnect();
+
+        // ViewportInteractionRequestBus overrides ...
+        AzFramework::CameraState GetCameraState() override;
+        AzFramework::ScreenPoint ViewportWorldToScreen(const AZ::Vector3& worldPosition) override;
+        AZ::Vector3 ViewportScreenToWorld(const AzFramework::ScreenPoint& screenPosition) override;
+        AzToolsFramework::ViewportInteraction::ProjectedViewportRay ViewportScreenToWorldRay(
+            const AzFramework::ScreenPoint& screenPosition) override;
+        float DeviceScalingFactor() override;
+
+        AZStd::function<AzFramework::ScreenSize()> m_screenSizeFn; //! Callback to determine the screen size.
+        AZStd::function<float()> m_deviceScalingFactorFn; //! Callback to determine the device scaling factor.
+
+    private:
+        AZ::RPI::ViewPtr m_viewPtr;
+    };
+} // namespace AtomToolsFramework