Fixes for viewport selection issues (#5494)

* improvements to editor selection in the viewport Signed-off-by: Tom Hulton-Harrop <82228511+hultonha@users.noreply.github.com> * fix issue with being able to select icons that are not showing for entities inside entity containers Signed-off-by: Tom Hulton-Harrop <82228511+hultonha@users.noreply.github.com> * update comment after review feedback Signed-off-by: Tom Hulton-Harrop <82228511+hultonha@users.noreply.github.com> * updates to viewport picking code to simplify the api Signed-off-by: Tom Hulton-Harrop <82228511+hultonha@users.noreply.github.com> * add test to replicate near clip intersection issue Signed-off-by: Tom Hulton-Harrop <82228511+hultonha@users.noreply.github.com> * small tidy-up changes Signed-off-by: Tom Hulton-Harrop <82228511+hultonha@users.noreply.github.com> * updates to how we perform world to screen and screen to world calculations, added test coverage and some tidy-up Signed-off-by: Tom Hulton-Harrop <82228511+hultonha@users.noreply.github.com> * add some more tests for ViewportInteractionImpl Signed-off-by: Tom Hulton-Harrop <82228511+hultonha@users.noreply.github.com> * minor tweaks before PR feedback Signed-off-by: Tom Hulton-Harrop <82228511+hultonha@users.noreply.github.com> * fix typo in fix Signed-off-by: Tom Hulton-Harrop <82228511+hultonha@users.noreply.github.com> * fix for manipulator test framework tests Signed-off-by: Tom Hulton-Harrop <82228511+hultonha@users.noreply.github.com> * updates to RPI::View and RenderPipeline after review feedback from VickyAtAZ Signed-off-by: Tom Hulton-Harrop <82228511+hultonha@users.noreply.github.com> * add constexpr to ScreenPoint, ScreenVector and ScreenSize initializing constructors Signed-off-by: Tom Hulton-Harrop <82228511+hultonha@users.noreply.github.com> * add PrintTo functions for Screen* types Signed-off-by: Tom Hulton-Harrop <82228511+hultonha@users.noreply.github.com> * downgrade error to warning temporarily Signed-off-by: Tom Hulton-Harrop <82228511+hultonha@users.noreply.github.com> * check incoming view is null Signed-off-by: Tom Hulton-Harrop <82228511+hultonha@users.noreply.github.com> * remove pragma optimize off Signed-off-by: Tom Hulton-Harrop <82228511+hultonha@users.noreply.github.com>
2021-11-17 17:12:21 +00:00
parent ef16207ac5
commit e34ea3bcaa
45 changed files with 680 additions and 309 deletions
@@ -547,13 +547,6 @@ QPoint Q2DViewport::WorldToView(const Vec3& wp) const
    QPoint p = QPoint(static_cast<int>(sp.x), static_cast<int>(sp.y));
    return p;
 }
-//////////////////////////////////////////////////////////////////////////
-QPoint Q2DViewport::WorldToViewParticleEditor(const Vec3& wp, [[maybe_unused]] int width, [[maybe_unused]] int height) const //Eric@conffx implement for the children class of IDisplayViewport
-{
-    Vec3 sp = m_screenTM.TransformPoint(wp);
-    QPoint p = QPoint(static_cast<int>(sp.x), static_cast<int>(sp.y));
-    return p;
-}

 //////////////////////////////////////////////////////////////////////////
 Vec3    Q2DViewport::ViewToWorld(const QPoint& vp, [[maybe_unused]] bool* collideWithTerrain, [[maybe_unused]] bool onlyTerrain, [[maybe_unused]] bool bSkipVegetation, [[maybe_unused]] bool bTestRenderMesh, [[maybe_unused]] bool* collideWithObject) const
@@ -50,8 +50,6 @@ public:
    //! Map world space position to viewport position.
    QPoint     WorldToView(const Vec3& wp) const override;

-    QPoint WorldToViewParticleEditor(const Vec3& wp, int width, int height) const override; //Eric@conffx
-
    //! Map viewport position to world space position.
    Vec3        ViewToWorld(const QPoint& vp, bool* collideWithTerrain = nullptr, bool onlyTerrain = false, bool bSkipVegetation = false, bool bTestRenderMesh = false, bool* collideWithObject = nullptr) const override;
    //! Map viewport position to world space ray from camera.
@@ -299,13 +299,9 @@ AzToolsFramework::ViewportInteraction::MousePick EditorViewportWidget::BuildMous
 {
    AzToolsFramework::ViewportInteraction::MousePick mousePick;
    mousePick.m_screenCoordinates = AzToolsFramework::ViewportInteraction::ScreenPointFromQPoint(point);
-    if (const auto& ray = m_renderViewport->ViewportScreenToWorldRay(mousePick.m_screenCoordinates);
-        ray.has_value())
-    {
-        mousePick.m_rayOrigin = ray.value().origin;
-        mousePick.m_rayDirection = ray.value().direction;
-    }
-
+    const auto[origin, direction] = m_renderViewport->ViewportScreenToWorldRay(mousePick.m_screenCoordinates);
+    mousePick.m_rayOrigin = origin;
+    mousePick.m_rayDirection = direction;
    return mousePick;
 }

@@ -912,23 +908,6 @@ AZ::Vector3 EditorViewportWidget::PickTerrain(const AzFramework::ScreenPoint& po
    return LYVec3ToAZVec3(ViewToWorld(AzToolsFramework::ViewportInteraction::QPointFromScreenPoint(point), nullptr, true));
 }

-AZ::EntityId EditorViewportWidget::PickEntity(const AzFramework::ScreenPoint& point)
-{
-    AZ::EntityId entityId;
-    HitContext hitInfo;
-    hitInfo.view = this;
-    if (HitTest(AzToolsFramework::ViewportInteraction::QPointFromScreenPoint(point), hitInfo))
-    {
-        if (hitInfo.object && (hitInfo.object->GetType() == OBJTYPE_AZENTITY))
-        {
-            auto entityObject = static_cast<CComponentEntityObject*>(hitInfo.object);
-            entityId = entityObject->GetAssociatedEntityId();
-        }
-    }
-
-    return entityId;
-}
-
 float EditorViewportWidget::TerrainHeight(const AZ::Vector2& position)
 {
    return GetIEditor()->GetTerrainElevation(position.GetX(), position.GetY());
@@ -1653,16 +1632,15 @@ void EditorViewportWidget::RenderSelectedRegion()
 Vec3 EditorViewportWidget::WorldToView3D(const Vec3& wp, [[maybe_unused]] int nFlags) const
 {
    Vec3 out(0, 0, 0);
-    float x, y, z;
+    float x, y;

-    ProjectToScreen(wp.x, wp.y, wp.z, &x, &y, &z);
-    if (_finite(x) && _finite(y) && _finite(z))
+    ProjectToScreen(wp.x, wp.y, wp.z, &x, &y);
+    if (_finite(x) && _finite(y))
    {
        out.x = (x / 100) * m_rcClient.width();
        out.y = (y / 100) * m_rcClient.height();
        out.x /= static_cast<float>(QHighDpiScaling::factor(windowHandle()->screen()));
        out.y /= static_cast<float>(QHighDpiScaling::factor(windowHandle()->screen()));
-        out.z = z;
    }
    return out;
 }
@@ -1672,24 +1650,6 @@ QPoint EditorViewportWidget::WorldToView(const Vec3& wp) const
 {
    return AzToolsFramework::ViewportInteraction::QPointFromScreenPoint(m_renderViewport->ViewportWorldToScreen(LYVec3ToAZVec3(wp)));
 }
-//////////////////////////////////////////////////////////////////////////
-QPoint EditorViewportWidget::WorldToViewParticleEditor(const Vec3& wp, int width, int height) const
-{
-    QPoint p;
-    float x, y, z;
-
-    ProjectToScreen(wp.x, wp.y, wp.z, &x, &y, &z);
-    if (_finite(x) || _finite(y))
-    {
-        p.rx() = static_cast<int>((x / 100) * width);
-        p.ry() = static_cast<int>((y / 100) * height);
-    }
-    else
-    {
-        QPoint(0, 0);
-    }
-    return p;
-}

 //////////////////////////////////////////////////////////////////////////
 Vec3 EditorViewportWidget::ViewToWorld(
@@ -1705,20 +1665,16 @@ Vec3 EditorViewportWidget::ViewToWorld(
    AZ_UNUSED(collideWithObject);

    auto ray = m_renderViewport->ViewportScreenToWorldRay(AzToolsFramework::ViewportInteraction::ScreenPointFromQPoint(vp));
-    if (!ray.has_value())
-    {
-        return Vec3(0, 0, 0);
-    }

    const float maxDistance = 10000.f;
-    Vec3 v = AZVec3ToLYVec3(ray.value().direction) * maxDistance;
+    Vec3 v = AZVec3ToLYVec3(ray.direction) * maxDistance;

    if (!_finite(v.x) || !_finite(v.y) || !_finite(v.z))
    {
        return Vec3(0, 0, 0);
    }

-    Vec3 colp = AZVec3ToLYVec3(ray.value().origin) + 0.002f * v;
+    Vec3 colp = AZVec3ToLYVec3(ray.origin) + 0.002f * v;

    return colp;
 }
@@ -1757,21 +1713,19 @@ bool EditorViewportWidget::RayRenderMeshIntersection(IRenderMesh* pRenderMesh, c
    return bRes;*/
 }

-void EditorViewportWidget::UnProjectFromScreen(float sx, float sy, float sz, float* px, float* py, float* pz) const
+void EditorViewportWidget::UnProjectFromScreen(float sx, float sy, float* px, float* py, float* pz) const
 {
-    AZ::Vector3 wp;
-    wp = m_renderViewport->ViewportScreenToWorld(AzFramework::ScreenPoint{(int)sx, m_rcClient.bottom() - ((int)sy)}, sz).value_or(wp);
+    const AZ::Vector3 wp = m_renderViewport->ViewportScreenToWorld(AzFramework::ScreenPoint{(int)sx, m_rcClient.bottom() - ((int)sy)});
    *px = wp.GetX();
    *py = wp.GetY();
    *pz = wp.GetZ();
 }

-void EditorViewportWidget::ProjectToScreen(float ptx, float pty, float ptz, float* sx, float* sy, float* sz) const
+void EditorViewportWidget::ProjectToScreen(float ptx, float pty, float ptz, float* sx, float* sy) const
 {
    AzFramework::ScreenPoint screenPosition = m_renderViewport->ViewportWorldToScreen(AZ::Vector3{ptx, pty, ptz});
    *sx = static_cast<float>(screenPosition.m_x);
    *sy = static_cast<float>(screenPosition.m_y);
-    *sz = 0.f;
 }

 //////////////////////////////////////////////////////////////////////////
@@ -1781,32 +1735,22 @@ void EditorViewportWidget::ViewToWorldRay(const QPoint& vp, Vec3& raySrc, Vec3&

    Vec3 pos0, pos1;
    float wx, wy, wz;
-    UnProjectFromScreen(static_cast<float>(vp.x()), static_cast<float>(rc.bottom() - vp.y()), 0.0f, &wx, &wy, &wz);
-    if (!_finite(wx) || !_finite(wy) || !_finite(wz))
-    {
-        return;
-    }
-    if (fabs(wx) > 1000000 || fabs(wy) > 1000000 || fabs(wz) > 1000000)
-    {
-        return;
-    }
-    pos0(wx, wy, wz);
-    UnProjectFromScreen(static_cast<float>(vp.x()), static_cast<float>(rc.bottom() - vp.y()), 1.0f, &wx, &wy, &wz);
-    if (!_finite(wx) || !_finite(wy) || !_finite(wz))
-    {
-        return;
-    }
-    if (fabs(wx) > 1000000 || fabs(wy) > 1000000 || fabs(wz) > 1000000)
-    {
-        return;
-    }
-    pos1(wx, wy, wz);
+    UnProjectFromScreen(static_cast<float>(vp.x()), static_cast<float>(rc.bottom() - vp.y()), &wx, &wy, &wz);

-    Vec3 v = (pos1 - pos0);
-    v = v.GetNormalized();
+    if (!_finite(wx) || !_finite(wy) || !_finite(wz))
+    {
+        return;
+    }
+
+    if (fabs(wx) > 1000000 || fabs(wy) > 1000000 || fabs(wz) > 1000000)
+    {
+        return;
+    }
+
+    pos0(wx, wy, wz);

    raySrc = pos0;
-    rayDir = v;
+    rayDir = (pos0 - AZVec3ToLYVec3(m_renderViewport->GetCameraState().m_position)).GetNormalized();
 }

 //////////////////////////////////////////////////////////////////////////
@@ -166,7 +166,6 @@ private:
        Qt::MouseButtons buttons, Qt::KeyboardModifiers modifiers, const QPoint& point) override;
    void SetViewportId(int id) override;
    QPoint WorldToView(const Vec3& wp) const override;
-    QPoint WorldToViewParticleEditor(const Vec3& wp, int width, int height) const override;
    Vec3 WorldToView3D(const Vec3& wp, int nFlags = 0) const override;
    Vec3 ViewToWorld(const QPoint& vp, bool* collideWithTerrain = nullptr, bool onlyTerrain = false, bool bSkipVegetation = false, bool bTestRenderMesh = false, bool* collideWithObject = nullptr) const override;
    void ViewToWorldRay(const QPoint& vp, Vec3& raySrc, Vec3& rayDir) const override;
@@ -208,7 +207,6 @@ private:
    void* GetSystemCursorConstraintWindow() const override;

    // AzToolsFramework::MainEditorViewportInteractionRequestBus overrides ...
-    AZ::EntityId PickEntity(const AzFramework::ScreenPoint& point) override;
    AZ::Vector3 PickTerrain(const AzFramework::ScreenPoint& point) override;
    float TerrainHeight(const AZ::Vector2& position) override;
    bool ShowingWorldSpace() override;
@@ -306,8 +304,8 @@ private:
    const DisplayContext& GetDisplayContext() const { return m_displayContext; }
    CBaseObject* GetCameraObject() const;

-    void UnProjectFromScreen(float sx, float sy, float sz, float* px, float* py, float* pz) const;
-    void ProjectToScreen(float ptx, float pty, float ptz, float* sx, float* sy, float* sz) const;
+    void UnProjectFromScreen(float sx, float sy, float* px, float* py, float* pz) const;
+    void ProjectToScreen(float ptx, float pty, float ptz, float* sx, float* sy) const;

    AZ::RPI::ViewPtr GetCurrentAtomView() const;

@@ -47,7 +47,6 @@ struct IDisplayViewport
    virtual const Matrix34& GetViewTM() const = 0;
    virtual const Matrix34& GetScreenTM() const = 0;
    virtual QPoint WorldToView(const Vec3& worldPoint) const = 0;
-    virtual QPoint WorldToViewParticleEditor(const Vec3& worldPoint, int width, int height) const = 0;
    virtual Vec3 WorldToView3D(const Vec3& worldPoint, int flags = 0) const = 0;
    virtual Vec3 ViewToWorld(const QPoint& vp, bool* collideWithTerrain = nullptr, bool onlyTerrain = false, bool bSkipVegetation = false, bool bTestRenderMesh = false, bool* collideWithObject = nullptr) const = 0;
    virtual void ViewToWorldRay(const QPoint& vp, Vec3& raySrc, Vec3& rayDir) const = 0;
@@ -8,13 +8,15 @@

 #include "ViewportManipulatorController.h"

+#include <AzCore/Script/ScriptTimePoint.h>
+#include <AzFramework/Input/Buses/Requests/InputSystemCursorRequestBus.h>
+#include <AzFramework/Input/Devices/Keyboard/InputDeviceKeyboard.h>
+#include <AzFramework/Input/Devices/Mouse/InputDeviceMouse.h>
+#include <AzFramework/Viewport/ScreenGeometry.h>
+#include <AzFramework/Viewport/ViewportScreen.h>
 #include <AzToolsFramework/Manipulators/ManipulatorManager.h>
 #include <AzToolsFramework/ViewportSelection/EditorInteractionSystemViewportSelectionRequestBus.h>
-#include <AzFramework/Input/Devices/Mouse/InputDeviceMouse.h>
-#include <AzFramework/Input/Devices/Keyboard/InputDeviceKeyboard.h>
-#include <AzFramework/Input/Buses/Requests/InputSystemCursorRequestBus.h>
-#include <AzFramework/Viewport/ScreenGeometry.h>
-#include <AzCore/Script/ScriptTimePoint.h>
+#include <AzToolsFramework/ViewportSelection/EditorSelectionUtil.h>

 #include <QApplication>

@@ -87,8 +89,14 @@ namespace SandboxEditor
        }

        using InteractionBus = AzToolsFramework::EditorInteractionSystemViewportSelectionRequestBus;
-        using namespace AzToolsFramework::ViewportInteraction;
        using AzFramework::InputChannel;
+        using AzToolsFramework::ViewportInteraction::KeyboardModifier;
+        using AzToolsFramework::ViewportInteraction::MouseButton;
+        using AzToolsFramework::ViewportInteraction::MouseEvent;
+        using AzToolsFramework::ViewportInteraction::MouseInteraction;
+        using AzToolsFramework::ViewportInteraction::MouseInteractionEvent;
+        using AzToolsFramework::ViewportInteraction::ProjectedViewportRay;
+        using AzToolsFramework::ViewportInteraction::ViewportInteractionRequestBus;

        bool interactionHandled = false;
        float wheelDelta = 0.0f;
@@ -117,16 +125,13 @@ namespace SandboxEditor
                    aznumeric_cast<int>(position->m_normalizedPosition.GetX() * windowSize.m_width),
                    aznumeric_cast<int>(position->m_normalizedPosition.GetY() * windowSize.m_height));

-                m_mouseInteraction.m_mousePick.m_screenCoordinates = screenPoint;
-                AZStd::optional<ProjectedViewportRay> ray;
+                ProjectedViewportRay ray{};
                ViewportInteractionRequestBus::EventResult(
                    ray, GetViewportId(), &ViewportInteractionRequestBus::Events::ViewportScreenToWorldRay, screenPoint);

-                if (ray.has_value())
-                {
-                    m_mouseInteraction.m_mousePick.m_rayOrigin = ray.value().origin;
-                    m_mouseInteraction.m_mousePick.m_rayDirection = ray.value().direction;
-                }
+                m_mouseInteraction.m_mousePick.m_rayOrigin = ray.origin;
+                m_mouseInteraction.m_mousePick.m_rayDirection = ray.direction;
+                m_mouseInteraction.m_mousePick.m_screenCoordinates = screenPoint;
            }

            eventType = MouseEvent::Move;
@@ -160,8 +165,8 @@ namespace SandboxEditor
            else if (state == InputChannel::State::Ended)
            {
                // If we've actually logged a mouse down event, forward a mouse up event.
-                // This prevents corner cases like the context menu thinking it should be opened even though no one clicked in this viewport,
-                // due to RenderViewportWidget ensuring all controllers get InputChannel::State::Ended events.
+                // This prevents corner cases like the context menu thinking it should be opened even though no one clicked in this
+                // viewport, due to RenderViewportWidget ensuring all controllers get InputChannel::State::Ended events.
                if (m_mouseInteraction.m_mouseButtons.m_mouseButtons & mouseButtonValue)
                {
                    // Erase the button from our state if we're done processing events.
@@ -259,4 +264,4 @@ namespace SandboxEditor
        const double doubleClickThresholdMilliseconds = qApp->doubleClickInterval();
        return (m_curTime.GetMilliseconds() - clickIt->second.GetMilliseconds()) < doubleClickThresholdMilliseconds;
    }
-} //namespace SandboxEditor
+} // namespace SandboxEditor