Disable PSO caching if Renderdoc or Pix dlls are loaded as they confl… (#3976)

* Disable PSO caching if Renderdoc or Pix dlls are loaded as they conflict with dx12 api

* Add support for command line param --enablePixGPU which manually loads the pix gpu capture dll
 - This will allow pix to attach to a running process if the enablePixGPU is enabled
 - Support for enabling/disabling gpu markers based on enablePixGPU

Signed-off-by: moudgils <moudgils@amazon.com>

Gems/Atom/RHI/Code/CMakeLists.txt

@@ -42,6 +42,7 @@ ly_add_target(
     FILES_CMAKE
         atom_rhi_public_files.cmake
         ${pal_source_dir}/platform_${PAL_PLATFORM_NAME_LOWERCASE}_files.cmake
+        ${pal_source_dir}/platform_private_${PAL_PLATFORM_NAME_LOWERCASE}_files.cmake
     INCLUDE_DIRECTORIES
         PRIVATE
             Source

Gems/Atom/RHI/Code/Include/Atom/RHI/Factory.h

@@ -50,24 +50,21 @@ namespace AZ
         class RayTracingPipelineState;
         class RayTracingShaderTable;
 
-        /* Priority of a Factory. The lower the number the higher the priority.
-         * Used when there's multiple factories available and the user hasn't define
-         * a priority.
-         */
+        //! Priority of a Factory. The lower the number the higher the priority.
+        //! Used when there's multiple factories available and the user hasn't define
+        //! a priority.
         using APIPriority = uint32_t;
         static const APIPriority APITopPriority = 1;
         static const APIPriority APILowPriority = 10;
         static const APIPriority APIMiddlePriority = (APILowPriority - APITopPriority) / 2;
 
-        /**
-         * The factory is an interface for creating RHI data structures. The platform system should
-         * register itself with the factory by calling Register, and unregister on shutdown with
-         * Unregister.
-         *
-         * A call to Get will return the active instance. In the event that it's unclear whether
-         * a platform instance exists, you must call IsReady to determine whether it's safe to
-         * call Get. Calling Get without a registered platform will result in an assert.
-         */
+        //! The factory is an interface for creating RHI data structures. The platform system should
+        //! register itself with the factory by calling Register, and unregister on shutdown with
+        //! Unregister.
+        //!
+        //! A call to Get will return the active instance. In the event that it's unclear whether
+        //! a platform instance exists, you must call IsReady to determine whether it's safe to
+        //! call Get. Calling Get without a registered platform will result in an assert.
         class Factory
         {
         public:
@@ -79,78 +76,78 @@ namespace AZ
             // Note that you have to delete these for safety reasons, you will trip a static_assert if you do not
             AZ_DISABLE_COPY_MOVE(Factory);
 
-            /// Returns the component service name CRC used by the platform RHI system component.
+            //! Returns the component service name CRC used by the platform RHI system component.
             static uint32_t GetComponentService();
 
-            /// Returns the component service name CRC used by the Factory manager component.
+            //! Returns the component service name CRC used by the Factory manager component.
             static uint32_t GetManagerComponentService();
 
-            /// Returns the component service name CRC used by the platform RHI system component.
+            //! Returns the component service name CRC used by the platform RHI system component.
             static uint32_t GetPlatformService();
 
-            /// Registers the global factory instance.
+            //! Registers the global factory instance.
             static void Register(Factory* instance);
 
-            /// Unregisters the global factory instance.
+            //! Unregisters the global factory instance.
             static void Unregister(Factory* instance);
 
-            /// Returns whether the factory is initialized and active in this module.
+            //! Returns whether the factory is initialized and active in this module.
             static bool IsReady();
 
-            /// Access the global factory instance.
+            //! Access the global factory instance.
             static Factory& Get();
 
 #if defined(USE_RENDERDOC)
-            /// Access the RenderDoc API pointer if available.
-            /// The availability of the render doc API at runtime depends on the following:
-            /// - You must not be building a packaged game/product (LY_MONOLITHIC_GAME not enabled in CMake)
-            /// - A valid renderdoc installation was found, either by auto-discovery, or by supplying ATOM_RENDERDOC_PATH as an environment variable
-            /// - The module loaded successfully at runtime, and the API function pointer was retrieved successfully
+            //! Access the RenderDoc API pointer if available.
+            //! The availability of the render doc API at runtime depends on the following:
+            //! - You must not be building a packaged game/product (LY_MONOLITHIC_GAME not enabled in CMake)
+            //! - A valid renderdoc installation was found, either by auto-discovery, or by supplying ATOM_RENDERDOC_PATH as an environment variable
+            //! - The module loaded successfully at runtime, and the API function pointer was retrieved successfully
             static RENDERDOC_API_1_1_2* GetRenderDocAPI();
 #endif
+            
+            //! Returns true if RenderDoc dll is loaded
+            static bool IsRenderDocModuleLoaded();
 
-            /// Returns the name of the Factory.
+            //! Returns true if Pix dll is loaded
+            static bool IsPixModuleLoaded();
+
+            //! Returns the name of the Factory.
             virtual Name GetName() = 0;
 
-            /// Returns the APIType of the factory.
+            //! Returns the APIType of the factory.
             virtual APIType GetType() = 0;
 
-            /// Returns the default priority of the factory in case there's no priorities set in the FactoryManager.
+            //! Returns the default priority of the factory in case there's no priorities set in the FactoryManager.
             virtual APIPriority GetDefaultPriority() = 0;
 
-            /**
-             * Purpose: The API Unique Index will be encoded in the 2 Most Significant Bits of a ShaderVariantAsset ProductSubId (a 32bits integer). 
-             * Returns a number in the range [0..3].
-             * In theory any given AssetBuilderSdk::PlatformInfo can support several RHI::APITypes.
-             * In reality "pc" only supports DX12 & Vulkan.
-             * "ios" supports only Metal.
-             * "mac" supports only Metal.
-             * "android" supports only Vulkan.
-             * So, for all practical purposes, a single PlatformInfo won't support more than 2 ShaderPlatformInterfaces, but for the sake of
-             * hedging our bets into the future We assume no more than 4 ShaderPlatformInterfaces will ever be supported for any given PlatformInfo.
-             * REMARK: It is the responsibility of the Factory subclass to return a unique number between 0...3.
-             * For example DX12 can return 0, while Vulkan should return 1 (Satisfies "pc", "android" and "linux").
-             * Metal can return 0 because it is the only ShaderPlatformInterface for "ios", "mac" and "appletv".
-             * See AZ::RHI::Limits::APIType::PerPlatformApiUniqueIndexMax.
-             */
+            //! Purpose: The API Unique Index will be encoded in the 2 Most Significant Bits of a ShaderVariantAsset ProductSubId (a 32bits integer). 
+            //! Returns a number in the range [0..3].
+            //! In theory any given AssetBuilderSdk::PlatformInfo can support several RHI::APITypes.
+            //! In reality "pc" only supports DX12 & Vulkan.
+            //! "ios" supports only Metal.
+            //! "mac" supports only Metal.
+            //! "android" supports only Vulkan.
+            //! So, for all practical purposes, a single PlatformInfo won't support more than 2 ShaderPlatformInterfaces, but for the sake of
+            //! hedging our bets into the future We assume no more than 4 ShaderPlatformInterfaces will ever be supported for any given PlatformInfo.
+            //! REMARK: It is the responsibility of the Factory subclass to return a unique number between 0...3.
+            //! For example DX12 can return 0, while Vulkan should return 1 (Satisfies "pc", "android" and "linux").
+            //! Metal can return 0 because it is the only ShaderPlatformInterface for "ios", "mac" and "appletv".
+            //! See AZ::RHI::Limits::APIType::PerPlatformApiUniqueIndexMax.
             virtual uint32_t GetAPIUniqueIndex() const = 0;
 
-            /**
-             * Collects the set of physical devices on the system and returns a list of them. Physical
-             * devices represent the hardware attached to the system. Physical devices can be grouped
-             * into nodes for linked setups (e.g. SLI / Crossfire). They can also represent software
-             * reference implementations. Check the PhysicalDeviceType on the descriptor to inspect
-             * this information.
-             */
+            //! Collects the set of physical devices on the system and returns a list of them. Physical
+            //! devices represent the hardware attached to the system. Physical devices can be grouped
+            //! into nodes for linked setups (e.g. SLI / Crossfire). They can also represent software
+            //! reference implementations. Check the PhysicalDeviceType on the descriptor to inspect
+            //! this information.
             virtual PhysicalDeviceList EnumeratePhysicalDevices() = 0;
 
-            /**
-             * Factory Creation Methods:
-             *
-             * Returns the platform-specific derived variant of the RHI type. All instances are created
-             * in an uninitialized state; the operation simply allocates the memory for the appropriate
-             * platform type and returns the pointer.
-             */
+            //! Factory Creation Methods:
+            //!
+            //! Returns the platform-specific derived variant of the RHI type. All instances are created
+            //! in an uninitialized state; the operation simply allocates the memory for the appropriate
+            //! platform type and returns the pointer.
 
             virtual Ptr<Buffer> CreateBuffer() = 0;
 

Gems/Atom/RHI/Code/Source/Platform/Android/platform_private_android_files.cmake

@@ -0,0 +1,11 @@
+#
+# Copyright (c) Contributors to the Open 3D Engine Project.
+# For complete copyright and license terms please see the LICENSE at the root of this distribution.
+#
+# SPDX-License-Identifier: Apache-2.0 OR MIT
+#
+#
+
+set(FILES
+    ../Common/Unimplemented/Empty_Unimplemented.cpp
+)

Gems/Atom/RHI/Code/Source/Platform/Common/Unimplemented/Empty_Unimplemented.cpp

@@ -0,0 +1,21 @@
+/*
+ * Copyright (c) Contributors to the Open 3D Engine Project.
+ * For complete copyright and license terms please see the LICENSE at the root of this distribution.
+ *
+ * SPDX-License-Identifier: Apache-2.0 OR MIT
+ *
+ */
+#include <AzCore/std/string/string.h>
+
+namespace AZ::RHI::Platform
+{
+    bool IsPixDllInjected([[maybe_unused]] const char* dllName)
+    {
+        return false;
+    }
+
+    AZStd::wstring GetLatestWinPixGpuCapturerPath()
+    {
+        return L"";
+    }
+} 

Gems/Atom/RHI/Code/Source/Platform/Linux/platform_private_linux_files.cmake

@@ -0,0 +1,11 @@
+#
+# Copyright (c) Contributors to the Open 3D Engine Project.
+# For complete copyright and license terms please see the LICENSE at the root of this distribution.
+#
+# SPDX-License-Identifier: Apache-2.0 OR MIT
+#
+#
+
+set(FILES
+    ../Common/Unimplemented/Empty_Unimplemented.cpp
+)

Gems/Atom/RHI/Code/Source/Platform/Mac/platform_private_mac_files.cmake

@@ -0,0 +1,11 @@
+#
+# Copyright (c) Contributors to the Open 3D Engine Project.
+# For complete copyright and license terms please see the LICENSE at the root of this distribution.
+#
+# SPDX-License-Identifier: Apache-2.0 OR MIT
+#
+#
+
+set(FILES
+    ../Common/Unimplemented/Empty_Unimplemented.cpp
+)

Gems/Atom/RHI/Code/Source/Platform/Windows/Atom_RHI_Traits_Windows.h

@@ -8,3 +8,4 @@
 #pragma once
 
 #define AZ_TRAIT_RENDERDOC_MODULE "renderdoc.dll"
+#define AZ_TRAIT_PIX_MODULE "WinPixGpuCapturer.dll"

Gems/Atom/RHI/Code/Source/Platform/Windows/RHI/Factory_windows.cpp

@@ -0,0 +1,59 @@
+/*
+ * Copyright (c) Contributors to the Open 3D Engine Project.
+ * For complete copyright and license terms please see the LICENSE at the root of this distribution.
+ *
+ * SPDX-License-Identifier: Apache-2.0 OR MIT
+ *
+ */
+
+#include <filesystem>
+#include <shlobj.h>
+#include <AzCore/base.h>
+#include <AzCore/std/string/string.h>
+
+namespace AZ::RHI::Platform
+{
+    bool IsPixDllInjected(const char* dllName)
+    {
+        bool isDllLoaded = false;
+
+        wchar_t fileNameW[256];
+        size_t numCharsConverted;
+        errno_t wcharResult = mbstowcs_s(&numCharsConverted, fileNameW, dllName, AZ_ARRAY_SIZE(fileNameW) - 1);
+        if (wcharResult == 0)
+        {
+            isDllLoaded = NULL != GetModuleHandleW(fileNameW);
+        }
+        return isDllLoaded;
+    }
+
+    AZStd::wstring GetLatestWinPixGpuCapturerPath()
+    {
+        LPWSTR programFilesPath = nullptr;
+        SHGetKnownFolderPath(FOLDERID_ProgramFiles, KF_FLAG_DEFAULT, NULL, &programFilesPath);
+
+        std::filesystem::path pixInstallationPath = programFilesPath;
+        pixInstallationPath /= "Microsoft PIX";
+
+        std::wstring newestVersionFound;
+
+        for (auto const& directory_entry : std::filesystem::directory_iterator(pixInstallationPath))
+        {
+            if (directory_entry.is_directory())
+            {
+                if (newestVersionFound.empty() || newestVersionFound < directory_entry.path().filename().c_str())
+                {
+                    newestVersionFound = directory_entry.path().filename().c_str();
+                }
+            }
+        }
+
+        if (newestVersionFound.empty())
+        {
+            return L"";
+        }
+
+        std::wstring finalPath = pixInstallationPath / newestVersionFound / L"WinPixGpuCapturer.dll";
+        return AZStd::wstring(finalPath.c_str());
+    }
+} 

Gems/Atom/RHI/Code/Source/Platform/Windows/platform_private_windows_files.cmake

@@ -0,0 +1,11 @@
+#
+# Copyright (c) Contributors to the Open 3D Engine Project.
+# For complete copyright and license terms please see the LICENSE at the root of this distribution.
+#
+# SPDX-License-Identifier: Apache-2.0 OR MIT
+#
+#
+
+set(FILES
+    RHI/Factory_windows.cpp
+)

Gems/Atom/RHI/Code/Source/Platform/iOS/platform_private_ios_files.cmake

@@ -0,0 +1,11 @@
+#
+# Copyright (c) Contributors to the Open 3D Engine Project.
+# For complete copyright and license terms please see the LICENSE at the root of this distribution.
+#
+# SPDX-License-Identifier: Apache-2.0 OR MIT
+#
+#
+
+set(FILES
+    ../Common/Unimplemented/Empty_Unimplemented.cpp
+)

Gems/Atom/RHI/Code/Source/RHI/Factory.cpp

@@ -11,20 +11,33 @@
 #include <AzCore/Interface/Interface.h>
 #include <AzCore/Component/TickBus.h>
 
-#if defined(USE_RENDERDOC)
+#if defined(USE_RENDERDOC) || defined(USE_PIX)
 #include <AzCore/Module/DynamicModuleHandle.h>
 #include <Atom/RHI/RHIUtils.h>
 #include <Atom_RHI_Traits_Platform.h>
-
-static AZStd::unique_ptr<AZ::DynamicModuleHandle> s_renderDocModule;
-static RENDERDOC_API_1_1_2* s_renderDocApi = nullptr;
 #endif
 
+#if defined(USE_RENDERDOC)
+static AZStd::unique_ptr<AZ::DynamicModuleHandle> s_renderDocModule;
+static RENDERDOC_API_1_1_2* s_renderDocApi = nullptr;
+static bool s_isRenderDocDllLoaded = false;
+#endif
+
+#if defined(USE_PIX)
+static AZStd::unique_ptr<AZ::DynamicModuleHandle> s_pixModule;
+static bool s_isPixGpuCaptureDllLoaded = false;
+#endif
 
 namespace AZ
 {
     namespace RHI
     {
+        namespace Platform
+        {
+            bool IsPixDllInjected(const char* dllName);
+            AZStd::wstring GetLatestWinPixGpuCapturerPath();
+        }
+
         uint32_t Factory::GetComponentService()
         {
             return AZ_CRC("RHIService", 0x45d8e053);
@@ -53,6 +66,7 @@ namespace AZ
                 {
                     if (s_renderDocModule->Load(false))
                     {
+                        s_isRenderDocDllLoaded = true;
                         pRENDERDOC_GetAPI renderDocGetAPI = s_renderDocModule->GetFunction<pRENDERDOC_GetAPI>("RENDERDOC_GetAPI");
                         if (renderDocGetAPI)
                         {
@@ -78,8 +92,30 @@ namespace AZ
                     }
                 }
             }
-#endif // defined(USE_RENDERDOC)
+#endif
 
+#if defined(USE_PIX)
+            // If GPU capture is requested, we need to load the pix library as early as possible (before device queries/factories are made)
+            bool enablePixGPU = RHI::QueryCommandLineOption("enablePixGPU");
+            if (enablePixGPU && AZ_TRAIT_PIX_MODULE && !s_pixModule)
+            {
+                //Get the path to the latest pix install directory
+                AZStd::wstring pixGpuDllPath = Platform::GetLatestWinPixGpuCapturerPath();
+                AZStd::string dllPath;
+                AZStd::to_string(dllPath, pixGpuDllPath);
+                s_pixModule = DynamicModuleHandle::Create(dllPath.c_str());
+                if (s_pixModule)
+                {
+                    if (!s_pixModule->Load(false))
+                    {
+                        AZ_Printf("RHISystem", "Pix capture requested but module failed to load.\n");
+                    }
+                }
+            }
+
+            //Pix dll can still be injected even if we do not pass in enablePixGPU. This can be done if we launch the app from Pix.
+            s_isPixGpuCaptureDllLoaded = Platform::IsPixDllInjected(AZ_TRAIT_PIX_MODULE);
+#endif
         }
 
         void Factory::Register(Factory* instance)
@@ -116,6 +152,12 @@ namespace AZ
             {
                 s_renderDocModule->Unload();
             }
+#endif
+#if defined(USE_PIX)
+            if (s_pixModule)
+            {
+                s_pixModule->Unload();
+            }
 #endif
         }
 
@@ -137,5 +179,23 @@ namespace AZ
             return s_renderDocApi;
         }
 #endif
+
+        bool Factory::IsRenderDocModuleLoaded()
+        {
+#if defined(USE_RENDERDOC)
+            return s_isRenderDocDllLoaded;
+#else
+            return false;
+#endif
+        }
+
+        bool Factory::IsPixModuleLoaded()
+        {
+#if defined(USE_PIX)
+            return s_isPixGpuCaptureDllLoaded;
+#else
+            return false;
+#endif
+        }
     }
 }

Gems/Atom/RHI/DX12/Code/Source/RHI/CommandList.cpp

@@ -41,6 +41,8 @@
 #define DX12_COMMANDLIST_TIMER_DETAIL(id)
 #endif
 
+#define PIX_MARKER_CMDLIST_COL 0xFF0000FF
+
 namespace AZ
 {
     namespace DX12
@@ -94,16 +96,22 @@ namespace AZ
         {
             SetName(name);
 
-            PIXBeginEvent(0xFF0000FF, name.GetCStr());
-            PIXBeginEvent(GetCommandList(), 0xFF0000FF, name.GetCStr());
+            PIXBeginEvent(PIX_MARKER_CMDLIST_COL, name.GetCStr());
+            if (RHI::Factory::Get().IsPixModuleLoaded() || RHI::Factory::Get().IsRenderDocModuleLoaded())
+            {
+                PIXBeginEvent(GetCommandList(), PIX_MARKER_CMDLIST_COL, name.GetCStr());
+            }
         }
 
         void CommandList::Close()
         {
             FlushBarriers();
-
-            PIXEndEvent(GetCommandList());
             PIXEndEvent();
+            if (RHI::Factory::Get().IsPixModuleLoaded() || RHI::Factory::Get().IsRenderDocModuleLoaded())
+            {
+                PIXEndEvent(GetCommandList());
+            }
+            
 
             CommandListBase::Close();
         }

Gems/Atom/RHI/DX12/Code/Source/RHI/PipelineLibrary.cpp

@@ -7,6 +7,7 @@
  */
 #include <RHI/Device.h>
 #include <RHI/PipelineLibrary.h>
+#include <Atom/RHI/Factory.h>
 
 namespace AZ
 {
@@ -46,7 +47,16 @@ namespace AZ
 
 #if defined (AZ_DX12_USE_PIPELINE_LIBRARY)
             AZStd::array_view<uint8_t> bytes;
-            if (serializedData)
+
+            bool shouldCreateLibFromSerializedData = true;
+            if (RHI::Factory::Get().IsRenderDocModuleLoaded() || RHI::Factory::Get().IsPixModuleLoaded())
+            {
+                // CreatePipelineLibrary api does not function properly if Renderdoc or Pix is enabled
+                shouldCreateLibFromSerializedData = false;
+            }
+
+
+            if (serializedData && shouldCreateLibFromSerializedData)
             {
                 bytes = serializedData->GetData();
             }
@@ -205,10 +215,11 @@ namespace AZ
 
         RHI::ResultCode PipelineLibrary::MergeIntoInternal([[maybe_unused]] AZStd::array_view<const RHI::PipelineLibrary*> pipelineLibraries)
         {
-#if defined(USE_PIX) || defined(USE_RENDERDOC)
-            // StorePipeline api does not function properly if Pix or RenderDoc is enabled
-            return RHI::ResultCode::Fail;
-#else
+            if (RHI::Factory::Get().IsRenderDocModuleLoaded() || RHI::Factory::Get().IsPixModuleLoaded())
+            {
+                // StorePipeline api does not function properly if RenderDoc or Pix is enabled
+                return RHI::ResultCode::Fail;
+            }
 
 #if defined (AZ_DX12_USE_PIPELINE_LIBRARY)
             AZStd::lock_guard<AZStd::mutex> lock(m_mutex);
@@ -226,8 +237,7 @@ namespace AZ
                 }
             }
 #endif
-            return RHI::ResultCode::Success;
-#endif           
+            return RHI::ResultCode::Success;           
         }
 
         RHI::ConstPtr<RHI::PipelineLibraryData> PipelineLibrary::GetSerializedDataInternal() const

Gems/Atom/RHI/DX12/Code/Source/RHI/Scope.cpp

@@ -18,6 +18,7 @@
 #include <Atom/RHI/ResourcePool.h>
 #include <Atom/RHI/ImageScopeAttachment.h>
 #include <Atom/RHI/BufferScopeAttachment.h>
+#include <Atom/RHI/Factory.h>
 #include <Atom/RHI/ResolveScopeAttachment.h>
 #include <AzCore/Debug/EventTrace.h>
 
@@ -309,8 +310,11 @@ namespace AZ
             commandList.GetValidator().BeginScope(*this);
 
             PIXBeginEvent(0xFFFF00FF, GetId().GetCStr());
-            PIXBeginEvent(commandList.GetCommandList(), 0xFFFF00FF, GetId().GetCStr());
 
+            if (RHI::Factory::Get().IsPixModuleLoaded() || RHI::Factory::Get().IsRenderDocModuleLoaded())
+            {
+                PIXBeginEvent(commandList.GetCommandList(), 0xFFFF00FF, GetId().GetCStr());
+            }
 
             commandList.SetAftermathEventMarker(GetId().GetCStr());
             
@@ -424,7 +428,10 @@ namespace AZ
                 }
             }
 
-            PIXEndEvent(commandList.GetCommandList());
+            if (RHI::Factory::Get().IsPixModuleLoaded() || RHI::Factory::Get().IsRenderDocModuleLoaded())
+            {
+                PIXEndEvent(commandList.GetCommandList());
+            }
             PIXEndEvent();
 
             commandList.GetValidator().EndScope();