Merge pull request #2680 from aws-lumberyard-dev/Atom/RenderDoc

Add runtime RenderDoc support for Windows dx12/vulkan via --enableRenderDoc option

Gems/Atom/RHI/Code/CMakeLists.txt

@@ -7,9 +7,26 @@
 #
 
 ly_get_list_relative_pal_filename(pal_dir ${CMAKE_CURRENT_LIST_DIR}/Platform/${PAL_PLATFORM_NAME})
+ly_get_list_relative_pal_filename(pal_source_dir ${CMAKE_CURRENT_LIST_DIR}/Source/Platform/${PAL_PLATFORM_NAME})
 
 include(${pal_dir}/AtomRHITests_traits_${PAL_PLATFORM_NAME_LOWERCASE}.cmake)
 
+set(RENDERDOC_CMAKE ${CMAKE_CURRENT_SOURCE_DIR}/${pal_dir}/renderdoc_${PAL_PLATFORM_NAME_LOWERCASE}.cmake)
+if(EXISTS ${RENDERDOC_CMAKE})
+    include(${RENDERDOC_CMAKE})
+endif()
+
+if(TARGET "3rdParty::renderdoc")
+    message(STATUS "Renderdoc found, adding as runtime dependency")
+    set(USE_RENDERDOC_DEFINE "USE_RENDERDOC")
+    set(RENDERDOC_BUILD_DEPENDENCY "3rdParty::renderdoc")
+    set(RENDERDOC_API_DEPENDENCY "3rdParty::renderdoc_api")
+else()
+    set(USE_RENDERDOC_DEFINE "")
+    set(RENDERDOC_BUILD_DEPENDENCY "")
+    set(RENDERDOC_API_DEPENDENCY "")
+endif()
+
 ly_add_target(
     NAME Atom_RHI.Reflect STATIC
     NAMESPACE Gem
@@ -33,9 +50,11 @@ ly_add_target(
     NAMESPACE Gem
     FILES_CMAKE
         atom_rhi_public_files.cmake
+        ${pal_source_dir}/platform_${PAL_PLATFORM_NAME_LOWERCASE}_files.cmake
     INCLUDE_DIRECTORIES
         PRIVATE
             Source
+            ${pal_source_dir}
         PUBLIC
             Include
     BUILD_DEPENDENCIES
@@ -43,6 +62,12 @@ ly_add_target(
             AZ::AzCore
             AZ::AzFramework
             Gem::Atom_RHI.Reflect
+            ${RENDERDOC_BUILD_DEPENDENCY}
+        PUBLIC
+            ${RENDERDOC_API_DEPENDENCY}
+    COMPILE_DEFINITIONS
+        PUBLIC
+            ${USE_RENDERDOC_DEFINE}
 )
 
 ly_add_target(

Gems/Atom/RHI/Code/Include/Atom/RHI/Factory.h

@@ -11,6 +11,10 @@
 #include <Atom/RHI/PhysicalDevice.h>
 #include <AzCore/Name/Name.h>
 
+#if defined(USE_RENDERDOC)
+#include <renderdoc_app.h>
+#endif
+
 namespace AZ
 {
     namespace RHI
@@ -66,7 +70,7 @@ namespace AZ
         public:
             AZ_TYPE_INFO(Factory, "{2C0231FD-DD11-4154-A4F5-177181E26D8E}");
 
-            Factory() = default;
+            Factory();
             virtual ~Factory() = default;
 
             // Note that you have to delete these for safety reasons, you will trip a static_assert if you do not
@@ -93,6 +97,15 @@ namespace AZ
             /// Access the global factory instance.
             static Factory& Get();
 
+#if defined(USE_RENDERDOC)
+            /// Access the RenderDoc API pointer if available.
+            /// The availability of the render doc API at runtime depends on the following:
+            /// - You must not be building a packaged game/product (LY_MONOLITHIC_GAME not enabled in CMake)
+            /// - A valid renderdoc installation was found, either by auto-discovery, or by supplying ATOM_RENDERDOC_PATH as an environment variable
+            /// - The module loaded successfully at runtime, and the API function pointer was retrieved successfully
+            static RENDERDOC_API_1_1_2* GetRenderDocAPI();
+#endif
+
             /// Returns the name of the Factory.
             virtual Name GetName() = 0;
 

Gems/Atom/RHI/Code/Include/Atom/RHI/RHIUtils.h

@@ -37,6 +37,9 @@ namespace AZ
         //! If multiple values exist it will return the last one
         AZStd::string GetCommandLineValue(const AZStd::string& commandLineOption);
 
+        //! Returns true if the command line option is set
+        bool QueryCommandLineOption(const AZStd::string& commandLineOption);
+
         //! Returns if the current bakcend is a null renderer
         bool IsNullRenderer();
     }

Gems/Atom/RHI/Code/Platform/Linux/renderdoc_linux.cmake

@@ -0,0 +1,36 @@
+#
+# Copyright (c) Contributors to the Open 3D Engine Project.
+# For complete copyright and license terms please see the LICENSE at the root of this distribution.
+#
+# SPDX-License-Identifier: Apache-2.0 OR MIT
+#
+#
+
+# Prevent bundling the renderdoc dll with a packaged title
+if(NOT LY_MONOLITHIC_GAME)
+    if(DEFINED ENV{"ATOM_RENDERDOC_PATH"})
+        set(RENDERDOC_PATH ENV{"ATOM_RENDERDOC_PATH"})
+    endif()
+
+    if(RENDERDOC_PATH)
+        # Normalize file path
+        file(TO_CMAKE_PATH "${RENDERDOC_PATH}" RENDERDOC_PATH)
+
+        if(EXISTS "${RENDERDOC_PATH}/librenderdoc.so")
+            ly_add_external_target(
+                NAME renderdoc
+                VERSION
+                3RDPARTY_ROOT_DIRECTORY ${RENDERDOC_PATH}
+                INCLUDE_DIRECTORIES "."
+                RUNTIME_DEPENDENCIES "${RENDERDOC_PATH}/librenderdoc.so"
+            )
+
+            ly_add_external_target(
+                NAME renderdoc_api
+                VERSION
+                3RDPARTY_ROOT_DIRECTORY ${RENDERDOC_PATH}
+                INCLUDE_DIRECTORIES "."
+            )
+        endif()
+    endif()
+endif()

Gems/Atom/RHI/Code/Platform/Windows/renderdoc_windows.cmake

@@ -0,0 +1,38 @@
+#
+# Copyright (c) Contributors to the Open 3D Engine Project.
+# For complete copyright and license terms please see the LICENSE at the root of this distribution.
+#
+# SPDX-License-Identifier: Apache-2.0 OR MIT
+#
+#
+
+# Prevent bundling the renderdoc dll with a packaged title
+if(NOT LY_MONOLITHIC_GAME)
+    # Common installation path for renderdoc path
+    set(RENDERDOC_PATH "C:/Program Files/RenderDoc")
+    if(DEFINED ENV{"ATOM_RENDERDOC_PATH"})
+        set(RENDERDOC_PATH ENV{"ATOM_RENDERDOC_PATH"})
+    endif()
+
+    if(RENDERDOC_PATH)
+        # Normalize file path
+        file(TO_CMAKE_PATH "${RENDERDOC_PATH}" RENDERDOC_PATH)
+
+        if(EXISTS "${RENDERDOC_PATH}/renderdoc.dll")
+            ly_add_external_target(
+                NAME renderdoc
+                VERSION
+                3RDPARTY_ROOT_DIRECTORY ${RENDERDOC_PATH}
+                INCLUDE_DIRECTORIES "."
+                RUNTIME_DEPENDENCIES "${RENDERDOC_PATH}/renderdoc.dll"
+            )
+
+            ly_add_external_target(
+                NAME renderdoc_api
+                VERSION
+                3RDPARTY_ROOT_DIRECTORY ${RENDERDOC_PATH}
+                INCLUDE_DIRECTORIES "."
+            )
+        endif()
+    endif()
+endif()

Gems/Atom/RHI/Code/Source/Platform/Android/Atom_RHI_Traits_Android.h

@@ -0,0 +1,10 @@
+/*
+ * Copyright (c) Contributors to the Open 3D Engine Project.
+ * For complete copyright and license terms please see the LICENSE at the root of this distribution.
+ *
+ * SPDX-License-Identifier: Apache-2.0 OR MIT
+ *
+ */
+#pragma once
+
+#define AZ_TRAIT_RENDERDOC_MODULE "libVkLayer_GLES_RenderDoc.so"

Gems/Atom/RHI/Code/Source/Platform/Android/Atom_RHI_Traits_Platform.h

@@ -0,0 +1,10 @@
+/*
+ * Copyright (c) Contributors to the Open 3D Engine Project.
+ * For complete copyright and license terms please see the LICENSE at the root of this distribution.
+ *
+ * SPDX-License-Identifier: Apache-2.0 OR MIT
+ *
+ */
+#pragma once
+
+#include "Atom_RHI_Traits_Android.h"

Gems/Atom/RHI/Code/Source/Platform/Android/platform_android_files.cmake

@@ -0,0 +1,12 @@
+#
+# Copyright (c) Contributors to the Open 3D Engine Project.
+# For complete copyright and license terms please see the LICENSE at the root of this distribution.
+#
+# SPDX-License-Identifier: Apache-2.0 OR MIT
+#
+#
+
+set(FILES
+    Atom_RHI_Traits_Platform.h
+    Atom_RHI_Traits_Android.h
+)

Gems/Atom/RHI/Code/Source/Platform/Linux/Atom_RHI_Traits_Linux.h

@@ -0,0 +1,10 @@
+/*
+ * Copyright (c) Contributors to the Open 3D Engine Project.
+ * For complete copyright and license terms please see the LICENSE at the root of this distribution.
+ *
+ * SPDX-License-Identifier: Apache-2.0 OR MIT
+ *
+ */
+#pragma once
+
+#define AZ_TRAIT_RENDERDOC_MODULE "librenderdoc.so"

Gems/Atom/RHI/Code/Source/Platform/Linux/Atom_RHI_Traits_Platform.h

@@ -0,0 +1,10 @@
+/*
+ * Copyright (c) Contributors to the Open 3D Engine Project.
+ * For complete copyright and license terms please see the LICENSE at the root of this distribution.
+ *
+ * SPDX-License-Identifier: Apache-2.0 OR MIT
+ *
+ */
+#pragma once
+
+#include "Atom_RHI_Traits_Linux.h"

Gems/Atom/RHI/Code/Source/Platform/Linux/platform_linux_files.cmake

@@ -0,0 +1,12 @@
+#
+# Copyright (c) Contributors to the Open 3D Engine Project.
+# For complete copyright and license terms please see the LICENSE at the root of this distribution.
+#
+# SPDX-License-Identifier: Apache-2.0 OR MIT
+#
+#
+
+set(FILES
+    Atom_RHI_Traits_Platform.h
+    Atom_RHI_Traits_Linux.h
+)

Gems/Atom/RHI/Code/Source/Platform/Mac/Atom_RHI_Traits_Mac.h

@@ -0,0 +1,8 @@
+/*
+ * Copyright (c) Contributors to the Open 3D Engine Project.
+ * For complete copyright and license terms please see the LICENSE at the root of this distribution.
+ *
+ * SPDX-License-Identifier: Apache-2.0 OR MIT
+ *
+ */
+#pragma once

Gems/Atom/RHI/Code/Source/Platform/Mac/Atom_RHI_Traits_Platform.h

@@ -0,0 +1,10 @@
+/*
+ * Copyright (c) Contributors to the Open 3D Engine Project.
+ * For complete copyright and license terms please see the LICENSE at the root of this distribution.
+ *
+ * SPDX-License-Identifier: Apache-2.0 OR MIT
+ *
+ */
+#pragma once
+
+#include "Atom_RHI_Traits_Mac.h"

Gems/Atom/RHI/Code/Source/Platform/Mac/platform_mac_files.cmake

@@ -0,0 +1,12 @@
+#
+# Copyright (c) Contributors to the Open 3D Engine Project.
+# For complete copyright and license terms please see the LICENSE at the root of this distribution.
+#
+# SPDX-License-Identifier: Apache-2.0 OR MIT
+#
+#
+
+set(FILES
+    Atom_RHI_Traits_Platform.h
+    Atom_RHI_Traits_Mac.h
+)

Gems/Atom/RHI/Code/Source/Platform/Windows/Atom_RHI_Traits_Platform.h

@@ -0,0 +1,10 @@
+/*
+ * Copyright (c) Contributors to the Open 3D Engine Project.
+ * For complete copyright and license terms please see the LICENSE at the root of this distribution.
+ *
+ * SPDX-License-Identifier: Apache-2.0 OR MIT
+ *
+ */
+#pragma once
+
+#include "Atom_RHI_Traits_Windows.h"

Gems/Atom/RHI/Code/Source/Platform/Windows/Atom_RHI_Traits_Windows.h

@@ -0,0 +1,10 @@
+/*
+ * Copyright (c) Contributors to the Open 3D Engine Project.
+ * For complete copyright and license terms please see the LICENSE at the root of this distribution.
+ *
+ * SPDX-License-Identifier: Apache-2.0 OR MIT
+ *
+ */
+#pragma once
+
+#define AZ_TRAIT_RENDERDOC_MODULE "renderdoc.dll"

Gems/Atom/RHI/Code/Source/Platform/Windows/platform_windows_files.cmake

@@ -0,0 +1,12 @@
+#
+# Copyright (c) Contributors to the Open 3D Engine Project.
+# For complete copyright and license terms please see the LICENSE at the root of this distribution.
+#
+# SPDX-License-Identifier: Apache-2.0 OR MIT
+#
+#
+
+set(FILES
+    Atom_RHI_Traits_Platform.h
+    Atom_RHI_Traits_Windows.h
+)

Gems/Atom/RHI/Code/Source/Platform/iOS/Atom_RHI_Traits_Platform.h

@@ -0,0 +1,10 @@
+/*
+ * Copyright (c) Contributors to the Open 3D Engine Project.
+ * For complete copyright and license terms please see the LICENSE at the root of this distribution.
+ *
+ * SPDX-License-Identifier: Apache-2.0 OR MIT
+ *
+ */
+#pragma once
+
+#include "Atom_RHI_Traits_iOS.h"

Gems/Atom/RHI/Code/Source/Platform/iOS/Atom_RHI_Traits_iOS.h

@@ -0,0 +1,8 @@
+/*
+ * Copyright (c) Contributors to the Open 3D Engine Project.
+ * For complete copyright and license terms please see the LICENSE at the root of this distribution.
+ *
+ * SPDX-License-Identifier: Apache-2.0 OR MIT
+ *
+ */
+#pragma once

Gems/Atom/RHI/Code/Source/Platform/iOS/platform_ios_files.cmake

@@ -0,0 +1,12 @@
+#
+# Copyright (c) Contributors to the Open 3D Engine Project.
+# For complete copyright and license terms please see the LICENSE at the root of this distribution.
+#
+# SPDX-License-Identifier: Apache-2.0 OR MIT
+#
+#
+
+set(FILES
+    Atom_RHI_Traits_Platform.h
+    Atom_RHI_Traits_iOS.h
+)

Gems/Atom/RHI/Code/Source/RHI/Factory.cpp

@@ -11,6 +11,16 @@
 #include <AzCore/Interface/Interface.h>
 #include <AzCore/Component/TickBus.h>
 
+#if defined(USE_RENDERDOC)
+#include <AzCore/Module/DynamicModuleHandle.h>
+#include <Atom/RHI/RHIUtils.h>
+#include <Atom_RHI_Traits_Platform.h>
+
+static AZStd::unique_ptr<AZ::DynamicModuleHandle> s_renderDocModule;
+static RENDERDOC_API_1_1_2* s_renderDocApi = nullptr;
+#endif
+
+
 namespace AZ
 {
     namespace RHI
@@ -30,6 +40,48 @@ namespace AZ
             return AZ_CRC("RHIPlatformService", 0xfff2cea4);
         }
 
+        Factory::Factory()
+        {
+#if defined(USE_RENDERDOC)
+            // If RenderDoc is requested, we need to load the library as early as possible (before device queries/factories are made)
+            bool enableRenderDoc = RHI::QueryCommandLineOption("enableRenderDoc");
+
+            if (enableRenderDoc && AZ_TRAIT_RENDERDOC_MODULE && !s_renderDocModule)
+            {
+                s_renderDocModule = DynamicModuleHandle::Create(AZ_TRAIT_RENDERDOC_MODULE);
+                if (s_renderDocModule)
+                {
+                    if (s_renderDocModule->Load(false))
+                    {
+                        pRENDERDOC_GetAPI renderDocGetAPI = s_renderDocModule->GetFunction<pRENDERDOC_GetAPI>("RENDERDOC_GetAPI");
+                        if (renderDocGetAPI)
+                        {
+                            if (!renderDocGetAPI(eRENDERDOC_API_Version_1_1_2, reinterpret_cast<void**>(&s_renderDocApi)))
+                            {
+                                s_renderDocApi = nullptr;
+                            }
+                        }
+
+                        if (s_renderDocApi)
+                        {
+                            // Prevent RenderDoc from handling any exceptions that may interfere with the O3DE exception handler
+                            s_renderDocApi->UnloadCrashHandler();
+                        }
+                        else
+                        {
+                            AZ_Printf("RHISystem", "RenderDoc module loaded but failed to retrieve API function pointer.\n");
+                        }
+                    }
+                    else
+                    {
+                        AZ_Printf("RHISystem", "RenderDoc module requested but module failed to load.\n");
+                    }
+                }
+            }
+#endif // defined(USE_RENDERDOC)
+
+        }
+
         void Factory::Register(Factory* instance)
         {
             Interface<Factory>::Register(instance);
@@ -58,6 +110,13 @@ namespace AZ
             ResourceInvalidateBus::ClearQueuedEvents();
 
             Interface<Factory>::Unregister(instance);
+
+#if defined(USE_RENDERDOC)
+            if (s_renderDocModule)
+            {
+                s_renderDocModule->Unload();
+            }
+#endif
         }
 
         bool Factory::IsReady()
@@ -71,5 +130,12 @@ namespace AZ
             AZ_Assert(factory, "RHI::Factory is not connected to a platform. Call IsReady() to get the status of the platform. A null de-reference is imminent.");
             return *factory;
         }
+
+#if defined(USE_RENDERDOC)
+        RENDERDOC_API_1_1_2* Factory::GetRenderDocAPI()
+        {
+            return s_renderDocApi;
+        }
+#endif
     }
 }

Gems/Atom/RHI/Code/Source/RHI/RHIUtils.cpp

@@ -122,5 +122,17 @@ namespace AZ
             }
             return commandLineValue;
         }
+
+        bool QueryCommandLineOption(const AZStd::string& commandLineOption)
+        {
+            const AzFramework::CommandLine* commandLine = nullptr;
+            AzFramework::ApplicationRequests::Bus::BroadcastResult(commandLine, &AzFramework::ApplicationRequests::GetApplicationCommandLine);
+
+            if (commandLine)
+            {
+                return commandLine->HasSwitch(commandLineOption);
+            }
+            return false;
+        }
     }
 }