Integrate from 1.0 to main: LYN-3436 AutomatedTesting.GameLauncher crashes at launch if assets are not all processed (#612)

* Atom/qingtao/lyn 3436 (#558)

* LYN-3436 AutomatedTesting.GameLauncher crashes at launch if assets are not all processed

Change RPISystem so that the application would exit if the RPI system couldn't load critical assets.
Added code to avoid the GetLayout crash when layout for each platforms were not ready.
Added LoadCriticalAsset function to force compile and load critical assets.
Added default value to viewport size for ViewportContext.

* Change RPISystem asset initialization order so it returns earlier when those critical assets are not ready

Gems/Atom/Bootstrap/Code/Source/BootstrapSystemComponent.cpp

@@ -15,6 +15,7 @@
 #include <AzCore/Asset/AssetCommon.h>
 #include <AzCore/Component/ComponentApplicationBus.h>
 #include <AzCore/Component/Entity.h>
+#include <AzCore/NativeUI/NativeUIRequests.h>
 #include <AzCore/Serialization/SerializeContext.h>
 #include <AzCore/std/smart_ptr/make_shared.h>
 
@@ -166,6 +167,18 @@ namespace AZ
 
                 RPI::RPISystemInterface::Get()->InitializeSystemAssets();
 
+                if (!RPI::RPISystemInterface::Get()->IsInitialized())
+                {
+                    AZ::OSString msgBoxMessage;
+                    msgBoxMessage.append("RPI System could not initialize correctly. Check log for detail.");
+
+                    AZ::NativeUI::NativeUIRequestBus::Broadcast(
+                        &AZ::NativeUI::NativeUIRequestBus::Events::DisplayOkDialog, "O3DE Fatal Error", msgBoxMessage.c_str(), false);
+                    AzFramework::ApplicationRequests::Bus::Broadcast(&AzFramework::ApplicationRequests::ExitMainLoop);
+                                        
+                    return;
+                }
+
                 // In the case of the game we want to call create and register the scene as a soon as we can
                 // because a level could be loaded in autoexec.cfg and that will assert if there is no scene registered
                 // to get the feature processors for the components. So we can't wait until the tick (whereas the Editor wants to wait)

Gems/Atom/RHI/Code/Include/Atom/RHI/CpuProfilerImpl.h

@@ -124,6 +124,8 @@ namespace AZ
 
             // This lock will only be contested when the CpuProfiler's Shutdown() method has been called
             AZStd::shared_mutex m_shutdownMutex;
+
+            bool m_initialized = false;
         };
 
     }; // namespace RPI

Gems/Atom/RHI/Code/Source/RHI/CpuProfilerImpl.cpp

@@ -82,10 +82,15 @@ namespace AZ
         void CpuProfilerImpl::Init()
         {
             Interface<CpuProfiler>::Register(this);
+            m_initialized = true;
         }
 
         void CpuProfilerImpl::Shutdown()
         {
+            if (!m_initialized)
+            {
+                return;
+            }
             // When this call is made, no more thread profiling calls can be performed anymore
             Interface<CpuProfiler>::Unregister(this);
 
@@ -97,6 +102,7 @@ namespace AZ
             // Cleanup all TLS
             m_registeredThreads.clear();
             m_timeRegionMap.clear();
+            m_initialized = false;
         }
 
         void CpuProfilerImpl::BeginTimeRegion(TimeRegion& timeRegion)

Gems/Atom/RHI/Code/Source/RHI/RHISystem.cpp

@@ -200,9 +200,12 @@ namespace AZ
 
             m_platformLimitsDescriptor = nullptr;
             m_pipelineStateCache = nullptr;
-            m_device->PreShutdown();
-            AZ_Assert(m_device->use_count()==1, "The ref count for Device is %i but it should be 1 here to ensure all the resources are released", m_device->use_count());
-            m_device = nullptr;
+            if (m_device)
+            {            
+                m_device->PreShutdown();
+                AZ_Assert(m_device->use_count()==1, "The ref count for Device is %i but it should be 1 here to ensure all the resources are released", m_device->use_count());
+                m_device = nullptr;
+            }
 
             m_cpuProfiler.Shutdown();
         }

Gems/Atom/RHI/DX12/Code/Source/RHI/AsyncUploadQueue.cpp

@@ -90,12 +90,16 @@ namespace AZ
 
         void AsyncUploadQueue::Shutdown()
         {
-            m_copyQueue->Shutdown();
+            if (m_copyQueue)
+            {
+                m_copyQueue->Shutdown();
+                m_copyQueue = nullptr;
+            }
             m_commandList = nullptr;
 
-            for (size_t i = 0; i < m_descriptor.m_frameCount; ++i)
+            for (auto& framePacket : m_framePackets)
             {
-                m_framePackets[i].m_fence.Shutdown();
+                framePacket.m_fence.Shutdown();
             }
             m_framePackets.clear();
             m_uploadFence.Shutdown();

Gems/Atom/RHI/DX12/Code/Source/RHI/CommandQueueContext.cpp

@@ -109,7 +109,10 @@ namespace AZ
             AZ_PROFILE_FUNCTION(AZ::Debug::ProfileCategory::AzRender);
             for (uint32_t hardwareQueueIdx = 0; hardwareQueueIdx < RHI::HardwareQueueClassCount; ++hardwareQueueIdx)
             {
-                m_commandQueues[hardwareQueueIdx]->WaitForIdle();
+                if (m_commandQueues[hardwareQueueIdx])
+                {
+                    m_commandQueues[hardwareQueueIdx]->WaitForIdle();
+                }
             }
         }
 

Gems/Atom/RPI/Code/Include/Atom/RPI.Public/Buffer/BufferSystem.h

@@ -45,6 +45,8 @@ namespace AZ
 
         private:
             RHI::Ptr<RHI::BufferPool> m_commonPools[static_cast<uint8_t>(CommonBufferPoolType::Count)];
+
+            bool m_initialized = false;
         };
     } // namespace RPI
 } // namespace AZ

Gems/Atom/RPI/Code/Include/Atom/RPI.Public/Image/ImageSystem.h

@@ -80,6 +80,8 @@ namespace AZ
             Data::Asset<DefaultStreamingImageControllerAsset> m_defaultStreamingImageControllerAsset;
 
             AZStd::fixed_vector<Data::Instance<Image>, static_cast<uint32_t>(SystemImage::Count)> m_systemImages;
+
+            bool m_initialized = false;
         };
     }
 }

Gems/Atom/RPI/Code/Include/Atom/RPI.Public/Pass/PassLibrary.h

@@ -82,7 +82,7 @@ namespace AZ
             void RemovePassFromLibrary(Pass* pass);
 
             //! Load pass templates which are list in an AssetAliases
-            void LoadPassTemplateMappings(const AZStd::string& templateMappingPath);
+            bool LoadPassTemplateMappings(const AZStd::string& templateMappingPath);
             bool LoadPassTemplateMappings(Data::Asset<AnyAsset> mappingAsset);
 
             //! Returns a list of passes found in the pass name mapping using the provided pass filter

Gems/Atom/RPI/Code/Include/Atom/RPI.Public/Pass/PassSystem.h

@@ -66,7 +66,7 @@ namespace AZ
 
             // PassSystemInterface functions...
             void ProcessQueuedChanges() override;
-            void LoadPassTemplateMappings(const AZStd::string& templateMappingPath) override;
+            bool LoadPassTemplateMappings(const AZStd::string& templateMappingPath) override;
             void WriteTemplateToFile(const PassTemplate& passTemplate, AZStd::string_view assetFilePath) override;
             void DebugPrintPassHierarchy() override;
             bool IsBuilding() const override;

Gems/Atom/RPI/Code/Include/Atom/RPI.Public/Pass/PassSystemInterface.h

@@ -65,7 +65,7 @@ namespace AZ
             virtual void ProcessQueuedChanges() = 0;
             
             //! Load pass templates listed in a name-assetid mapping asset 
-            virtual void LoadPassTemplateMappings(const AZStd::string& templateMappingPath) = 0;
+            virtual bool LoadPassTemplateMappings(const AZStd::string& templateMappingPath) = 0;
             
             //! Writes a pass template to a .pass file which can then be used as a pass asset. Useful for
             //! quickly authoring a pass template in code and then outputting it as a pass asset using JSON

Gems/Atom/RPI/Code/Include/Atom/RPI.Public/RPISystem.h

@@ -70,6 +70,7 @@ namespace AZ
             void Shutdown();
 
             // RPISystemInterface overrides...
+            bool IsInitialized() const override;
             void InitializeSystemAssets() override;
             void RegisterScene(ScenePtr scene) override;
             void UnregisterScene(ScenePtr scene) override;

Gems/Atom/RPI/Code/Include/Atom/RPI.Public/RPISystemInterface.h

@@ -40,6 +40,9 @@ namespace AZ
             //! Note: can't rely on the AzFramework::AssetCatalogEventBus's OnCatalogLoaded since the order of calling handlers is undefined.
             virtual void InitializeSystemAssets() = 0;
 
+            //! Was the RPI system initialized properly
+            virtual bool IsInitialized() const = 0;
+
             //! Register a created scene to RPISystem. Registered scene will be simulated and rendered in RPISystem ticks
             virtual void RegisterScene(ScenePtr scene) = 0;
 

Gems/Atom/RPI/Code/Include/Atom/RPI.Reflect/Asset/AssetUtils.h

@@ -15,6 +15,8 @@
 #include <AzCore/Asset/AssetCommon.h>
 #include <AzCore/Asset/AssetManager.h>
 
+#include <AzFramework/Asset/AssetSystemBus.h>
+
 namespace AZ
 {
     namespace RPI
@@ -48,6 +50,12 @@ namespace AZ
             //! @return a null asset if the asset could not be found or loaded.
             template<typename AssetDataT>
             Data::Asset<AssetDataT> LoadAssetById(Data::AssetId assetId, TraceLevel reporting = TraceLevel::Warning);
+            
+            //! Loads a critial asset using a file path (both source and product path should be same), on the current thread.
+            //! If the asset wasn't compiled, wait until the asset is compiled.
+            //! @return a null asset if the asset could not be compiled or loaded.
+            template<typename AssetDataT>
+            Data::Asset<AssetDataT> LoadCriticalAsset(const AZStd::string& assetFilePath, TraceLevel reporting = TraceLevel::Error);
 
             template<typename AssetDataT>
             bool LoadBlocking(AZ::Data::Asset<AssetDataT>& asset, TraceLevel reporting = TraceLevel::Warning);
@@ -89,7 +97,7 @@ namespace AZ
                         assetId, AZ::Data::AssetLoadBehavior::PreLoad);
                     asset.BlockUntilLoadComplete();
 
-                    if (!asset.Get())
+                    if (!asset.IsReady())
                     {
                         AssetUtilsInternal::ReportIssue(reporting, AZStd::string::format("Could not load '%s'", productPath).c_str());
                         return {};
@@ -117,7 +125,7 @@ namespace AZ
                     );
                     asset.BlockUntilLoadComplete();
 
-                if (!asset.Get())
+                if (!asset.IsReady())
                 {
                     AssetUtilsInternal::ReportIssue(reporting, AZStd::string::format("Could not load '%s'", assetId.ToString<AZStd::string>().c_str()).c_str());
                     return {};
@@ -126,6 +134,21 @@ namespace AZ
                 return asset;
             }
 
+            template<typename AssetDataT>
+            Data::Asset<AssetDataT> LoadCriticalAsset(const AZStd::string& assetFilePath, TraceLevel reporting)
+            {
+                AzFramework::AssetSystem::AssetStatus status = AzFramework::AssetSystem::AssetStatus_Unknown;
+                AzFramework::AssetSystemRequestBus::BroadcastResult(status, &AzFramework::AssetSystemRequestBus::Events::CompileAssetSync, assetFilePath);
+
+                if (status != AzFramework::AssetSystem::AssetStatus_Compiled)
+                {
+                    AssetUtilsInternal::ReportIssue(reporting, AZStd::string::format("Could not compile asset '%s'", assetFilePath.c_str()).c_str());
+                    return {};
+                }
+
+                return LoadAssetByProductPath<AssetDataT>(assetFilePath.c_str(), reporting);
+            }
+
             template<typename AssetDataT>
             bool LoadBlocking(AZ::Data::Asset<AssetDataT>& asset, TraceLevel reporting)
             {

Gems/Atom/RPI/Code/Source/RPI.Public/Buffer/BufferSystem.cpp

@@ -61,10 +61,16 @@ namespace AZ
                 Data::InstanceDatabase<BufferPool>::Create(azrtti_typeid<ResourcePoolAsset>(), handler);
             }
             Interface<BufferSystemInterface>::Register(this);
+
+            m_initialized = true;
         }
 
         void BufferSystem::Shutdown()
         {
+            if (!m_initialized)
+            {
+                return;
+            }
             for (uint8_t index = 0; index < static_cast<uint8_t>(CommonBufferPoolType::Count); index++)
             {
                 m_commonPools[index] = nullptr;
@@ -72,6 +78,7 @@ namespace AZ
             Interface<BufferSystemInterface>::Unregister(this);
             Data::InstanceDatabase<Buffer>::Destroy();
             Data::InstanceDatabase<BufferPool>::Destroy();
+            m_initialized = false;
         }
         
         RHI::Ptr<RHI::BufferPool> BufferSystem::GetCommonBufferPool(CommonBufferPoolType poolType)
@@ -87,6 +94,10 @@ namespace AZ
         
         bool BufferSystem::CreateCommonBufferPool(CommonBufferPoolType poolType)
         {
+            if (!m_initialized)
+            {
+                return false;
+            }
             auto* device = RHI::RHISystemInterface::Get()->GetDevice();
             
             RHI::Ptr<RHI::BufferPool> bufferPool = RHI::Factory::Get().CreateBufferPool();

Gems/Atom/RPI/Code/Source/RPI.Public/Image/ImageSystem.cpp

@@ -148,10 +148,16 @@ namespace AZ
             CreateDefaultResources(desc);
 
             Interface<ImageSystemInterface>::Register(this);
+
+            m_initialized = true;
         }
 
         void ImageSystem::Shutdown()
         {
+            if (!m_initialized)
+            {
+                return;
+            }
             Interface<ImageSystemInterface>::Unregister(this);
 
             m_defaultStreamingImageControllerAsset.Release();
@@ -167,6 +173,7 @@ namespace AZ
             Data::InstanceDatabase<StreamingImageController>::Destroy();
 
             m_activeStreamingPools.clear();
+            m_initialized = false;
         }
 
         void ImageSystem::Update()

Gems/Atom/RPI/Code/Source/RPI.Public/Pass/PassLibrary.cpp

@@ -327,14 +327,15 @@ namespace AZ
             }
         }
 
-        void PassLibrary::LoadPassTemplateMappings(const AZStd::string& templateMappingPath)
+        bool PassLibrary::LoadPassTemplateMappings(const AZStd::string& templateMappingPath)
         {
-            Data::Asset<AnyAsset> mappingAsset = AssetUtils::LoadAssetByProductPath<AnyAsset>(templateMappingPath.c_str(), AssetUtils::TraceLevel::Error);
+            Data::Asset<AnyAsset> mappingAsset = AssetUtils::LoadCriticalAsset<AnyAsset>(templateMappingPath.c_str(), AssetUtils::TraceLevel::Error);
             bool success = LoadPassTemplateMappings(mappingAsset);
             if (success)
             {
                 Data::AssetBus::MultiHandler::BusConnect(mappingAsset->GetId());
             }
+            return success;
         }
 
         bool PassLibrary::LoadPassTemplateMappings(Data::Asset<AnyAsset> mappingAsset)

Gems/Atom/RPI/Code/Source/RPI.Public/Pass/PassSystem.cpp

@@ -100,9 +100,9 @@ namespace AZ
             m_rootPass->m_flags.m_partOfHierarchy = true;
         }
 
-        void PassSystem::LoadPassTemplateMappings(const AZStd::string& templateMappingPath)
+        bool PassSystem::LoadPassTemplateMappings(const AZStd::string& templateMappingPath)
         {
-            m_passLibrary.LoadPassTemplateMappings(templateMappingPath);
+            return m_passLibrary.LoadPassTemplateMappings(templateMappingPath);
         }
 
         void PassSystem::WriteTemplateToFile(const PassTemplate& passTemplate, AZStd::string_view assetFilePath)

Gems/Atom/RPI/Code/Source/RPI.Public/RPISystem.cpp

@@ -334,7 +334,6 @@ namespace AZ
 
         void RPISystem::InitializeSystemAssets()
         {
-            AzFramework::AssetSystem::AssetStatus status = AzFramework::AssetSystem::AssetStatus_Unknown;
             if (m_systemAssetsInitialized)
             {
                 AZ_Warning("RPISystem", false , "InitializeSystemAssets should only be called once'");
@@ -344,36 +343,47 @@ namespace AZ
             //[GFX TODO][ATOM-5867] - Move file loading code within RHI to reduce coupling with RPI
             AZStd::string platformLimitsFilePath = AZStd::string::format("config/platform/%s/%s/platformlimits.azasset", AZ_TRAIT_OS_PLATFORM_NAME, GetRenderApiName().GetCStr());
             AZStd::to_lower(platformLimitsFilePath.begin(), platformLimitsFilePath.end());
-            // Wait for the platformlimits asset to be compiled (if it exists)
-            AzFramework::AssetSystemRequestBus::BroadcastResult(
-                status, &AzFramework::AssetSystemRequestBus::Events::CompileAssetSync, platformLimitsFilePath);
-            Data::Asset<RPI::AnyAsset> platformLimitsAsset = RPI::AssetUtils::LoadAssetByProductPath<RPI::AnyAsset>(platformLimitsFilePath.c_str(), RPI::AssetUtils::TraceLevel::Error); 
-            m_descriptor.m_rhiSystemDescriptor.m_platformLimits = RPI::GetDataFromAnyAsset<RHI::PlatformLimits>(platformLimitsAsset);
+            
+            Data::Asset<AnyAsset> platformLimitsAsset;
+            platformLimitsAsset = RPI::AssetUtils::LoadCriticalAsset<AnyAsset>(platformLimitsFilePath.c_str(), RPI::AssetUtils::TraceLevel::None);
+            // Only read the m_platformLimits if the platformLimitsAsset is ready.
+            // The platformLimitsAsset may not exist for null renderer which is allowed
+            if (platformLimitsAsset.IsReady())
+            {
+                m_descriptor.m_rhiSystemDescriptor.m_platformLimits = RPI::GetDataFromAnyAsset<RHI::PlatformLimits>(platformLimitsAsset);
+            }
+
+            m_viewSrgAsset = AssetUtils::LoadCriticalAsset<ShaderResourceGroupAsset>( m_descriptor.m_viewSrgAssetPath.c_str());
+            if (!m_viewSrgAsset.IsReady())
+            {
+                return;
+            }
+            m_sceneSrgAsset = AssetUtils::LoadCriticalAsset<ShaderResourceGroupAsset>(m_descriptor.m_sceneSrgAssetPath.c_str());
+            if (!m_sceneSrgAsset.IsReady())
+            {
+                return;
+            }
 
             m_rhiSystem.Init(m_descriptor.m_rhiSystemDescriptor);
             m_imageSystem.Init(m_descriptor.m_imageSystemDescriptor);
             m_bufferSystem.Init();
             m_dynamicDraw.Init(m_descriptor.m_dynamicDrawSystemDescriptor);
 
-            // Wait for the assets be compiled
-            AzFramework::AssetSystemRequestBus::BroadcastResult(
-                status, &AzFramework::AssetSystemRequestBus::Events::CompileAssetSync, m_descriptor.m_viewSrgAssetPath);
-            AZ_Error("RPISystem", status == AzFramework::AssetSystem::AssetStatus_Compiled, "Could not compile view SRG at '%s'", m_descriptor.m_viewSrgAssetPath.c_str());
-            AzFramework::AssetSystemRequestBus::BroadcastResult(
-                status, &AzFramework::AssetSystemRequestBus::Events::CompileAssetSync, m_descriptor.m_sceneSrgAssetPath);
-            AZ_Error("RPISystem", status == AzFramework::AssetSystem::AssetStatus_Compiled, "Could not compile scene SRG at '%s'", m_descriptor.m_sceneSrgAssetPath.c_str());
-            AzFramework::AssetSystemRequestBus::BroadcastResult(
-                status, &AzFramework::AssetSystemRequestBus::Events::CompileAssetSync, m_descriptor.m_passTemplatesMappingPath);
-            AZ_Error("RPISystem", status == AzFramework::AssetSystem::AssetStatus_Compiled, "Could not compile pass template mapping at '%s'", m_descriptor.m_passTemplatesMappingPath.c_str());
-
-            m_viewSrgAsset = AssetUtils::LoadAssetByProductPath<ShaderResourceGroupAsset>(m_descriptor.m_viewSrgAssetPath.c_str(), AssetUtils::TraceLevel::Error);
-            m_sceneSrgAsset = AssetUtils::LoadAssetByProductPath<ShaderResourceGroupAsset>(m_descriptor.m_sceneSrgAssetPath.c_str(), AssetUtils::TraceLevel::Error);
-
             // Have pass system load default pass template mapping
-            m_passSystem.LoadPassTemplateMappings(m_descriptor.m_passTemplatesMappingPath);
+            bool passSystemReady = m_passSystem.LoadPassTemplateMappings(m_descriptor.m_passTemplatesMappingPath);
+            if (!passSystemReady)
+            {
+                return;
+            }
+
             m_systemAssetsInitialized = true;
         }
 
+        bool RPISystem::IsInitialized() const
+        {
+            return m_systemAssetsInitialized;
+        }
+
         void RPISystem::InitializeSystemAssetsForTests()
         {
             if (m_systemAssetsInitialized)

Gems/Atom/RPI/Code/Source/RPI.Public/ViewportContext.cpp

@@ -26,6 +26,7 @@ namespace AZ
             , m_windowContext(AZStd::make_shared<WindowContext>())
             , m_manager(manager)
             , m_name(name)
+            , m_viewportSize(1, 1)
         {
             m_windowContext->Initialize(device, nativeWindow);
             AzFramework::WindowRequestBus::EventResult(

Gems/Atom/RPI/Code/Source/RPI.Reflect/Shader/ShaderResourceGroupAsset.cpp

@@ -41,7 +41,11 @@ namespace AZ
 
         const RHI::ShaderResourceGroupLayout* ShaderResourceGroupAsset::GetLayout() const
         {
-            AZ_Assert(m_currentAPITypeIndex < m_perAPILayout.size(), "Invalid API Type index");
+            AZ_Error("RHI::ShaderResourceGroupLayout", m_currentAPITypeIndex < m_perAPILayout.size(), "Invalid API Type index");
+            if (m_currentAPITypeIndex >= m_perAPILayout.size())
+            {
+                return nullptr;
+            }
             return m_perAPILayout[m_currentAPITypeIndex].second.get();
         }