Model Hot-Reloading (#4304)

* Added a ModelReloader class that handles reloading the hierarchy of assets in the correct order. A ModelReloaderSystem that is used to make sure there is only one ModelReloader at a time for a given asset. Modified the Model, ModelLodAsset, and Buffer assets to not automatically reload, and instead handle reloads manually via AssetCatalog events. Modified the MeshLoader to kick off a reload via the ModelReloaderSystem whenever a model asset is added or changed. Signed-off-by: amzn-tommy <waltont@amazon.com> * Comment updates Signed-off-by: amzn-tommy <waltont@amazon.com> * Minor naming and comment updates based on PR feedback Signed-off-by: amzn-tommy <waltont@amazon.com> * Correcting previous commit. I flipped == to != when switching from count to find, when it should have stayed == Signed-off-by: amzn-tommy <waltont@amazon.com> * Updating RenderCommon header Signed-off-by: amzn-tommy <waltont@amazon.com> * Removing unneeded headers Signed-off-by: amzn-tommy <waltont@amazon.com> * Reverting RayTracingFeatureProcessor change following Doug's guidance. This logic was tricky to get right initially, and leads to TDR when incorrect, so leaving as is. Signed-off-by: amzn-tommy <waltont@amazon.com> * Removing a tab Signed-off-by: amzn-tommy <waltont@amazon.com> * Added missing #include for AssetId Signed-off-by: amzn-tommy <waltont@amazon.com> * Adding missing RTTI header Signed-off-by: amzn-tommy <waltont@amazon.com> * Include ModelAsset definition intead of forward declaring it to avoid a static assert on Linux Signed-off-by: amzn-tommy <waltont@amazon.com>
2021-10-05 09:59:13 -07:00
parent aef1f01de2
commit 256564a3e0
15 changed files with 532 additions and 8 deletions
@@ -14,10 +14,14 @@
 #include <Atom/RPI.Public/Shader/ShaderSystemInterface.h>
 #include <Atom/Feature/Material/MaterialAssignment.h>
 #include <Atom/Feature/TransformService/TransformServiceFeatureProcessor.h>
+#include <Atom/Feature/Mesh/ModelReloaderSystemInterface.h>
 #include <RayTracing/RayTracingFeatureProcessor.h>
 #include <AzCore/Asset/AssetCommon.h>
 #include <AtomCore/std/parallel/concurrency_checker.h>
 #include <AzCore/Console/Console.h>
+#include <AzFramework/Asset/AssetCatalogBus.h>
+
+#include <AzCore/Component/TickBus.h>

 namespace AZ
 {
@@ -38,6 +42,7 @@ namespace AZ
        private:
            class MeshLoader
                : private Data::AssetBus::Handler
+                , private AzFramework::AssetCatalogEventBus::Handler
            {
            public:
                using ModelChangedEvent = MeshFeatureProcessorInterface::ModelChangedEvent;
@@ -52,6 +57,15 @@ namespace AZ
                void OnAssetReady(Data::Asset<Data::AssetData> asset) override;
                void OnAssetError(Data::Asset<Data::AssetData> asset) override;

+                // AssetCatalogEventBus::Handler overrides...
+                void OnCatalogAssetChanged(const AZ::Data::AssetId& assetId) override;
+                void OnCatalogAssetAdded(const AZ::Data::AssetId& assetId) override;
+
+                void OnModelReloaded(Data::Asset<Data::AssetData> asset);
+                ModelReloadedEvent::Handler m_modelReloadedEventHandler { [&](Data::Asset<RPI::ModelAsset> modelAsset)
+                                                                  {
+                                                                             OnModelReloaded(modelAsset);
+                                                                  } };
                MeshFeatureProcessorInterface::ModelChangedEvent m_modelChangedEvent;
                Data::Asset<RPI::ModelAsset> m_modelAsset;
                MeshDataInstance* m_parent = nullptr;
@@ -61,6 +75,7 @@ namespace AZ
            void Init(Data::Instance<RPI::Model> model);
            void BuildDrawPacketList(size_t modelLodIndex);
            void SetRayTracingData();
+            void RemoveRayTracingData();
            void SetSortKey(RHI::DrawItemSortKey sortKey);
            RHI::DrawItemSortKey GetSortKey() const;
            void SetMeshLodConfiguration(RPI::Cullable::LodConfiguration meshLodConfig);
@@ -0,0 +1,57 @@
+/*
+ * Copyright (c) Contributors to the Open 3D Engine Project.
+ * For complete copyright and license terms please see the LICENSE at the root of this distribution.
+ *
+ * SPDX-License-Identifier: Apache-2.0 OR MIT
+ *
+ */
+
+#pragma once
+
+#include <Atom/RPI.Reflect/Model/ModelAsset.h>
+#include <AzCore/EBus/Event.h>
+#include <AzCore/Interface/Interface.h>
+#include <AzCore/RTTI/RTTI.h>
+
+namespace AZ
+{
+    namespace Data
+    {
+        template<class T>
+        class Asset;
+    }
+
+    namespace Render
+    {
+        using ModelReloadedEvent = Event<const Data::Asset<RPI::ModelAsset>>;
+
+        //! A system that handles reloading the hierarchy of model assets in the correct order
+        class ModelReloaderSystemInterface
+        {
+        public:
+            AZ_RTTI(AZ::Render::ModelReloaderSystemInterface, "{E7E05B1F-8928-4A1B-B75D-3D5433E65BCA}");
+
+            ModelReloaderSystemInterface()
+            {
+                Interface<ModelReloaderSystemInterface>::Register(this);
+            }
+
+            virtual ~ModelReloaderSystemInterface()
+            {
+                Interface<ModelReloaderSystemInterface>::Unregister(this);
+            }
+
+            static ModelReloaderSystemInterface* Get()
+            {
+                return Interface<ModelReloaderSystemInterface>::Get();
+            }
+
+            //! Requests a model reload and passes in a callback event handler for when the reload is finished
+            virtual void ReloadModel(
+                Data::Asset<RPI::ModelAsset> modelAsset, ModelReloadedEvent::Handler& onReloadedEventHandler) = 0;
+
+            // Note that you have to delete these for safety reasons, you will trip a static_assert if you do not
+            AZ_DISABLE_COPY_MOVE(ModelReloaderSystemInterface);
+        };
+    } // namespace Render
+} // namespace AZ
@@ -103,11 +103,15 @@
 #include <ReflectionScreenSpace/ReflectionScreenSpaceCompositePass.h>
 #include <ReflectionScreenSpace/ReflectionCopyFrameBufferPass.h>
 #include <OcclusionCullingPlane/OcclusionCullingPlaneFeatureProcessor.h>
+#include <Mesh/ModelReloaderSystem.h>

 namespace AZ
 {
    namespace Render
    {
+        CommonSystemComponent::CommonSystemComponent() = default;
+        CommonSystemComponent::~CommonSystemComponent() = default;
+
        void CommonSystemComponent::Reflect(ReflectContext* context)
        {
            AuxGeomFeatureProcessor::Reflect(context);
@@ -292,10 +296,13 @@ namespace AZ
            // setup handler for load pass template mappings
            m_loadTemplatesHandler = RPI::PassSystemInterface::OnReadyLoadTemplatesEvent::Handler([this]() { this->LoadPassTemplateMappings(); });
            RPI::PassSystemInterface::Get()->ConnectEvent(m_loadTemplatesHandler);
+            
+            m_modelReloaderSystem = AZStd::make_unique<ModelReloaderSystem>();
        }

        void CommonSystemComponent::Deactivate()
        {
+            m_modelReloaderSystem.reset();
            m_loadTemplatesHandler.Disconnect();
            AZ::RPI::FeatureProcessorFactory::Get()->UnregisterFeatureProcessor<RayTracingFeatureProcessor>();
            AZ::RPI::FeatureProcessorFactory::Get()->UnregisterFeatureProcessor<DiffuseGlobalIlluminationFeatureProcessor>();
@@ -20,12 +20,17 @@ namespace AZ
 {
    namespace Render
    {
+        class ModelReloaderSystem;
+
        class CommonSystemComponent
            : public AZ::Component
        {
        public:
            AZ_COMPONENT(CommonSystemComponent, "{BFB8FE2B-C952-4D0C-8E32-4FE7C7A97757}");

+            CommonSystemComponent();
+            ~CommonSystemComponent();
+
            static void Reflect(AZ::ReflectContext* context);

            static void GetProvidedServices(AZ::ComponentDescriptor::DependencyArrayType& provided);
@@ -44,6 +49,8 @@ namespace AZ

            RPI::PassSystemInterface::OnReadyLoadTemplatesEvent::Handler m_loadTemplatesHandler;

+            AZStd::unique_ptr<ModelReloaderSystem> m_modelReloaderSystem;
+
 #if AZ_TRAIT_LUXCORE_SUPPORTED
            // LuxCore
            LuxCoreRenderer m_luxCore;
@@ -10,6 +10,7 @@
 #include <Atom/RHI.Reflect/InputStreamLayoutBuilder.h>
 #include <Atom/Feature/RenderCommon.h>
 #include <Atom/Feature/Mesh/MeshFeatureProcessor.h>
+#include <Atom/Feature/Mesh/ModelReloaderSystemInterface.h>
 #include <Atom/Feature/ReflectionProbe/ReflectionProbeFeatureProcessor.h>
 #include <Atom/RPI.Public/Model/ModelLodUtils.h>
 #include <Atom/RPI.Public/Scene.h>
@@ -18,6 +19,8 @@

 #include <Atom/RPI.Reflect/Model/ModelAssetCreator.h>

+#include <AzFramework/Asset/AssetSystemBus.h>
+
 #include <AtomCore/Instance/InstanceDatabase.h>

 #include <AzCore/Console/IConsole.h>
@@ -175,6 +178,7 @@ namespace AZ
        {
            if (meshHandle.IsValid())
            {
+                meshHandle->m_meshLoader.reset();
                meshHandle->DeInit();
                m_transformService->ReleaseObjectId(meshHandle->m_objectId);

@@ -487,10 +491,12 @@ namespace AZ
            }

            Data::AssetBus::Handler::BusConnect(modelAsset.GetId());
+            AzFramework::AssetCatalogEventBus::Handler::BusConnect();
        }

        MeshDataInstance::MeshLoader::~MeshLoader()
        {
+            AzFramework::AssetCatalogEventBus::Handler::BusDisconnect();
            Data::AssetBus::Handler::BusDisconnect();
        }

@@ -533,6 +539,7 @@ namespace AZ
            
            if (model)
            {
+                m_parent->RemoveRayTracingData();
                m_parent->Init(model);
                m_modelChangedEvent.Signal(AZStd::move(model));
            }
@@ -545,10 +552,51 @@ namespace AZ
            }
        }

+        
+        void MeshDataInstance::MeshLoader::OnModelReloaded(Data::Asset<Data::AssetData> asset)
+        {
+            OnAssetReady(asset);
+        }
+
        void MeshDataInstance::MeshLoader::OnAssetError(Data::Asset<Data::AssetData> asset)
        {
            // Note: m_modelAsset and asset represents same asset, but only m_modelAsset contains the file path in its hint from serialization
-            AZ_Error("MeshDataInstance::MeshLoader", false, "Failed to load asset %s.", m_modelAsset.GetHint().c_str()); 
+            AZ_Error(
+                "MeshDataInstance::MeshLoader", false, "Failed to load asset %s. It may be missing, or not be finished processing",
+                m_modelAsset.GetHint().c_str());
+
+            AzFramework::AssetSystemRequestBus::Broadcast(
+                &AzFramework::AssetSystem::AssetSystemRequests::EscalateAssetByUuid, m_modelAsset.GetId().m_guid);
+        }
+        
+        void MeshDataInstance::MeshLoader::OnCatalogAssetChanged(const AZ::Data::AssetId& assetId)
+        {
+            if (assetId == m_modelAsset.GetId())
+            {
+                Data::Asset<RPI::ModelAsset> modelAssetReference = m_modelAsset;
+
+                // If the asset was modified, reload it
+                AZ::SystemTickBus::QueueFunction(
+                    [=]() mutable
+                    {
+                        ModelReloaderSystemInterface::Get()->ReloadModel(modelAssetReference, m_modelReloadedEventHandler);
+                    });
+            }
+        }
+
+        void MeshDataInstance::MeshLoader::OnCatalogAssetAdded(const AZ::Data::AssetId& assetId)
+        {
+            if (assetId == m_modelAsset.GetId())
+            {
+                Data::Asset<RPI::ModelAsset> modelAssetReference = m_modelAsset;
+                
+                // If the asset didn't exist in the catalog when it first attempted to load, we need to try loading it again
+                AZ::SystemTickBus::QueueFunction(
+                    [=]() mutable
+                    {
+                        ModelReloaderSystemInterface::Get()->ReloadModel(modelAssetReference, m_modelReloadedEventHandler);
+                    });
+            }
        }

        // MeshDataInstance...
@@ -557,14 +605,8 @@ namespace AZ
        {
            m_scene->GetCullingScene()->UnregisterCullable(m_cullable);

-            // remove from ray tracing
-            RayTracingFeatureProcessor* rayTracingFeatureProcessor = m_scene->GetFeatureProcessor<RayTracingFeatureProcessor>();
-            if (rayTracingFeatureProcessor)
-            {
-                rayTracingFeatureProcessor->RemoveMesh(m_objectId);
-            }
+            RemoveRayTracingData();

-            m_meshLoader.reset();
            m_drawPacketListsByLod.clear();
            m_materialAssignments.clear();
            m_shaderResourceGroup = {};
@@ -951,6 +993,16 @@ namespace AZ
            rayTracingFeatureProcessor->SetMesh(m_objectId, m_model->GetModelAsset()->GetId(), subMeshes);
        }

+        void MeshDataInstance::RemoveRayTracingData()
+        {
+            // remove from ray tracing
+            RayTracingFeatureProcessor* rayTracingFeatureProcessor = m_scene->GetFeatureProcessor<RayTracingFeatureProcessor>();
+            if (rayTracingFeatureProcessor)
+            {
+                rayTracingFeatureProcessor->RemoveMesh(m_objectId);
+            }
+        }
+
        void MeshDataInstance::SetSortKey(RHI::DrawItemSortKey sortKey)
        {
            m_sortKey = sortKey;
@@ -0,0 +1,175 @@
+/*
+ * Copyright (c) Contributors to the Open 3D Engine Project.
+ * For complete copyright and license terms please see the LICENSE at the root of this distribution.
+ *
+ * SPDX-License-Identifier: Apache-2.0 OR MIT
+ *
+ */
+
+#include <Source/Mesh/ModelReloader.h>
+#include <Atom/RPI.Reflect/Model/ModelAsset.h>
+#include <Atom/RPI.Public/Model/Model.h>
+
+namespace AZ
+{
+    namespace Render
+    {
+        ModelReloader::ModelReloader(
+            Data::Asset<RPI::ModelAsset> modelAsset, RemoveModelFromReloaderSystemEvent::Handler& removeReloaderFromSystemHandler)
+        {
+            m_modelAsset.push_back(modelAsset);
+            m_pendingDependencyListStatus.reset();
+            removeReloaderFromSystemHandler.Connect(m_onRemoveReloaderFromSystem);
+
+            // Iterate over the model and track the assets that need to be reloaded
+            for (auto& modelLodAsset : modelAsset->GetLodAssets())
+            {
+                for (auto& mesh : modelLodAsset->GetMeshes())
+                {
+                    for (auto& streamBufferInfo : mesh.GetStreamBufferInfoList())
+                    {
+                        InsertMeshDependencyIfUnique(streamBufferInfo.m_bufferAssetView.GetBufferAsset());
+                        
+                    }
+                    InsertMeshDependencyIfUnique(mesh.GetIndexBufferAssetView().GetBufferAsset());
+                }
+                m_modelDependencies.push_back(modelLodAsset);
+            }
+
+            AZ_Assert(
+                m_meshDependencies.size() <= m_pendingDependencyListStatus.size(),
+                "There are more buffers used by the model %s than are supported by the ModelReloader.", modelAsset.GetHint().c_str());
+
+            m_state = State::WaitingForMeshDependencies;
+            ReloadDependenciesAndWait();
+        }
+
+        void ModelReloader::ConnectOnReloadedEventHandler(ModelReloadedEvent::Handler& onReloadedEventHandler)
+        {
+            onReloadedEventHandler.Connect(m_onModelReloaded);
+        }
+
+        void ModelReloader::OnAssetReloaded(AZ::Data::Asset<AZ::Data::AssetData> asset)
+        {
+            DependencyList& pendingDependencies = GetPendingDependencyList();
+
+            const Data::AssetId& reloadedAssetId = asset.GetId();
+
+            // Find the index of the asset that was reloaded
+            const auto matchesId = [reloadedAssetId](const Data::Asset<Data::AssetData>& asset){ return asset.GetId() == reloadedAssetId;};
+            const auto& iter = AZStd::find_if(AZStd::begin(pendingDependencies), AZStd::end(pendingDependencies), matchesId);
+            AZ_Assert(
+                iter != AZStd::end(pendingDependencies),
+                "ModelReloader - handling an AssetReloaded event for an asset that is not part of the dependency list.");
+            size_t currentIndex = AZStd::distance(AZStd::begin(pendingDependencies), iter);
+
+            // Keep a reference to the newly reloaded asset to prevent it from being immediately released
+            pendingDependencies[currentIndex] = asset;
+            Data::AssetBus::MultiHandler::BusDisconnect(reloadedAssetId);
+
+            // Clear the bit, now that it has been reloaded
+            m_pendingDependencyListStatus.reset(currentIndex);
+
+            if (m_pendingDependencyListStatus.none())
+            {
+                AdvanceToNextLevelOfHierarchy();
+            }
+        }
+
+        void ModelReloader::OnAssetReloadError(Data::Asset<Data::AssetData> asset)
+        {
+            // An error is actually okay/expected in some situations.
+            // For example, if the 2nd UV set was removed, and we tried to reload the second uv set, the reload would fail.
+            // We want to treat it as a success, and mark that dependency as 'up to date'
+            OnAssetReloaded(asset);
+        }
+
+        void ModelReloader::InsertMeshDependencyIfUnique(Data::Asset<Data::AssetData> asset)
+        {
+            if (AZStd::find(AZStd::begin(m_meshDependencies), AZStd::end(m_meshDependencies), asset) == AZStd::end(m_meshDependencies))
+            {
+                // Multiple meshes may reference the same buffer, so only add the dependency if it is unique
+                m_meshDependencies.push_back(asset);
+            }
+        }
+
+        void ModelReloader::ReloadDependenciesAndWait()
+        {
+            // Get the current list of dependencies depending on the current state
+            DependencyList& dependencies = GetPendingDependencyList();
+
+            if (!m_pendingDependencyListStatus.none())
+            {
+                AZ_Assert(
+                    m_pendingDependencyListStatus.none(),
+                    "ModelReloader attempting to add new dependencies while still waiting for other dependencies in the hierarchy to "
+                    "load.");
+            }
+            if (dependencies.empty())
+            {
+                // If the original model asset failed to load, it won't have any dependencies to reload
+                AdvanceToNextLevelOfHierarchy();
+            }
+            AZ_Assert(
+                dependencies.size() <= m_pendingDependencyListStatus.size(),
+                "ModelReloader has more dependencies than can fit in the bitset. The size of m_pendingDependencyListStatus needs to be increased.");
+
+            // Set all bits to 1
+            m_pendingDependencyListStatus.set();
+            // Clear the least significant n-bits
+            m_pendingDependencyListStatus <<= dependencies.size();
+            // Set the least significant n-bits to 1, and the rest to 0
+            m_pendingDependencyListStatus.flip();
+
+            // Reload all the assets
+            for (Data::Asset<Data::AssetData>& dependencyAsset : dependencies)
+            {
+                Data::AssetBus::MultiHandler::BusConnect(dependencyAsset.GetId());
+                dependencyAsset.Reload();
+            }
+        }
+
+        void ModelReloader::AdvanceToNextLevelOfHierarchy()
+        {
+            switch (m_state)
+            {
+                case State::WaitingForMeshDependencies:
+                    m_state = State::WaitingForModelDependencies;
+                    ReloadDependenciesAndWait();
+                    break;
+                case State::WaitingForModelDependencies:
+                    m_state = State::WaitingForModel;
+                    ReloadDependenciesAndWait();
+                    break;
+                case State::WaitingForModel:
+                    Data::AssetBus::MultiHandler::BusDisconnect();
+                    // Since the model asset is finished reloading, orphan model from the instance database
+                    // so that all of the buffer instances are re-created with the latest data
+                    RPI::Model::TEMPOrphanFromDatabase(m_modelAsset.front());
+                    // Signal that the model is ready
+                    m_onModelReloaded.Signal(m_modelAsset.front());
+                    // Remove this reloader from the ModelReloaderSystem
+                    m_onRemoveReloaderFromSystem.Signal(m_modelAsset.front().GetId());
+                    delete this;
+                    break;
+            }
+        }
+
+        ModelReloader::DependencyList& ModelReloader::GetPendingDependencyList()
+        {
+            switch (m_state)
+            {
+                case State::WaitingForMeshDependencies:
+                    return m_meshDependencies;
+                    break;
+                case State::WaitingForModelDependencies:
+                    return m_modelDependencies;
+                    break;
+                case State::WaitingForModel:
+                default:
+                    return m_modelAsset;
+                    break;
+            }
+        }
+    } // namespace Render
+} // namespace AZ
@@ -0,0 +1,74 @@
+/*
+ * Copyright (c) Contributors to the Open 3D Engine Project.
+ * For complete copyright and license terms please see the LICENSE at the root of this distribution.
+ *
+ * SPDX-License-Identifier: Apache-2.0 OR MIT
+ *
+ */
+
+#pragma once
+
+#include <AzCore/Asset/AssetCommon.h>
+#include <AzCore/RTTI/TypeInfo.h>
+#include <Atom/Feature/Mesh/ModelReloaderSystemInterface.h>
+#include <Source/Mesh/ModelReloaderSystem.h>
+
+namespace AZ
+{
+    namespace RPI
+    {
+        class ModelAsset;
+    }
+
+    namespace Render
+    {
+        //! ModelReloader takes care of reloading Buffer, ModelLod, and Model assets in the correct order
+        //! The ModelReloaderSystem should be used to reload a model, rather than using a ModelReloader directly
+        class ModelReloader
+            : private Data::AssetBus::MultiHandler
+        {
+            using DependencyList = AZStd::vector<Data::Asset<Data::AssetData>>;
+        public:
+            AZ_RTTI(AZ::Render::ModelReloader, "{99B75A6A-62B6-490A-9953-029BE7D69452}");
+
+            ModelReloader() = default;
+
+            //! Reload a model asset
+            //! @param modelAsset - the asset to be reloaded
+            //! @param removeReloaderFromSystemHandler - an event that will tell the ModelReloaderSystem when to remove the reloader because it is finished
+            ModelReloader(Data::Asset<RPI::ModelAsset> modelAsset, RemoveModelFromReloaderSystemEvent::Handler& removeReloaderFromSystemHandler);
+
+            //! Connects a handler that will handle an event when the model is finished reloading
+            void ConnectOnReloadedEventHandler(ModelReloadedEvent::Handler& onReloadedEventHandler);
+
+        private:
+            enum class State
+            {
+                WaitingForMeshDependencies,
+                WaitingForModelDependencies,
+                WaitingForModel
+            };
+
+            // Data::AssetBus::MultiHandler overrides...
+            void OnAssetReloaded(AZ::Data::Asset<AZ::Data::AssetData> asset) override;            
+            void OnAssetReloadError(Data::Asset<Data::AssetData> asset) override;
+
+            void InsertMeshDependencyIfUnique(Data::Asset<Data::AssetData> asset);
+            void ReloadDependenciesAndWait();
+            void AdvanceToNextLevelOfHierarchy();
+            DependencyList& GetPendingDependencyList();
+
+            ModelReloadedEvent m_onModelReloaded;
+            RemoveModelFromReloaderSystemEvent m_onRemoveReloaderFromSystem;
+
+            // Keep track of all the asset references for each level of the hierarchy
+            DependencyList m_modelAsset;
+            DependencyList m_meshDependencies;
+            DependencyList m_modelDependencies;
+            
+            AZStd::bitset<1024> m_pendingDependencyListStatus;
+            State m_state;
+        };
+
+    } // namespace Render
+} // namespace AZ
@@ -0,0 +1,38 @@
+/*
+ * Copyright (c) Contributors to the Open 3D Engine Project.
+ * For complete copyright and license terms please see the LICENSE at the root of this distribution.
+ *
+ * SPDX-License-Identifier: Apache-2.0 OR MIT
+ *
+ */
+
+#include <Source/Mesh/ModelReloaderSystem.h>
+#include <Source/Mesh/ModelReloader.h>
+#include <AzCore/std/parallel/scoped_lock.h>
+
+namespace AZ
+{
+    namespace Render
+    {
+        void ModelReloaderSystem::ReloadModel(Data::Asset<RPI::ModelAsset> modelAsset, ModelReloadedEvent::Handler& onReloadedEventHandler)
+        {
+            AZStd::scoped_lock lock(m_pendingReloadMutex);
+            if (m_pendingReloads.find(modelAsset.GetId()) == m_pendingReloads.end())
+            {
+                ModelReloader* reloader = new ModelReloader(modelAsset, m_removeModelHandler);
+                m_pendingReloads[modelAsset.GetId()] = reloader;
+            }
+
+            m_pendingReloads[modelAsset.GetId()]->ConnectOnReloadedEventHandler(onReloadedEventHandler);
+        }
+
+        void ModelReloaderSystem::RemoveReloader(const Data::AssetId& assetId)
+        {
+            AZStd::scoped_lock lock(m_pendingReloadMutex);
+            // We don't delete the ModelReloader here, because its in the middle of signaling this RemoveReloader event.
+            // We only remove it from the pending reloads here.
+            // The ModelReloader will delete itself after it finishes firing this event.
+            m_pendingReloads.erase(assetId);
+        }
+    } // namespace Render
+} // namespace AZ
@@ -0,0 +1,50 @@
+/*
+ * Copyright (c) Contributors to the Open 3D Engine Project.
+ * For complete copyright and license terms please see the LICENSE at the root of this distribution.
+ *
+ * SPDX-License-Identifier: Apache-2.0 OR MIT
+ *
+ */
+
+#pragma once
+
+#include <Atom/Feature/Mesh/ModelReloaderSystemInterface.h>
+
+#include <AzCore/Asset/AssetCommon.h>
+#include <AzCore/RTTI/RTTI.h>
+#include <AzCore/std/containers/unordered_map.h>
+
+namespace AZ
+{
+    namespace Render
+    {
+        class ModelReloader;
+
+        using RemoveModelFromReloaderSystemEvent = Event<const Data::AssetId&>;
+        
+        class ModelReloaderSystem
+            : public ModelReloaderSystemInterface
+        {
+        public:
+            AZ_RTTI(Render::ModelReloaderSystem, "{8C85ECCD-B6C8-4949-B26C-9C4F1020F2B8}", Render::ModelReloaderSystemInterface);
+            
+            void ReloadModel(Data::Asset<RPI::ModelAsset> modelAsset, ModelReloadedEvent::Handler& onReloadedEventHandler) override;
+
+        private:
+            void RemoveReloader(const Data::AssetId& assetId);
+
+            // Keep track of all the pending reloads so there are no duplicates
+            AZStd::unordered_map<Data::AssetId, ModelReloader*> m_pendingReloads;
+            AZStd::mutex m_pendingReloadMutex;
+
+            RemoveModelFromReloaderSystemEvent::Handler m_removeModelHandler{
+                [&](const Data::AssetId& assetId)
+                {
+                    RemoveReloader(assetId);
+                } };
+
+            friend class ModelReloader;
+        };
+
+    } // namespace Render
+} // namespace AZ
@@ -26,6 +26,7 @@ set(FILES
    Include/Atom/Feature/ImageBasedLights/ImageBasedLightFeatureProcessor.h
    Include/Atom/Feature/LookupTable/LookupTableAsset.h
    Include/Atom/Feature/Mesh/MeshFeatureProcessor.h
+    Include/Atom/Feature/Mesh/ModelReloaderSystemInterface.h
    Include/Atom/Feature/PostProcessing/PostProcessingConstants.h
    Include/Atom/Feature/PostProcessing/SMAAFeatureProcessorInterface.h
    Include/Atom/Feature/PostProcess/PostFxLayerCategoriesConstants.h
@@ -169,6 +170,10 @@ set(FILES
    Source/Math/MathFilter.cpp
    Source/Math/MathFilterDescriptor.h
    Source/Mesh/MeshFeatureProcessor.cpp
+    Source/Mesh/ModelReloader.cpp
+    Source/Mesh/ModelReloader.h
+    Source/Mesh/ModelReloaderSystem.cpp
+    Source/Mesh/ModelReloaderSystem.h
    Source/MorphTargets/MorphTargetComputePass.cpp
    Source/MorphTargets/MorphTargetComputePass.h
    Source/MorphTargets/MorphTargetDispatchItem.cpp
@@ -37,6 +37,10 @@ namespace AZ

            static Data::Instance<Model> FindOrCreate(const Data::Asset<ModelAsset>& modelAsset);

+            //! Orphan the model, its lods, and all their buffers so that they can be replaced in the instance database
+            //! This is a temporary function, that will be removed once the Model/ModelAsset classes no longer need it
+            static void TEMPOrphanFromDatabase(const Data::Asset<ModelAsset>& modelAsset);
+
            ~Model() = default;

            //! Blocks the CPU until the streaming upload is complete. Returns immediately if no
@@ -60,6 +60,12 @@ namespace AZ
            const AZStd::string& GetName() const;

        private:
+            // AssetData overrides...
+            bool HandleAutoReload() override
+            {
+                return false;
+            }
+
            // Called by asset creators to assign the asset to a ready state.
            void SetReady();

@@ -77,6 +77,12 @@ namespace AZ
                float& distanceNormalized, AZ::Vector3& normal) const;

        private:
+            // AssetData overrides...
+            bool HandleAutoReload() override
+            {
+                return false;
+            }
+
            void SetReady();

            AZ::Name m_name;
@@ -149,6 +149,12 @@ namespace AZ
            const AZ::Aabb& GetAabb() const;

        private:
+            // AssetData overrides...
+            bool HandleAutoReload() override
+            {
+                return false;
+            }
+            
            AZStd::vector<Mesh> m_meshes;
            AZ::Aabb m_aabb = AZ::Aabb::CreateNull();
            
@@ -31,6 +31,28 @@ namespace AZ
                modelAsset);
        }

+        
+        void Model::TEMPOrphanFromDatabase(const Data::Asset<ModelAsset>& modelAsset)
+        {
+            for (auto& modelLodAsset : modelAsset->GetLodAssets())
+            {
+                for(auto& mesh : modelLodAsset->GetMeshes())
+                {
+                    for (auto& streamBufferInfo : mesh.GetStreamBufferInfoList())
+                    {
+                        Data::InstanceDatabase<Buffer>::Instance().TEMPOrphan(
+                            Data::InstanceId::CreateFromAssetId(streamBufferInfo.m_bufferAssetView.GetBufferAsset().GetId()));
+                    }
+                    Data::InstanceDatabase<Buffer>::Instance().TEMPOrphan(
+                        Data::InstanceId::CreateFromAssetId(mesh.GetIndexBufferAssetView().GetBufferAsset().GetId()));
+                }
+                Data::InstanceDatabase<ModelLod>::Instance().TEMPOrphan(Data::InstanceId::CreateFromAssetId(modelLodAsset.GetId()));
+            }
+
+            Data::InstanceDatabase<Model>::Instance().TEMPOrphan(
+                Data::InstanceId::CreateFromAssetId(modelAsset.GetId()));
+        }
+
        size_t Model::GetLodCount() const
        {
            return m_lods.size();