From b79c2f385d52afa5ff146f41a4ca3508bc12cf65 Mon Sep 17 00:00:00 2001 From: Tommy Walton <82672795+amzn-tommy@users.noreply.github.com> Date: Wed, 29 Sep 2021 11:08:05 -0700 Subject: [PATCH] Adding a temporary Orphan function to the InstanceDatabase (#4297) * Adding a temporary Orphan function to the InstanceDatabase, which will remove an instance from the database so it will not be found using Find or FindOrCreate. The instance will still persist until its use-count drops to 0, at which point it will be deleted. This is to enable the model asset to remove existing buffer/modellod/model instances and replace them with new instances that have the up to date data. Added unit tests for testing. Signed-off-by: amzn-tommy * Fix an incorrect ceil and update ParallelInstance test cases for readability Signed-off-by: amzn-tommy --- .../AtomCore/AtomCore/Instance/InstanceData.h | 3 + .../AtomCore/Instance/InstanceDatabase.h | 30 ++++ .../AtomCore/Tests/InstanceDatabase.cpp | 169 +++++++++++++++--- 3 files changed, 179 insertions(+), 23 deletions(-) diff --git a/Code/Framework/AtomCore/AtomCore/Instance/InstanceData.h b/Code/Framework/AtomCore/AtomCore/Instance/InstanceData.h index 33fc93c0f5..6b07d12e9c 100644 --- a/Code/Framework/AtomCore/AtomCore/Instance/InstanceData.h +++ b/Code/Framework/AtomCore/AtomCore/Instance/InstanceData.h @@ -89,6 +89,9 @@ namespace AZ // Tracks the asset type used to create the instance. AssetType m_assetType; + + // Boolean to indicate if the instance has been orphaned from the instance database + bool m_isOrphaned = false; }; /// @cond EXCLUDE_DOCS diff --git a/Code/Framework/AtomCore/AtomCore/Instance/InstanceDatabase.h b/Code/Framework/AtomCore/AtomCore/Instance/InstanceDatabase.h index ac97af3629..4b4ad572c2 100644 --- a/Code/Framework/AtomCore/AtomCore/Instance/InstanceDatabase.h +++ b/Code/Framework/AtomCore/AtomCore/Instance/InstanceDatabase.h @@ -203,6 +203,16 @@ namespace AZ //! Calls FindOrCreate using a random InstanceId Data::Instance Create(const Asset& asset, const AZStd::any* param = nullptr); + /** + * Removes the instance data from the database. Does not release it. + * References to existing instances will remain valid, but new calls to Create/FindOrCreate will create a new instance + * This function is temporary, to provide functionality needed for Model hot-reloading, but will be removed + * once the Model class does not need it anymore. + * + * @param id The id of the instance to remove + */ + void TEMPOrphan(const InstanceId& id); + private: InstanceDatabase(const AssetType& assetType); ~InstanceDatabase(); @@ -356,6 +366,20 @@ namespace AZ return FindOrCreate(Data::InstanceId::CreateRandom(), asset, param); } + template + void InstanceDatabase::TEMPOrphan(const InstanceId& id) + { + AZStd::scoped_lock lock(m_databaseMutex); + // Check if the instance is still in the database, in case it was orphaned twice + auto instanceItr = m_database.find(id); + if (instanceItr != m_database.end()) + { + // Mark the instance as orphaned, and remove it from the database + instanceItr->second->m_isOrphaned = true; + m_database.erase(instanceItr); + } + } + template void InstanceDatabase::ReleaseInstance(InstanceData* instance, const InstanceId& instanceId) { @@ -374,6 +398,12 @@ namespace AZ m_database.erase(instance->GetId()); m_instanceHandler.m_deleteFunction(static_cast(instance)); } + else if (instance->m_isOrphaned && instance->m_useCount.compare_exchange_strong(expectedRefCount, -1)) + { + // If the instance was orphaned, it has already been removed from the database, + // but still needs to be deleted when the refcount drops to 0 + m_instanceHandler.m_deleteFunction(static_cast(instance)); + } } template diff --git a/Code/Framework/AtomCore/Tests/InstanceDatabase.cpp b/Code/Framework/AtomCore/Tests/InstanceDatabase.cpp index 5d1edc5a09..6a5c65ac7a 100644 --- a/Code/Framework/AtomCore/Tests/InstanceDatabase.cpp +++ b/Code/Framework/AtomCore/Tests/InstanceDatabase.cpp @@ -181,7 +181,76 @@ namespace UnitTest EXPECT_EQ(instance, instance3); } - void ParallelInstanceCreateHelper(size_t threadCountMax, size_t assetIdCount, size_t durationSeconds) + TEST_F(InstanceDatabaseTest, InstanceOrphan) + { + auto& assetManager = AssetManager::Instance(); + auto& instanceDatabase = InstanceDatabase::Instance(); + + Asset someAsset = assetManager.CreateAsset(s_assetId0, AZ::Data::AssetLoadBehavior::Default); + + Instance orphanedInstance = instanceDatabase.FindOrCreate(s_instanceId0, someAsset); + EXPECT_NE(orphanedInstance, nullptr); + + instanceDatabase.TEMPOrphan(s_instanceId0); + // After orphan, the instance should not be found in the database, but it should still be valid + EXPECT_EQ(instanceDatabase.Find(s_instanceId0), nullptr); + EXPECT_NE(orphanedInstance, nullptr); + + instanceDatabase.TEMPOrphan(s_instanceId0); + // Orphaning twice should be a no-op + EXPECT_EQ(instanceDatabase.Find(s_instanceId0), nullptr); + EXPECT_NE(orphanedInstance, nullptr); + + Instance instance2 = instanceDatabase.FindOrCreate(s_instanceId0, someAsset); + // Creating another instance with the same id should return a different instance than the one that was orphaned + EXPECT_NE(orphanedInstance, instance2); + } + + enum class ParallelInstanceTestCases + { + Create, + CreateAndDeferRemoval, + CreateAndOrphan, + CreateDeferRemovalAndOrphan + }; + + enum class ParralleInstanceCurrentAction + { + Create, + DeferredRemoval, + Orphan + }; + + ParralleInstanceCurrentAction ParallelInstanceGetCurrentAction(ParallelInstanceTestCases testCase) + { + switch (testCase) + { + case ParallelInstanceTestCases::CreateAndDeferRemoval: + switch (rand() % 2) + { + case 0: return ParralleInstanceCurrentAction::Create; + case 1: return ParralleInstanceCurrentAction::DeferredRemoval; + } + case ParallelInstanceTestCases::CreateAndOrphan: + switch (rand() % 2) + { + case 0: return ParralleInstanceCurrentAction::Create; + case 1: return ParralleInstanceCurrentAction::Orphan; + } + case ParallelInstanceTestCases::CreateDeferRemovalAndOrphan: + switch (rand() % 3) + { + case 0: return ParralleInstanceCurrentAction::Create; + case 1: return ParralleInstanceCurrentAction::DeferredRemoval; + case 2: return ParralleInstanceCurrentAction::Orphan; + } + case ParallelInstanceTestCases::Create: + default: + return ParralleInstanceCurrentAction::Create; + } + } + + void ParallelInstanceCreateHelper(size_t threadCountMax, size_t assetIdCount, float durationSeconds, ParallelInstanceTestCases testCase) { printf("Testing threads=%zu assetIds=%zu ... ", threadCountMax, assetIdCount); @@ -192,6 +261,7 @@ namespace UnitTest auto& instanceManager = InstanceDatabase::Instance(); AZStd::vector guids; + AZStd::vector> instances; AZStd::vector> assets; for (size_t i = 0; i < assetIdCount; ++i) @@ -199,6 +269,7 @@ namespace UnitTest Uuid guid = Uuid::CreateRandom(); guids.emplace_back(guid); + instances.emplace_back(nullptr); // Pre-create asset so we don't attempt to load it from the catalog. assets.emplace_back(assetManager.CreateAsset(guid, AZ::Data::AssetLoadBehavior::Default)); @@ -206,6 +277,7 @@ namespace UnitTest AZStd::vector threads; AZStd::mutex mutex; + AZStd::mutex referenceTableMutex; AZStd::atomic threadCount((int)threadCountMax); AZStd::condition_variable cv; AZStd::atomic_bool keepDispatching(true); @@ -225,11 +297,15 @@ namespace UnitTest for (size_t i = 0; i < threadCountMax; ++i) { threads.emplace_back( - [&instanceManager, &threadCount, &cv, &guids, &assets, &durationSeconds]() + [&instanceManager, &threadCount, &cv, &guids, &instances, &assets, &durationSeconds, &testCase, &referenceTableMutex]() { AZ::Debug::Timer timer; timer.Stamp(); + bool deferRemoval = testCase == ParallelInstanceTestCases::CreateAndDeferRemoval || + testCase == ParallelInstanceTestCases::CreateDeferRemovalAndOrphan + ? true : false; + while (timer.GetDeltaTimeInSeconds() < durationSeconds) { const size_t index = rand() % guids.size(); @@ -237,11 +313,36 @@ namespace UnitTest const InstanceId instanceId{ uuid }; const AssetId assetId{ uuid }; - Instance instance = - instanceManager.FindOrCreate(instanceId, Asset(assetId, azrtti_typeid())); - EXPECT_NE(instance, nullptr); - EXPECT_EQ(instance->GetId(), instanceId); - EXPECT_EQ(instance->m_asset, assets[index]); + ParralleInstanceCurrentAction currentAction = ParallelInstanceGetCurrentAction(testCase); + + if (currentAction == ParralleInstanceCurrentAction::Orphan) + { + // Orphan the instance, but don't decrease its refcount + instanceManager.TEMPOrphan(instanceId); + } + else if (currentAction == ParralleInstanceCurrentAction::DeferredRemoval) + { + // Drop the refcount to zero so the instance will be released + referenceTableMutex.lock(); + instances[index] = nullptr; + referenceTableMutex.unlock(); + } + else + { + // Otherwise, add a new instance + Instance instance = instanceManager.FindOrCreate(instanceId, assets[index]); + EXPECT_NE(instance, nullptr); + EXPECT_EQ(instance->GetId(), instanceId); + EXPECT_EQ(instance->m_asset, assets[index]); + + if (deferRemoval) + { + // Keep a reference to the instance alive so it can be removed later + referenceTableMutex.lock(); + instances[index] = instance; + referenceTableMutex.unlock(); + } + } } threadCount--; @@ -254,10 +355,12 @@ namespace UnitTest // Used to detect a deadlock. If we wait for more than 10 seconds, it's likely a deadlock has occurred while (threadCount > 0 && !timedOut) { + size_t durationSecondsRoundedUp = static_cast(std::ceil(durationSeconds)); + AZStd::unique_lock lock(mutex); timedOut = (AZStd::cv_status::timeout == - cv.wait_until(lock, AZStd::chrono::system_clock::now() + AZStd::chrono::seconds(durationSeconds * 2))); + cv.wait_until(lock, AZStd::chrono::system_clock::now() + AZStd::chrono::seconds(durationSecondsRoundedUp * 2))); } EXPECT_TRUE(threadCount == 0) << "One or more threads appear to be deadlocked at " << timer.GetDeltaTimeInSeconds() << " seconds"; @@ -273,11 +376,11 @@ namespace UnitTest printf("Took %f seconds\n", timer.GetDeltaTimeInSeconds()); } - TEST_F(InstanceDatabaseTest, ParallelInstanceCreate) + void ParallelCreateTest(ParallelInstanceTestCases testCase) { // This is the original test scenario from when InstanceDatabase was first implemented // threads, AssetIds, seconds - ParallelInstanceCreateHelper(8, 100, 5); + ParallelInstanceCreateHelper(8, 100, 5, testCase); // This value is checked in as 1 so this test doesn't take too much time, but can be increased locally to soak the test. const size_t attempts = 1; @@ -289,11 +392,11 @@ namespace UnitTest // The idea behind this series of tests is that there are two threads sharing one Instance, and both threads try to // create or release that instance at the same time. // At the time, this set of scenarios has something like a 10% failure rate. - const size_t duration = 2; + const float duration = 2.0f; // threads, AssetIds, seconds - ParallelInstanceCreateHelper(2, 1, duration); - ParallelInstanceCreateHelper(4, 1, duration); - ParallelInstanceCreateHelper(8, 1, duration); + ParallelInstanceCreateHelper(2, 1, duration, testCase); + ParallelInstanceCreateHelper(4, 1, duration, testCase); + ParallelInstanceCreateHelper(8, 1, duration, testCase); } for (size_t i = 0; i < attempts; ++i) @@ -301,19 +404,39 @@ namespace UnitTest printf("Attempt %zu of %zu... \n", i, attempts); // Here we try a bunch of different threadCount:assetCount ratios to be thorough - const size_t duration = 2; + const float duration = 2.0f; // threads, AssetIds, seconds - ParallelInstanceCreateHelper(2, 1, duration); - ParallelInstanceCreateHelper(4, 1, duration); - ParallelInstanceCreateHelper(4, 2, duration); - ParallelInstanceCreateHelper(4, 4, duration); - ParallelInstanceCreateHelper(8, 1, duration); - ParallelInstanceCreateHelper(8, 2, duration); - ParallelInstanceCreateHelper(8, 3, duration); - ParallelInstanceCreateHelper(8, 4, duration); + ParallelInstanceCreateHelper(2, 1, duration, testCase); + ParallelInstanceCreateHelper(4, 1, duration, testCase); + ParallelInstanceCreateHelper(4, 2, duration, testCase); + ParallelInstanceCreateHelper(4, 4, duration, testCase); + ParallelInstanceCreateHelper(8, 1, duration, testCase); + ParallelInstanceCreateHelper(8, 2, duration, testCase); + ParallelInstanceCreateHelper(8, 3, duration, testCase); + ParallelInstanceCreateHelper(8, 4, duration, testCase); } } + TEST_F(InstanceDatabaseTest, ParallelInstanceCreate) + { + ParallelCreateTest(ParallelInstanceTestCases::Create); + } + + TEST_F(InstanceDatabaseTest, ParallelInstanceCreateAndDeferRemoval) + { + ParallelCreateTest(ParallelInstanceTestCases::CreateAndDeferRemoval); + } + + TEST_F(InstanceDatabaseTest, ParallelInstanceCreateAndOrphan) + { + ParallelCreateTest(ParallelInstanceTestCases::CreateAndOrphan); + } + + TEST_F(InstanceDatabaseTest, ParallelInstanceCreateDeferRemovalAndOrphan) + { + ParallelCreateTest(ParallelInstanceTestCases::CreateDeferRemovalAndOrphan); + } + TEST_F(InstanceDatabaseTest, InstanceCreateNoDatabase) { bool m_deleted = false;