diff --git a/Code/Framework/AtomCore/AtomCore/Instance/InstanceData.h b/Code/Framework/AtomCore/AtomCore/Instance/InstanceData.h index 33fc93c0f5..6b07d12e9c 100644 --- a/Code/Framework/AtomCore/AtomCore/Instance/InstanceData.h +++ b/Code/Framework/AtomCore/AtomCore/Instance/InstanceData.h @@ -89,6 +89,9 @@ namespace AZ // Tracks the asset type used to create the instance. AssetType m_assetType; + + // Boolean to indicate if the instance has been orphaned from the instance database + bool m_isOrphaned = false; }; /// @cond EXCLUDE_DOCS diff --git a/Code/Framework/AtomCore/AtomCore/Instance/InstanceDatabase.h b/Code/Framework/AtomCore/AtomCore/Instance/InstanceDatabase.h index ac97af3629..4b4ad572c2 100644 --- a/Code/Framework/AtomCore/AtomCore/Instance/InstanceDatabase.h +++ b/Code/Framework/AtomCore/AtomCore/Instance/InstanceDatabase.h @@ -203,6 +203,16 @@ namespace AZ //! Calls FindOrCreate using a random InstanceId Data::Instance Create(const Asset& asset, const AZStd::any* param = nullptr); + /** + * Removes the instance data from the database. Does not release it. + * References to existing instances will remain valid, but new calls to Create/FindOrCreate will create a new instance + * This function is temporary, to provide functionality needed for Model hot-reloading, but will be removed + * once the Model class does not need it anymore. + * + * @param id The id of the instance to remove + */ + void TEMPOrphan(const InstanceId& id); + private: InstanceDatabase(const AssetType& assetType); ~InstanceDatabase(); @@ -356,6 +366,20 @@ namespace AZ return FindOrCreate(Data::InstanceId::CreateRandom(), asset, param); } + template + void InstanceDatabase::TEMPOrphan(const InstanceId& id) + { + AZStd::scoped_lock lock(m_databaseMutex); + // Check if the instance is still in the database, in case it was orphaned twice + auto instanceItr = m_database.find(id); + if (instanceItr != m_database.end()) + { + // Mark the instance as orphaned, and remove it from the database + instanceItr->second->m_isOrphaned = true; + m_database.erase(instanceItr); + } + } + template void InstanceDatabase::ReleaseInstance(InstanceData* instance, const InstanceId& instanceId) { @@ -374,6 +398,12 @@ namespace AZ m_database.erase(instance->GetId()); m_instanceHandler.m_deleteFunction(static_cast(instance)); } + else if (instance->m_isOrphaned && instance->m_useCount.compare_exchange_strong(expectedRefCount, -1)) + { + // If the instance was orphaned, it has already been removed from the database, + // but still needs to be deleted when the refcount drops to 0 + m_instanceHandler.m_deleteFunction(static_cast(instance)); + } } template diff --git a/Code/Framework/AtomCore/Tests/InstanceDatabase.cpp b/Code/Framework/AtomCore/Tests/InstanceDatabase.cpp index 5d1edc5a09..6a5c65ac7a 100644 --- a/Code/Framework/AtomCore/Tests/InstanceDatabase.cpp +++ b/Code/Framework/AtomCore/Tests/InstanceDatabase.cpp @@ -181,7 +181,76 @@ namespace UnitTest EXPECT_EQ(instance, instance3); } - void ParallelInstanceCreateHelper(size_t threadCountMax, size_t assetIdCount, size_t durationSeconds) + TEST_F(InstanceDatabaseTest, InstanceOrphan) + { + auto& assetManager = AssetManager::Instance(); + auto& instanceDatabase = InstanceDatabase::Instance(); + + Asset someAsset = assetManager.CreateAsset(s_assetId0, AZ::Data::AssetLoadBehavior::Default); + + Instance orphanedInstance = instanceDatabase.FindOrCreate(s_instanceId0, someAsset); + EXPECT_NE(orphanedInstance, nullptr); + + instanceDatabase.TEMPOrphan(s_instanceId0); + // After orphan, the instance should not be found in the database, but it should still be valid + EXPECT_EQ(instanceDatabase.Find(s_instanceId0), nullptr); + EXPECT_NE(orphanedInstance, nullptr); + + instanceDatabase.TEMPOrphan(s_instanceId0); + // Orphaning twice should be a no-op + EXPECT_EQ(instanceDatabase.Find(s_instanceId0), nullptr); + EXPECT_NE(orphanedInstance, nullptr); + + Instance instance2 = instanceDatabase.FindOrCreate(s_instanceId0, someAsset); + // Creating another instance with the same id should return a different instance than the one that was orphaned + EXPECT_NE(orphanedInstance, instance2); + } + + enum class ParallelInstanceTestCases + { + Create, + CreateAndDeferRemoval, + CreateAndOrphan, + CreateDeferRemovalAndOrphan + }; + + enum class ParralleInstanceCurrentAction + { + Create, + DeferredRemoval, + Orphan + }; + + ParralleInstanceCurrentAction ParallelInstanceGetCurrentAction(ParallelInstanceTestCases testCase) + { + switch (testCase) + { + case ParallelInstanceTestCases::CreateAndDeferRemoval: + switch (rand() % 2) + { + case 0: return ParralleInstanceCurrentAction::Create; + case 1: return ParralleInstanceCurrentAction::DeferredRemoval; + } + case ParallelInstanceTestCases::CreateAndOrphan: + switch (rand() % 2) + { + case 0: return ParralleInstanceCurrentAction::Create; + case 1: return ParralleInstanceCurrentAction::Orphan; + } + case ParallelInstanceTestCases::CreateDeferRemovalAndOrphan: + switch (rand() % 3) + { + case 0: return ParralleInstanceCurrentAction::Create; + case 1: return ParralleInstanceCurrentAction::DeferredRemoval; + case 2: return ParralleInstanceCurrentAction::Orphan; + } + case ParallelInstanceTestCases::Create: + default: + return ParralleInstanceCurrentAction::Create; + } + } + + void ParallelInstanceCreateHelper(size_t threadCountMax, size_t assetIdCount, float durationSeconds, ParallelInstanceTestCases testCase) { printf("Testing threads=%zu assetIds=%zu ... ", threadCountMax, assetIdCount); @@ -192,6 +261,7 @@ namespace UnitTest auto& instanceManager = InstanceDatabase::Instance(); AZStd::vector guids; + AZStd::vector> instances; AZStd::vector> assets; for (size_t i = 0; i < assetIdCount; ++i) @@ -199,6 +269,7 @@ namespace UnitTest Uuid guid = Uuid::CreateRandom(); guids.emplace_back(guid); + instances.emplace_back(nullptr); // Pre-create asset so we don't attempt to load it from the catalog. assets.emplace_back(assetManager.CreateAsset(guid, AZ::Data::AssetLoadBehavior::Default)); @@ -206,6 +277,7 @@ namespace UnitTest AZStd::vector threads; AZStd::mutex mutex; + AZStd::mutex referenceTableMutex; AZStd::atomic threadCount((int)threadCountMax); AZStd::condition_variable cv; AZStd::atomic_bool keepDispatching(true); @@ -225,11 +297,15 @@ namespace UnitTest for (size_t i = 0; i < threadCountMax; ++i) { threads.emplace_back( - [&instanceManager, &threadCount, &cv, &guids, &assets, &durationSeconds]() + [&instanceManager, &threadCount, &cv, &guids, &instances, &assets, &durationSeconds, &testCase, &referenceTableMutex]() { AZ::Debug::Timer timer; timer.Stamp(); + bool deferRemoval = testCase == ParallelInstanceTestCases::CreateAndDeferRemoval || + testCase == ParallelInstanceTestCases::CreateDeferRemovalAndOrphan + ? true : false; + while (timer.GetDeltaTimeInSeconds() < durationSeconds) { const size_t index = rand() % guids.size(); @@ -237,11 +313,36 @@ namespace UnitTest const InstanceId instanceId{ uuid }; const AssetId assetId{ uuid }; - Instance instance = - instanceManager.FindOrCreate(instanceId, Asset(assetId, azrtti_typeid())); - EXPECT_NE(instance, nullptr); - EXPECT_EQ(instance->GetId(), instanceId); - EXPECT_EQ(instance->m_asset, assets[index]); + ParralleInstanceCurrentAction currentAction = ParallelInstanceGetCurrentAction(testCase); + + if (currentAction == ParralleInstanceCurrentAction::Orphan) + { + // Orphan the instance, but don't decrease its refcount + instanceManager.TEMPOrphan(instanceId); + } + else if (currentAction == ParralleInstanceCurrentAction::DeferredRemoval) + { + // Drop the refcount to zero so the instance will be released + referenceTableMutex.lock(); + instances[index] = nullptr; + referenceTableMutex.unlock(); + } + else + { + // Otherwise, add a new instance + Instance instance = instanceManager.FindOrCreate(instanceId, assets[index]); + EXPECT_NE(instance, nullptr); + EXPECT_EQ(instance->GetId(), instanceId); + EXPECT_EQ(instance->m_asset, assets[index]); + + if (deferRemoval) + { + // Keep a reference to the instance alive so it can be removed later + referenceTableMutex.lock(); + instances[index] = instance; + referenceTableMutex.unlock(); + } + } } threadCount--; @@ -254,10 +355,12 @@ namespace UnitTest // Used to detect a deadlock. If we wait for more than 10 seconds, it's likely a deadlock has occurred while (threadCount > 0 && !timedOut) { + size_t durationSecondsRoundedUp = static_cast(std::ceil(durationSeconds)); + AZStd::unique_lock lock(mutex); timedOut = (AZStd::cv_status::timeout == - cv.wait_until(lock, AZStd::chrono::system_clock::now() + AZStd::chrono::seconds(durationSeconds * 2))); + cv.wait_until(lock, AZStd::chrono::system_clock::now() + AZStd::chrono::seconds(durationSecondsRoundedUp * 2))); } EXPECT_TRUE(threadCount == 0) << "One or more threads appear to be deadlocked at " << timer.GetDeltaTimeInSeconds() << " seconds"; @@ -273,11 +376,11 @@ namespace UnitTest printf("Took %f seconds\n", timer.GetDeltaTimeInSeconds()); } - TEST_F(InstanceDatabaseTest, ParallelInstanceCreate) + void ParallelCreateTest(ParallelInstanceTestCases testCase) { // This is the original test scenario from when InstanceDatabase was first implemented // threads, AssetIds, seconds - ParallelInstanceCreateHelper(8, 100, 5); + ParallelInstanceCreateHelper(8, 100, 5, testCase); // This value is checked in as 1 so this test doesn't take too much time, but can be increased locally to soak the test. const size_t attempts = 1; @@ -289,11 +392,11 @@ namespace UnitTest // The idea behind this series of tests is that there are two threads sharing one Instance, and both threads try to // create or release that instance at the same time. // At the time, this set of scenarios has something like a 10% failure rate. - const size_t duration = 2; + const float duration = 2.0f; // threads, AssetIds, seconds - ParallelInstanceCreateHelper(2, 1, duration); - ParallelInstanceCreateHelper(4, 1, duration); - ParallelInstanceCreateHelper(8, 1, duration); + ParallelInstanceCreateHelper(2, 1, duration, testCase); + ParallelInstanceCreateHelper(4, 1, duration, testCase); + ParallelInstanceCreateHelper(8, 1, duration, testCase); } for (size_t i = 0; i < attempts; ++i) @@ -301,19 +404,39 @@ namespace UnitTest printf("Attempt %zu of %zu... \n", i, attempts); // Here we try a bunch of different threadCount:assetCount ratios to be thorough - const size_t duration = 2; + const float duration = 2.0f; // threads, AssetIds, seconds - ParallelInstanceCreateHelper(2, 1, duration); - ParallelInstanceCreateHelper(4, 1, duration); - ParallelInstanceCreateHelper(4, 2, duration); - ParallelInstanceCreateHelper(4, 4, duration); - ParallelInstanceCreateHelper(8, 1, duration); - ParallelInstanceCreateHelper(8, 2, duration); - ParallelInstanceCreateHelper(8, 3, duration); - ParallelInstanceCreateHelper(8, 4, duration); + ParallelInstanceCreateHelper(2, 1, duration, testCase); + ParallelInstanceCreateHelper(4, 1, duration, testCase); + ParallelInstanceCreateHelper(4, 2, duration, testCase); + ParallelInstanceCreateHelper(4, 4, duration, testCase); + ParallelInstanceCreateHelper(8, 1, duration, testCase); + ParallelInstanceCreateHelper(8, 2, duration, testCase); + ParallelInstanceCreateHelper(8, 3, duration, testCase); + ParallelInstanceCreateHelper(8, 4, duration, testCase); } } + TEST_F(InstanceDatabaseTest, ParallelInstanceCreate) + { + ParallelCreateTest(ParallelInstanceTestCases::Create); + } + + TEST_F(InstanceDatabaseTest, ParallelInstanceCreateAndDeferRemoval) + { + ParallelCreateTest(ParallelInstanceTestCases::CreateAndDeferRemoval); + } + + TEST_F(InstanceDatabaseTest, ParallelInstanceCreateAndOrphan) + { + ParallelCreateTest(ParallelInstanceTestCases::CreateAndOrphan); + } + + TEST_F(InstanceDatabaseTest, ParallelInstanceCreateDeferRemovalAndOrphan) + { + ParallelCreateTest(ParallelInstanceTestCases::CreateDeferRemovalAndOrphan); + } + TEST_F(InstanceDatabaseTest, InstanceCreateNoDatabase) { bool m_deleted = false;