birkeh/o3de
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Adding a temporary Orphan function to the InstanceDatabase (#4297)

Browse Source 
* Adding a temporary Orphan function to the InstanceDatabase, which will remove an instance from the database so it will not be found using Find or FindOrCreate. The instance will still persist until its use-count drops to 0, at which point it will be deleted. This is to enable the model asset to remove existing buffer/modellod/model instances and replace them with new instances that have the up to date data. Added unit tests for testing.

Signed-off-by: amzn-tommy <waltont@amazon.com>

* Fix an incorrect ceil and update ParallelInstance test cases for readability

Signed-off-by: amzn-tommy <waltont@amazon.com>
This commit is contained in:
Tommy Walton
2021-09-29 11:08:05 -07:00
committed by GitHub
parent a8559c4000
commit b79c2f385d
3 changed files with 179 additions and 23 deletions
Download Patch File Download Diff File Expand all files Collapse all files
Code/Framework/AtomCore/AtomCore/Instance/InstanceData.h
+3
View File
@@ -89,6 +89,9 @@ namespace AZ
// Tracks the asset type used to create the instance.
AssetType m_assetType;
// Boolean to indicate if the instance has been orphaned from the instance database
bool m_isOrphaned = false;
};
/// @cond EXCLUDE_DOCS
Code/Framework/AtomCore/AtomCore/Instance/InstanceDatabase.h
+30
View File
@@ -203,6 +203,16 @@ namespace AZ
//! Calls FindOrCreate using a random InstanceId
Data::Instance<Type> Create(const Asset<AssetData>& asset, const AZStd::any* param = nullptr);
/**
* Removes the instance data from the database. Does not release it.
* References to existing instances will remain valid, but new calls to Create/FindOrCreate will create a new instance
* This function is temporary, to provide functionality needed for Model hot-reloading, but will be removed
* once the Model class does not need it anymore.
*
* @param id The id of the instance to remove
*/
void TEMPOrphan(const InstanceId& id);
private:
InstanceDatabase(const AssetType& assetType);
~InstanceDatabase();
@@ -356,6 +366,20 @@ namespace AZ
return FindOrCreate(Data::InstanceId::CreateRandom(), asset, param);
}
template<typename Type>
void InstanceDatabase<Type>::TEMPOrphan(const InstanceId& id)
{
AZStd::scoped_lock<AZStd::recursive_mutex> lock(m_databaseMutex);
// Check if the instance is still in the database, in case it was orphaned twice
auto instanceItr = m_database.find(id);
if (instanceItr != m_database.end())
{
// Mark the instance as orphaned, and remove it from the database
instanceItr->second->m_isOrphaned = true;
m_database.erase(instanceItr);
}
}
template<typename Type>
void InstanceDatabase<Type>::ReleaseInstance(InstanceData* instance, const InstanceId& instanceId)
{
@@ -374,6 +398,12 @@ namespace AZ
m_database.erase(instance->GetId());
m_instanceHandler.m_deleteFunction(static_cast<Type*>(instance));
}
else if (instance->m_isOrphaned && instance->m_useCount.compare_exchange_strong(expectedRefCount, -1))
{
// If the instance was orphaned, it has already been removed from the database,
// but still needs to be deleted when the refcount drops to 0
m_instanceHandler.m_deleteFunction(static_cast<Type*>(instance));
}
}
template<typename Type>
Code/Framework/AtomCore/Tests/InstanceDatabase.cpp
+146 -23
View File
@@ -181,7 +181,76 @@ namespace UnitTest
EXPECT_EQ(instance, instance3);
}
void ParallelInstanceCreateHelper(size_t threadCountMax, size_t assetIdCount, size_t durationSeconds)
TEST_F(InstanceDatabaseTest, InstanceOrphan)
{
auto& assetManager = AssetManager::Instance();
auto& instanceDatabase = InstanceDatabase<TestInstanceA>::Instance();
Asset<TestAssetType> someAsset = assetManager.CreateAsset<TestAssetType>(s_assetId0, AZ::Data::AssetLoadBehavior::Default);
Instance<TestInstanceA> orphanedInstance = instanceDatabase.FindOrCreate(s_instanceId0, someAsset);
EXPECT_NE(orphanedInstance, nullptr);
instanceDatabase.TEMPOrphan(s_instanceId0);
// After orphan, the instance should not be found in the database, but it should still be valid
EXPECT_EQ(instanceDatabase.Find(s_instanceId0), nullptr);
EXPECT_NE(orphanedInstance, nullptr);
instanceDatabase.TEMPOrphan(s_instanceId0);
// Orphaning twice should be a no-op
EXPECT_EQ(instanceDatabase.Find(s_instanceId0), nullptr);
EXPECT_NE(orphanedInstance, nullptr);
Instance<TestInstanceA> instance2 = instanceDatabase.FindOrCreate(s_instanceId0, someAsset);
// Creating another instance with the same id should return a different instance than the one that was orphaned
EXPECT_NE(orphanedInstance, instance2);
}
enum class ParallelInstanceTestCases
{
Create,
CreateAndDeferRemoval,
CreateAndOrphan,
CreateDeferRemovalAndOrphan
};
enum class ParralleInstanceCurrentAction
{
Create,
DeferredRemoval,
Orphan
};
ParralleInstanceCurrentAction ParallelInstanceGetCurrentAction(ParallelInstanceTestCases testCase)
{
switch (testCase)
{
case ParallelInstanceTestCases::CreateAndDeferRemoval:
switch (rand() % 2)
{
case 0: return ParralleInstanceCurrentAction::Create;
case 1: return ParralleInstanceCurrentAction::DeferredRemoval;
}
case ParallelInstanceTestCases::CreateAndOrphan:
switch (rand() % 2)
{
case 0: return ParralleInstanceCurrentAction::Create;
case 1: return ParralleInstanceCurrentAction::Orphan;
}
case ParallelInstanceTestCases::CreateDeferRemovalAndOrphan:
switch (rand() % 3)
{
case 0: return ParralleInstanceCurrentAction::Create;
case 1: return ParralleInstanceCurrentAction::DeferredRemoval;
case 2: return ParralleInstanceCurrentAction::Orphan;
}
case ParallelInstanceTestCases::Create:
default:
return ParralleInstanceCurrentAction::Create;
}
}
void ParallelInstanceCreateHelper(size_t threadCountMax, size_t assetIdCount, float durationSeconds, ParallelInstanceTestCases testCase)
{
printf("Testing threads=%zu assetIds=%zu ... ", threadCountMax, assetIdCount);
@@ -192,6 +261,7 @@ namespace UnitTest
auto& instanceManager = InstanceDatabase<TestInstanceA>::Instance();
AZStd::vector<Uuid> guids;
AZStd::vector<Data::Instance<Data::InstanceData>> instances;
AZStd::vector<Asset<TestAssetType>> assets;
for (size_t i = 0; i < assetIdCount; ++i)
@@ -199,6 +269,7 @@ namespace UnitTest
Uuid guid = Uuid::CreateRandom();
guids.emplace_back(guid);
instances.emplace_back(nullptr);
// Pre-create asset so we don't attempt to load it from the catalog.
assets.emplace_back(assetManager.CreateAsset<TestAssetType>(guid, AZ::Data::AssetLoadBehavior::Default));
@@ -206,6 +277,7 @@ namespace UnitTest
AZStd::vector<AZStd::thread> threads;
AZStd::mutex mutex;
AZStd::mutex referenceTableMutex;
AZStd::atomic<int> threadCount((int)threadCountMax);
AZStd::condition_variable cv;
AZStd::atomic_bool keepDispatching(true);
@@ -225,11 +297,15 @@ namespace UnitTest
for (size_t i = 0; i < threadCountMax; ++i)
{
threads.emplace_back(
[&instanceManager, &threadCount, &cv, &guids, &assets, &durationSeconds]()
[&instanceManager, &threadCount, &cv, &guids, &instances, &assets, &durationSeconds, &testCase, &referenceTableMutex]()
{
AZ::Debug::Timer timer;
timer.Stamp();
bool deferRemoval = testCase == ParallelInstanceTestCases::CreateAndDeferRemoval ||
testCase == ParallelInstanceTestCases::CreateDeferRemovalAndOrphan
? true : false;
while (timer.GetDeltaTimeInSeconds() < durationSeconds)
{
const size_t index = rand() % guids.size();
@@ -237,11 +313,36 @@ namespace UnitTest
const InstanceId instanceId{ uuid };
const AssetId assetId{ uuid };
Instance<TestInstanceA> instance =
instanceManager.FindOrCreate(instanceId, Asset<TestAssetType>(assetId, azrtti_typeid<TestAssetType>()));
EXPECT_NE(instance, nullptr);
EXPECT_EQ(instance->GetId(), instanceId);
EXPECT_EQ(instance->m_asset, assets[index]);
ParralleInstanceCurrentAction currentAction = ParallelInstanceGetCurrentAction(testCase);
if (currentAction == ParralleInstanceCurrentAction::Orphan)
{
// Orphan the instance, but don't decrease its refcount
instanceManager.TEMPOrphan(instanceId);
}
else if (currentAction == ParralleInstanceCurrentAction::DeferredRemoval)
{
// Drop the refcount to zero so the instance will be released
referenceTableMutex.lock();
instances[index] = nullptr;
referenceTableMutex.unlock();
}
else
{
// Otherwise, add a new instance
Instance<TestInstanceA> instance = instanceManager.FindOrCreate(instanceId, assets[index]);
EXPECT_NE(instance, nullptr);
EXPECT_EQ(instance->GetId(), instanceId);
EXPECT_EQ(instance->m_asset, assets[index]);
if (deferRemoval)
{
// Keep a reference to the instance alive so it can be removed later
referenceTableMutex.lock();
instances[index] = instance;
referenceTableMutex.unlock();
}
}
}
threadCount--;
@@ -254,10 +355,12 @@ namespace UnitTest
// Used to detect a deadlock. If we wait for more than 10 seconds, it's likely a deadlock has occurred
while (threadCount > 0 && !timedOut)
{
size_t durationSecondsRoundedUp = static_cast<size_t>(std::ceil(durationSeconds));
AZStd::unique_lock<AZStd::mutex> lock(mutex);
timedOut =
(AZStd::cv_status::timeout ==
cv.wait_until(lock, AZStd::chrono::system_clock::now() + AZStd::chrono::seconds(durationSeconds * 2)));
cv.wait_until(lock, AZStd::chrono::system_clock::now() + AZStd::chrono::seconds(durationSecondsRoundedUp * 2)));
}
EXPECT_TRUE(threadCount == 0) << "One or more threads appear to be deadlocked at " << timer.GetDeltaTimeInSeconds() << " seconds";
@@ -273,11 +376,11 @@ namespace UnitTest
printf("Took %f seconds\n", timer.GetDeltaTimeInSeconds());
}
TEST_F(InstanceDatabaseTest, ParallelInstanceCreate)
void ParallelCreateTest(ParallelInstanceTestCases testCase)
{
// This is the original test scenario from when InstanceDatabase was first implemented
// threads, AssetIds, seconds
ParallelInstanceCreateHelper(8, 100, 5);
ParallelInstanceCreateHelper(8, 100, 5, testCase);
// This value is checked in as 1 so this test doesn't take too much time, but can be increased locally to soak the test.
const size_t attempts = 1;
@@ -289,11 +392,11 @@ namespace UnitTest
// The idea behind this series of tests is that there are two threads sharing one Instance, and both threads try to
// create or release that instance at the same time.
// At the time, this set of scenarios has something like a 10% failure rate.
const size_t duration = 2;
const float duration = 2.0f;
// threads, AssetIds, seconds
ParallelInstanceCreateHelper(2, 1, duration);
ParallelInstanceCreateHelper(4, 1, duration);
ParallelInstanceCreateHelper(8, 1, duration);
ParallelInstanceCreateHelper(2, 1, duration, testCase);
ParallelInstanceCreateHelper(4, 1, duration, testCase);
ParallelInstanceCreateHelper(8, 1, duration, testCase);
}
for (size_t i = 0; i < attempts; ++i)
@@ -301,19 +404,39 @@ namespace UnitTest
printf("Attempt %zu of %zu... \n", i, attempts);
// Here we try a bunch of different threadCount:assetCount ratios to be thorough
const size_t duration = 2;
const float duration = 2.0f;
// threads, AssetIds, seconds
ParallelInstanceCreateHelper(2, 1, duration);
ParallelInstanceCreateHelper(4, 1, duration);
ParallelInstanceCreateHelper(4, 2, duration);
ParallelInstanceCreateHelper(4, 4, duration);
ParallelInstanceCreateHelper(8, 1, duration);
ParallelInstanceCreateHelper(8, 2, duration);
ParallelInstanceCreateHelper(8, 3, duration);
ParallelInstanceCreateHelper(8, 4, duration);
ParallelInstanceCreateHelper(2, 1, duration, testCase);
ParallelInstanceCreateHelper(4, 1, duration, testCase);
ParallelInstanceCreateHelper(4, 2, duration, testCase);
ParallelInstanceCreateHelper(4, 4, duration, testCase);
ParallelInstanceCreateHelper(8, 1, duration, testCase);
ParallelInstanceCreateHelper(8, 2, duration, testCase);
ParallelInstanceCreateHelper(8, 3, duration, testCase);
ParallelInstanceCreateHelper(8, 4, duration, testCase);
}
}
TEST_F(InstanceDatabaseTest, ParallelInstanceCreate)
{
ParallelCreateTest(ParallelInstanceTestCases::Create);
}
TEST_F(InstanceDatabaseTest, ParallelInstanceCreateAndDeferRemoval)
{
ParallelCreateTest(ParallelInstanceTestCases::CreateAndDeferRemoval);
}
TEST_F(InstanceDatabaseTest, ParallelInstanceCreateAndOrphan)
{
ParallelCreateTest(ParallelInstanceTestCases::CreateAndOrphan);
}
TEST_F(InstanceDatabaseTest, ParallelInstanceCreateDeferRemovalAndOrphan)
{
ParallelCreateTest(ParallelInstanceTestCases::CreateDeferRemovalAndOrphan);
}
TEST_F(InstanceDatabaseTest, InstanceCreateNoDatabase)
{
bool m_deleted = false;