AP: product dependency optimization (#6619)

* Initial pass at optimizing product path dependency resolution Signed-off-by: amzn-mike <80125227+amzn-mike@users.noreply.github.com> * Add version of StripAssetPlatform that doesn't allocate or copy strings. Re-add missing test and fix up compile errors. Add benchmark test Signed-off-by: amzn-mike <80125227+amzn-mike@users.noreply.github.com> * Change UpdateProductDependencies to directly call s_InsertProductDependencyQuery.BindAndStep Signed-off-by: amzn-mike <80125227+amzn-mike@users.noreply.github.com> * Add test for same filename on multiple platforms Signed-off-by: amzn-mike <80125227+amzn-mike@users.noreply.github.com> * Rework search logic to keep track of the source of a search path (source vs product) and keep track of which search matches which dependency to avoid doing another search through every product later on Signed-off-by: amzn-mike <80125227+amzn-mike@users.noreply.github.com> * Clean up code, expand test Signed-off-by: amzn-mike <80125227+amzn-mike@users.noreply.github.com> * Fix paths not being lowercased by SanitizeForDatabase. Fix UpdateProductDependencies not updating existing dependencies Signed-off-by: amzn-mike <80125227+amzn-mike@users.noreply.github.com> * Add test for duplicate dependency matches. Fix saving duplicates Signed-off-by: amzn-mike <80125227+amzn-mike@users.noreply.github.com> * Clean up code Signed-off-by: amzn-mike <80125227+amzn-mike@users.noreply.github.com> * Separate test into test and benchmark versions Signed-off-by: amzn-mike <80125227+amzn-mike@users.noreply.github.com> * Cleanup include Signed-off-by: amzn-mike <80125227+amzn-mike@users.noreply.github.com> * Fix includes, switch hardcoded job manager setup to use JobManagerComponent instead Signed-off-by: amzn-mike <80125227+amzn-mike@users.noreply.github.com> * Replaced wildcard_match with PathView::Match. Changed StripAssetPlatformNoCopy to use TokenizeNext. Removed Environment Create/Destroy calls. Made ScopedAllocatorFixture a base class of ScopedAllocatorSetupFixture Signed-off-by: amzn-mike <80125227+amzn-mike@users.noreply.github.com> * Add AZ Environment create/destroy on AP test environment Signed-off-by: amzn-mike <80125227+amzn-mike@users.noreply.github.com> * Add missing asserts on database functions Signed-off-by: amzn-mike <80125227+amzn-mike@users.noreply.github.com> * Fix incorrect usage of StripAssetPlatformNoCopy Signed-off-by: amzn-mike <80125227+amzn-mike@users.noreply.github.com> * Fix source/product dependency type being ignored. Removed need for unordered_set for list of resolved dependencies. Updated unit tests Signed-off-by: amzn-mike <80125227+amzn-mike@users.noreply.github.com> * Better variable names Signed-off-by: amzn-mike <80125227+amzn-mike@users.noreply.github.com> * Remove testing code Signed-off-by: amzn-mike <80125227+amzn-mike@users.noreply.github.com> * Fix missing includes and namespaces Signed-off-by: amzn-mike <80125227+amzn-mike@users.noreply.github.com>
2022-01-13 08:56:24 -06:00
parent 6d1a2382e8
commit fed1278fe6
14 changed files with 742 additions and 237 deletions
@@ -15,6 +15,10 @@
 #include <AzTest/AzTest.h>

 #include <limits>
+#include <AzCore/Jobs/JobContext.h>
+#include <AzCore/Jobs/JobManager.h>
+#include <AzCore/Jobs/JobManagerComponent.h>
+#include <AzCore/Jobs/JobManagerDesc.h>

 using namespace AssetProcessor;

@@ -26,6 +30,7 @@ public:

    friend class GTEST_TEST_CLASS_NAME_(MultiplatformPathDependencyTest, AssetProcessed_Impl_MultiplatformDependencies);
    friend class GTEST_TEST_CLASS_NAME_(MultiplatformPathDependencyTest, AssetProcessed_Impl_MultiplatformDependencies_DeferredResolution);
+    friend class GTEST_TEST_CLASS_NAME_(MultiplatformPathDependencyTest, SameFilenameForAllPlatforms);

    friend class GTEST_TEST_CLASS_NAME_(MultiplatformPathDependencyTest, AssetProcessed_Impl_MultiplatformDependencies_SourcePath);

@@ -176,8 +181,21 @@ void AssetProcessorManagerTest::SetUp()

    AssetProcessorTest::SetUp();

+    AZ::AllocatorInstance<AZ::PoolAllocator>::Create();
+    AZ::AllocatorInstance<AZ::ThreadPoolAllocator>::Create();
+
    m_data = AZStd::make_unique<StaticData>();

+    m_data->m_serializeContext = AZStd::make_unique<AZ::SerializeContext>();
+
+    m_data->m_descriptor = AZ::JobManagerComponent::CreateDescriptor();
+    m_data->m_descriptor->Reflect(m_data->m_serializeContext.get());
+
+    m_data->m_jobManagerEntity = aznew AZ::Entity{};
+    m_data->m_jobManagerEntity->CreateComponent<AZ::JobManagerComponent>();
+    m_data->m_jobManagerEntity->Init();
+    m_data->m_jobManagerEntity->Activate();
+
    m_config.reset(new AssetProcessor::PlatformConfiguration());
    m_mockApplicationManager.reset(new AssetProcessor::MockApplicationManager());

@@ -256,6 +274,10 @@ void AssetProcessorManagerTest::SetUp()

 void AssetProcessorManagerTest::TearDown()
 {
+    m_data->m_jobManagerEntity->Deactivate();
+    delete m_data->m_jobManagerEntity;
+    delete m_data->m_descriptor;
+
    m_data = nullptr;

    QObject::disconnect(m_idleConnection);
@@ -271,6 +293,9 @@ void AssetProcessorManagerTest::TearDown()
    m_qApp.reset();
    m_scopeDir.reset();

+    AZ::AllocatorInstance<AZ::ThreadPoolAllocator>::Destroy();
+    AZ::AllocatorInstance<AZ::PoolAllocator>::Destroy();
+
    AssetProcessor::AssetProcessorTest::TearDown();
 }

@@ -1270,7 +1295,8 @@ TEST_F(AbsolutePathProductDependencyTest, AbsolutePathProductDependency_RetryDef
        AZ::Data::AssetType::CreateNull());
    m_assetProcessorManager->m_stateData->SetProduct(matchingProductForDependency);

-    m_assetProcessorManager->m_pathDependencyManager->RetryDeferredDependencies(matchingSource);
+    m_assetProcessorManager->m_pathDependencyManager->QueueSourceForDependencyResolution(matchingSource);
+    m_assetProcessorManager->m_pathDependencyManager->ProcessQueuedDependencyResolves();

    // The product dependency ID shouldn't change when it goes from unresolved to resolved.
    AZStd::vector<ProductDependencyDatabaseEntry> resolvedProductDependencies;
@@ -1405,6 +1431,7 @@ bool PathDependencyTest::ProcessAsset(TestAsset& asset, const OutputAssetSet& ou
        // tell the APM that the asset has been processed and allow it to bubble through its event queue:
        m_isIdling = false;
        m_assetProcessorManager->AssetProcessed(capturedDetails[jobSet].m_jobEntry, processJobResponse);
+        m_assetProcessorManager->CheckForIdle();

        jobSet++;
    }
@@ -2646,6 +2673,44 @@ TEST_F(MultiplatformPathDependencyTest, AssetProcessed_Impl_MultiplatformDepende
    ASSERT_NE(SearchDependencies(dependencyContainer, asset1.m_products[0]), SearchDependencies(dependencyContainer, asset1.m_products[1]));
 }

+TEST_F(MultiplatformPathDependencyTest, SameFilenameForAllPlatforms)
+{
+    TestAsset asset2("asset2");
+    bool result = ProcessAsset(
+        asset2, { { ".output" }, { ".output" } }, { { "*1.output", AssetBuilderSDK::ProductPathDependencyType::ProductFile } }, "subfolder1/",
+        ".txt");
+
+    ASSERT_TRUE(result);
+
+    TestAsset asset1("asset1");
+    result = ProcessAsset(asset1, { { ".output" }, { ".output" } }, {});
+
+    ASSERT_TRUE(result);
+
+    AssetDatabaseConnection* sharedConnection = m_assetProcessorManager->m_stateData.get();
+    ASSERT_TRUE(sharedConnection);
+
+    AzToolsFramework::AssetDatabase::ProductDependencyDatabaseEntryContainer dependencyContainer;
+
+    sharedConnection->GetProductDependencies(dependencyContainer);
+    int resolvedCount = 0;
+    int unresolvedCount = 0;
+    for (const auto& dep : dependencyContainer)
+    {
+        if (dep.m_unresolvedPath.empty())
+        {
+            resolvedCount++;
+        }
+        else
+        {
+            unresolvedCount++;
+        }
+    }
+    ASSERT_EQ(resolvedCount, 2);
+    ASSERT_EQ(unresolvedCount, 2);
+    VerifyDependencies(dependencyContainer, { asset1.m_products[0], asset1.m_products[1] }, { "*1.output", "*1.output" });
+}
+
 TEST_F(MultiplatformPathDependencyTest, AssetProcessed_Impl_MultiplatformDependencies_SourcePath)
 {
    // One product will be pc, one will be console (order is non-deterministic)
@@ -4185,7 +4250,7 @@ TEST_F(LockedFileTest, DeleteFile_LockedProduct_DeleteFails)
    auto theFile = m_data->m_absolutePath[1].toUtf8();
    const char* theFileString = theFile.constData();
    auto [sourcePath, productPath] = *m_data->m_productPaths.find(theFileString);
-    
+
    {
        QFile file(theFileString);
        file.remove();
@@ -4257,7 +4322,7 @@ TEST_F(LockedFileTest, DeleteFile_LockedProduct_DeletesWhenReleased)

        EXPECT_FALSE(QFile::exists(productPath));
        EXPECT_EQ(m_data->m_deletedSources.size(), 1);
-        
+
        EXPECT_GT(m_deleteCounter, 1); // Make sure the AP tried more than once to delete the file
        m_errorAbsorber->ExpectAsserts(0);
    }
@@ -5364,7 +5429,7 @@ AZStd::vector<AZStd::string> WildcardSourceDependencyTest::FileAddedTest(const Q
 void WildcardSourceDependencyTest::SetUp()
 {
    AssetProcessorManagerTest::SetUp();
-    
+
    QDir tempPath(m_tempDir.path());

    // Add a non-recursive scan folder.  Only files directly inside of this folder should be picked up, subfolders are ignored
@@ -5454,7 +5519,7 @@ TEST_F(WildcardSourceDependencyTest, Relative_Broad)
 {
    // Expect all files except for the 2 invalid ones (e and f)
    AZStd::vector<AZStd::string> resolvedPaths;
-    
+
    ASSERT_TRUE(Test("*.foo", resolvedPaths));
    ASSERT_THAT(resolvedPaths, ::testing::UnorderedElementsAre("a.foo", "b.foo", "folder/one/c.foo", "folder/one/d.foo", "1a.foo", "1b.foo"));
 }
@@ -5585,7 +5650,7 @@ TEST_F(WildcardSourceDependencyTest, Relative_CacheFolder)
 {
    AZStd::vector<AZStd::string> resolvedPaths;
    QDir tempPath(m_tempDir.path());
-    
+
    ASSERT_TRUE(Test("*cache.foo", resolvedPaths));
    ASSERT_THAT(resolvedPaths, ::testing::UnorderedElementsAre());
 }
@@ -5635,7 +5700,7 @@ TEST_F(WildcardSourceDependencyTest, FilesRemovedAfterInitialCache)

        ASSERT_EQ(excludedFolders.size(), 3);
    }
-    
+
    m_fileStateCache->SignalDeleteEvent(tempPath.absoluteFilePath("subfolder2/redirected/folder/two/ignored"));

    const auto& excludedFolders = excludedFolderCacheInterface->GetExcludedFolders();