From fed1278fe64e8a435b1c19ba8092b18a98fa9eb1 Mon Sep 17 00:00:00 2001 From: amzn-mike <80125227+amzn-mike@users.noreply.github.com> Date: Thu, 13 Jan 2022 08:56:24 -0600 Subject: [PATCH] AP: product dependency optimization (#6619) * Initial pass at optimizing product path dependency resolution Signed-off-by: amzn-mike <80125227+amzn-mike@users.noreply.github.com> * Add version of StripAssetPlatform that doesn't allocate or copy strings. Re-add missing test and fix up compile errors. Add benchmark test Signed-off-by: amzn-mike <80125227+amzn-mike@users.noreply.github.com> * Change UpdateProductDependencies to directly call s_InsertProductDependencyQuery.BindAndStep Signed-off-by: amzn-mike <80125227+amzn-mike@users.noreply.github.com> * Add test for same filename on multiple platforms Signed-off-by: amzn-mike <80125227+amzn-mike@users.noreply.github.com> * Rework search logic to keep track of the source of a search path (source vs product) and keep track of which search matches which dependency to avoid doing another search through every product later on Signed-off-by: amzn-mike <80125227+amzn-mike@users.noreply.github.com> * Clean up code, expand test Signed-off-by: amzn-mike <80125227+amzn-mike@users.noreply.github.com> * Fix paths not being lowercased by SanitizeForDatabase. Fix UpdateProductDependencies not updating existing dependencies Signed-off-by: amzn-mike <80125227+amzn-mike@users.noreply.github.com> * Add test for duplicate dependency matches. Fix saving duplicates Signed-off-by: amzn-mike <80125227+amzn-mike@users.noreply.github.com> * Clean up code Signed-off-by: amzn-mike <80125227+amzn-mike@users.noreply.github.com> * Separate test into test and benchmark versions Signed-off-by: amzn-mike <80125227+amzn-mike@users.noreply.github.com> * Cleanup include Signed-off-by: amzn-mike <80125227+amzn-mike@users.noreply.github.com> * Fix includes, switch hardcoded job manager setup to use JobManagerComponent instead Signed-off-by: amzn-mike <80125227+amzn-mike@users.noreply.github.com> * Replaced wildcard_match with PathView::Match. Changed StripAssetPlatformNoCopy to use TokenizeNext. Removed Environment Create/Destroy calls. Made ScopedAllocatorFixture a base class of ScopedAllocatorSetupFixture Signed-off-by: amzn-mike <80125227+amzn-mike@users.noreply.github.com> * Add AZ Environment create/destroy on AP test environment Signed-off-by: amzn-mike <80125227+amzn-mike@users.noreply.github.com> * Add missing asserts on database functions Signed-off-by: amzn-mike <80125227+amzn-mike@users.noreply.github.com> * Fix incorrect usage of StripAssetPlatformNoCopy Signed-off-by: amzn-mike <80125227+amzn-mike@users.noreply.github.com> * Fix source/product dependency type being ignored. Removed need for unordered_set for list of resolved dependencies. Updated unit tests Signed-off-by: amzn-mike <80125227+amzn-mike@users.noreply.github.com> * Better variable names Signed-off-by: amzn-mike <80125227+amzn-mike@users.noreply.github.com> * Remove testing code Signed-off-by: amzn-mike <80125227+amzn-mike@users.noreply.github.com> * Fix missing includes and namespaces Signed-off-by: amzn-mike <80125227+amzn-mike@users.noreply.github.com> --- .../AzCore/AzCore/UnitTest/TestTypes.h | 38 +- .../assetprocessor_test_files.cmake | 1 + .../native/AssetDatabase/AssetDatabase.cpp | 84 ++-- .../native/AssetManager/AssetCatalog.cpp | 4 +- .../AssetManager/PathDependencyManager.cpp | 187 +++++---- .../AssetManager/PathDependencyManager.h | 23 +- .../AssetManager/assetProcessorManager.cpp | 138 ++++--- .../native/tests/BaseAssetProcessorTest.h | 2 + .../tests/PathDependencyManagerTests.cpp | 362 ++++++++++++++++-- .../AssetProcessorManagerTest.cpp | 79 +++- .../assetmanager/AssetProcessorManagerTest.h | 11 +- .../utilities/PlatformConfiguration.cpp | 14 +- .../native/utilities/assetUtils.cpp | 28 +- .../native/utilities/assetUtils.h | 6 +- 14 files changed, 741 insertions(+), 236 deletions(-) diff --git a/Code/Framework/AzCore/AzCore/UnitTest/TestTypes.h b/Code/Framework/AzCore/AzCore/UnitTest/TestTypes.h index afa6898944..0b1d7caefe 100644 --- a/Code/Framework/AzCore/AzCore/UnitTest/TestTypes.h +++ b/Code/Framework/AzCore/AzCore/UnitTest/TestTypes.h @@ -71,18 +71,35 @@ namespace UnitTest }; /** - * RAII wrapper of AllocatorBase. - * The benefit of using this wrapper instead of AllocatorsTestFixture is that SetUp/TearDown of the allocator is managed - * on construction/destruction, allowing member variables of derived classes to exist as value (and do heap allocation). - */ - class ScopedAllocatorSetupFixture + * RAII wrapper of AllocatorBase. + * The benefit of using this wrapper instead of AllocatorsTestFixture is that SetUp/TearDown of the allocator is managed + * on construction/destruction, allowing member variables of derived classes to exist as value (and do heap allocation). + */ + class ScopedAllocatorFixture : AllocatorsBase + { + public: + ScopedAllocatorFixture() + { + SetupAllocator(); + } + explicit ScopedAllocatorFixture(const AZ::SystemAllocator::Descriptor& allocatorDesc) + { + SetupAllocator(allocatorDesc); + } + ~ScopedAllocatorFixture() override + { + TeardownAllocator(); + } + }; + + // Like ScopedAllocatorFixture, but includes the Test base class + class ScopedAllocatorSetupFixture : public ::testing::Test - , AllocatorsBase + , public ScopedAllocatorFixture { public: - ScopedAllocatorSetupFixture() { SetupAllocator(); } - explicit ScopedAllocatorSetupFixture(const AZ::SystemAllocator::Descriptor& allocatorDesc) { SetupAllocator(allocatorDesc); } - ~ScopedAllocatorSetupFixture() { TeardownAllocator(); } + ScopedAllocatorSetupFixture() = default; + explicit ScopedAllocatorSetupFixture(const AZ::SystemAllocator::Descriptor& allocatorDesc) : ScopedAllocatorFixture(allocatorDesc){} }; /** @@ -114,6 +131,7 @@ namespace UnitTest using AllocatorsFixture = AllocatorsTestFixture; #if defined(HAVE_BENCHMARK) + /** * Helper class to handle the boiler plate of setting up a benchmark fixture that uses the system allocators * If you wish to do additional setup and tear down be sure to call the base class SetUp first and TearDown @@ -218,7 +236,7 @@ namespace UnitTest static constexpr bool sHasPadding = size < alignment; AZStd::enable_if mPadding; }; - + template int CreationCounter::s_count = 0; template diff --git a/Code/Tools/AssetProcessor/assetprocessor_test_files.cmake b/Code/Tools/AssetProcessor/assetprocessor_test_files.cmake index 2c4c53642c..a7a46ad62c 100644 --- a/Code/Tools/AssetProcessor/assetprocessor_test_files.cmake +++ b/Code/Tools/AssetProcessor/assetprocessor_test_files.cmake @@ -48,6 +48,7 @@ set(FILES native/tests/InternalBuilders/SettingsRegistryBuilderTests.cpp native/tests/MissingDependencyScannerTests.cpp native/tests/SourceFileRelocatorTests.cpp + native/tests/PathDependencyManagerTests.cpp native/tests/AssetProcessorMessagesTests.cpp native/unittests/AssetProcessingStateDataUnitTests.cpp native/unittests/AssetProcessingStateDataUnitTests.h diff --git a/Code/Tools/AssetProcessor/native/AssetDatabase/AssetDatabase.cpp b/Code/Tools/AssetProcessor/native/AssetDatabase/AssetDatabase.cpp index cf33e559ac..5edf4e1d53 100644 --- a/Code/Tools/AssetProcessor/native/AssetDatabase/AssetDatabase.cpp +++ b/Code/Tools/AssetProcessor/native/AssetDatabase/AssetDatabase.cpp @@ -172,7 +172,7 @@ namespace AssetProcessor static const char* CREATEINDEX_BUILDERGUID_SOURCE_SOURCEDEPENDENCY_STATEMENT = "CREATE INDEX IF NOT EXISTS BuilderGuid_Source_SourceDependency ON SourceDependency (BuilderGuid, Source);"; static const char* CREATEINDEX_TYPEOFDEPENDENCY_SOURCEDEPENDENCY = "AssetProcessor::CreateIndexTypeOfDependency_SourceDependency"; - static const char* CREATEINDEX_TYPEOFDEPENDENCY_SOURCEDEPENDENCY_STATEMENT = + static const char* CREATEINDEX_TYPEOFDEPENDENCY_SOURCEDEPENDENCY_STATEMENT = "CREATE INDEX IF NOT EXISTS TypeOfDependency_SourceDependency ON SourceDependency (TypeOfDependency);"; static const char* CREATEINDEX_SCANFOLDERS_SOURCES_SCANFOLDER = "AssetProcesser::CreateIndexScanFoldersSourcesScanFolder"; @@ -611,7 +611,7 @@ namespace AssetProcessor SqlParam(":missingDependencyString"), SqlParam(":lastScanTime"), SqlParam(":scanTimeSecondsSinceEpoch")); - + static const auto s_DeleteMissingProductDependencyByProductIdQuery = MakeSqlQuery( DELETE_MISSING_PRODUCT_DEPENDENCY_BY_PRODUCTID, @@ -643,7 +643,7 @@ namespace AssetProcessor SqlParam(":analysisFingerprint")); static const char* INSERT_COLUMN_ANALYSISFINGERPRINT = "AssetProcessor::AddColumnAnalysisFingerprint"; - static const char* INSERT_COLUMN_ANALYSISFINGERPRINT_STATEMENT = + static const char* INSERT_COLUMN_ANALYSISFINGERPRINT_STATEMENT = "ALTER TABLE Sources " "ADD AnalysisFingerprint TEXT NOT NULL collate nocase default('');"; @@ -653,7 +653,7 @@ namespace AssetProcessor "ADD TypeOfDependency INTEGER NOT NULL DEFAULT 0;"; static const char* INSERT_COLUMN_FILE_MODTIME = "AssetProcessor::AddFiles_ModTime"; - static const char* INSERT_COLUMN_FILE_MODTIME_STATEMENT = + static const char* INSERT_COLUMN_FILE_MODTIME_STATEMENT = "ALTER TABLE Files " "ADD ModTime INTEGER NOT NULL DEFAULT 0;"; @@ -673,7 +673,7 @@ namespace AssetProcessor "ADD UnresolvedDependencyType INTEGER NOT NULL DEFAULT 0;"; static const char* INSERT_COLUMN_PRODUCTDEPENDENCY_PLATFORM = "AssetProcessor::AddProductDependency_Platform"; - static const char* INSERT_COLUMN_PRODUCTDEPENDENCY_PLATFORM_STATEMENT = + static const char* INSERT_COLUMN_PRODUCTDEPENDENCY_PLATFORM_STATEMENT = "ALTER TABLE ProductDependencies " "ADD Platform TEXT NOT NULL collate nocase default('');"; @@ -721,7 +721,7 @@ namespace AssetProcessor SqlParam(":isfolder"), SqlParam(":modtime"), SqlParam(":hash")); - + static const char* UPDATE_FILE = "AssetProcessor::UpdateFile"; static const char* UPDATE_FILE_STATEMENT = "UPDATE Files SET " @@ -961,7 +961,7 @@ namespace AssetProcessor AZ_TracePrintf(AssetProcessor::ConsoleChannel, "Upgraded Asset Database to version %i (AddedTypeOfDependencyIndex)\n", foundVersion) } } - + if (foundVersion == AssetDatabase::DatabaseVersion::AddedTypeOfDependencyIndex) { if (m_databaseConnection->ExecuteOneOffStatement(INSERT_COLUMN_PRODUCTDEPENDENCY_PLATFORM)) @@ -1178,7 +1178,7 @@ namespace AssetProcessor AddStatement(m_databaseConnection, s_InsertJobQuery); AddStatement(m_databaseConnection, s_UpdateJobQuery); AddStatement(m_databaseConnection, s_DeleteJobQuery); - + // --------------------------------------------------------------------------------------------- // Builder Info Table // --------------------------------------------------------------------------------------------- @@ -1207,7 +1207,7 @@ namespace AssetProcessor m_databaseConnection->AddStatement(CREATE_SOURCE_DEPENDENCY_TABLE, CREATE_SOURCE_DEPENDENCY_TABLE_STATEMENT); m_databaseConnection->AddStatement(INSERT_COLUMN_SOURCEDEPENDENCY_TYPEOFDEPENDENCY, INSERT_COLUMN_SOURCEDEPENDENCY_TYPEOFDEPENDENCY_STATEMENT); m_databaseConnection->AddStatement(INSERT_COLUMNS_SOURCEDEPENDENCY_FROM_ASSETID, INSERT_COLUMNS_SOURCEDEPENDENCY_FROM_ASSETID_STATEMENT); - + m_createStatements.push_back(CREATE_SOURCE_DEPENDENCY_TABLE); AddStatement(m_databaseConnection, s_InsertSourceDependencyQuery); @@ -1242,7 +1242,7 @@ namespace AssetProcessor AddStatement(m_databaseConnection, s_InsertProductDependencyQuery); AddStatement(m_databaseConnection, s_UpdateProductDependencyQuery); AddStatement(m_databaseConnection, s_DeleteProductDependencyByProductIdQuery); - + // --------------------------------------------------------------------------------------------- // Missing Product Dependency table // --------------------------------------------------------------------------------------------- @@ -1253,7 +1253,7 @@ namespace AssetProcessor AddStatement(m_databaseConnection, s_InsertMissingProductDependencyQuery); AddStatement(m_databaseConnection, s_UpdateMissingProductDependencyQuery); AddStatement(m_databaseConnection, s_DeleteMissingProductDependencyByProductIdQuery); - + // --------------------------------------------------------------------------------------------- // Files table // --------------------------------------------------------------------------------------------- @@ -1344,7 +1344,7 @@ namespace AssetProcessor bool AssetDatabaseConnection::GetScanFolderByScanFolderID(AZ::s64 scanfolderID, ScanFolderDatabaseEntry& entry) { bool found = false; - QueryScanFolderByScanFolderID( scanfolderID, + QueryScanFolderByScanFolderID( scanfolderID, [&](ScanFolderDatabaseEntry& scanFolderEntry) { entry = scanFolderEntry; @@ -1357,7 +1357,7 @@ namespace AssetProcessor bool AssetDatabaseConnection::GetScanFolderBySourceID(AZ::s64 sourceID, ScanFolderDatabaseEntry& entry) { bool found = false; - QueryScanFolderBySourceID( sourceID, + QueryScanFolderBySourceID( sourceID, [&](ScanFolderDatabaseEntry& scanFolderEntry) { entry = scanFolderEntry; @@ -1370,7 +1370,7 @@ namespace AssetProcessor bool AssetDatabaseConnection::GetScanFolderByJobID(AZ::s64 jobID, ScanFolderDatabaseEntry& entry) { bool found = false; - QueryScanFolderByJobID( jobID, + QueryScanFolderByJobID( jobID, [&](ScanFolderDatabaseEntry& scanFolderEntry) { entry = scanFolderEntry; @@ -1383,7 +1383,7 @@ namespace AssetProcessor bool AssetDatabaseConnection::GetScanFolderByProductID(AZ::s64 productID, ScanFolderDatabaseEntry& entry) { bool found = false; - QueryScanFolderByProductID( productID, + QueryScanFolderByProductID( productID, [&](ScanFolderDatabaseEntry& scanFolderEntry) { entry = scanFolderEntry; @@ -2105,7 +2105,7 @@ namespace AssetProcessor bool AssetDatabaseConnection::GetProductsLikeProductName(QString likeProductName, LikeType likeType, ProductDatabaseEntryContainer& container, AZ::Uuid builderGuid, QString jobKey, QString platform, JobStatus status) { bool found = false; - + if (likeProductName.isEmpty()) { return false; @@ -2198,7 +2198,7 @@ namespace AssetProcessor bool AssetDatabaseConnection::GetProductByJobIDSubId(AZ::s64 jobID, AZ::u32 subID, AzToolsFramework::AssetDatabase::ProductDatabaseEntry& result) { bool found = false; - QueryProductByJobIDSubID(jobID, subID, + QueryProductByJobIDSubID(jobID, subID, [&](ProductDatabaseEntry& resultFromDB) { found = true; @@ -2312,13 +2312,19 @@ namespace AssetProcessor { return false; } - - bool succeeded = true; + + ScopedTransaction transaction(m_databaseConnection); + for (auto& entry : container) { - succeeded &= SetProduct(entry); + if(!SetProduct(entry)) + { + return false; + } } - return succeeded; + + transaction.Commit(); + return true; } //! Clear the products for a given source. This removes the entry entirely, not just sets it to empty. @@ -2407,7 +2413,7 @@ namespace AssetProcessor if(!platform.isEmpty()) { AZStd::string platformStr = platform.toUtf8().constData(); - + if (!s_DeleteProductsBySourceidPlatformQuery.BindAndStep(*m_databaseConnection, sourceID, platformStr.c_str())) { return false; @@ -2521,7 +2527,7 @@ namespace AssetProcessor { succeeded = succeeded && RemoveSourceFileDependency(entry); } - + if (succeeded) { transaction.Commit(); @@ -2577,7 +2583,7 @@ namespace AssetProcessor } bool AssetDatabaseConnection::GetDependsOnSourceBySource( - const char* source, + const char* source, AzToolsFramework::AssetDatabase::SourceFileDependencyEntry::TypeOfDependency typeOfDependency, AzToolsFramework::AssetDatabase::SourceFileDependencyEntryContainer& container) { @@ -2596,7 +2602,7 @@ namespace AssetProcessor bool AssetDatabaseConnection::GetSourceFileDependencyBySourceDependencyId(AZ::s64 sourceDependencyId, SourceFileDependencyEntry& sourceDependencyEntry) { bool found = false; - QuerySourceDependencyBySourceDependencyId(sourceDependencyId, + QuerySourceDependencyBySourceDependencyId(sourceDependencyId, [&](SourceFileDependencyEntry& entry) { found = true; @@ -2624,7 +2630,7 @@ namespace AssetProcessor return false; } - + if (creatingNew) { AZ::s64 rowID = m_databaseConnection->GetLastRowID(); @@ -2958,9 +2964,25 @@ namespace AssetProcessor for(auto& entry : container) { - if(!SetProductDependency(entry)) + if(entry.m_productDependencyID == InvalidEntryId) { - return false; + if (!s_InsertProductDependencyQuery.BindAndStep( + *m_databaseConnection, entry.m_productPK, entry.m_dependencySourceGuid, entry.m_dependencySubID, + entry.m_dependencyFlags.to_ullong(), entry.m_platform.c_str(), entry.m_unresolvedPath.c_str(), + entry.m_dependencyType, entry.m_fromAssetId)) + { + return false; + } + } + else + { + if(!s_UpdateProductDependencyQuery.BindAndStep( + *m_databaseConnection, entry.m_productPK, entry.m_dependencySourceGuid, entry.m_dependencySubID, + entry.m_dependencyFlags.to_ullong(), entry.m_platform.c_str(), entry.m_unresolvedPath.c_str(), + entry.m_productDependencyID, entry.m_dependencyType, entry.m_fromAssetId)) + { + return false; + } } } @@ -2989,7 +3011,7 @@ namespace AssetProcessor } // now insert the new ones since we know there's no collisions: - + for (auto& entry : container) { @@ -3109,7 +3131,7 @@ namespace AssetProcessor } Statement* statement = autoFinal.Get(); - + if (statement->Step() == Statement::SqlError) { AZ_Warning(LOG_NAME, false, "Failed to write the new source into the database. %s", entry.m_fileName.c_str()); @@ -3126,7 +3148,7 @@ namespace AssetProcessor return UpdateFile(entry, entryAlreadyExists); } - bool AssetDatabaseConnection::UpdateFile(FileDatabaseEntry& entry, bool& entryAlreadyExists) + bool AssetDatabaseConnection::UpdateFile(FileDatabaseEntry& entry, bool& entryAlreadyExists) { entryAlreadyExists = false; diff --git a/Code/Tools/AssetProcessor/native/AssetManager/AssetCatalog.cpp b/Code/Tools/AssetProcessor/native/AssetManager/AssetCatalog.cpp index d7cdd30be8..bda5eb7842 100644 --- a/Code/Tools/AssetProcessor/native/AssetManager/AssetCatalog.cpp +++ b/Code/Tools/AssetProcessor/native/AssetManager/AssetCatalog.cpp @@ -480,12 +480,12 @@ namespace AssetProcessor AZ::Data::AssetId assetId(combined.m_sourceGuid, combined.m_subID); // relative file path is gotten by removing the platform and game from the product name - QString relativeProductPath = AssetUtilities::StripAssetPlatform(combined.m_productName); + AZStd::string_view relativeProductPath = AssetUtilities::StripAssetPlatformNoCopy(combined.m_productName); QString fullProductPath = m_cacheRoot.absoluteFilePath(combined.m_productName.c_str()); AZ::Data::AssetInfo info; info.m_assetType = combined.m_assetType; - info.m_relativePath = relativeProductPath.toUtf8().data(); + info.m_relativePath = relativeProductPath; info.m_assetId = assetId; info.m_sizeBytes = AZ::IO::SystemFile::Length(fullProductPath.toUtf8().constData()); diff --git a/Code/Tools/AssetProcessor/native/AssetManager/PathDependencyManager.cpp b/Code/Tools/AssetProcessor/native/AssetManager/PathDependencyManager.cpp index dc10dc2237..6dddfd62d3 100644 --- a/Code/Tools/AssetProcessor/native/AssetManager/PathDependencyManager.cpp +++ b/Code/Tools/AssetProcessor/native/AssetManager/PathDependencyManager.cpp @@ -9,18 +9,24 @@ #include "PathDependencyManager.h" #include #include +#include #include #include #include +#include namespace AssetProcessor { void SanitizeForDatabase(AZStd::string& str) { - // Not calling normalize because wildcards should be preserved. AZStd::to_lower(str.begin(), str.end()); - AZStd::replace(str.begin(), str.end(), AZ_WRONG_DATABASE_SEPARATOR, AZ_CORRECT_DATABASE_SEPARATOR); - AzFramework::StringFunc::Replace(str, AZ_DOUBLE_CORRECT_DATABASE_SEPARATOR, AZ_CORRECT_DATABASE_SEPARATOR_STRING); + + // Not calling normalize because wildcards should be preserved. + if (AZ::StringFunc::Contains(str, AZ_WRONG_DATABASE_SEPARATOR, true)) + { + AZStd::replace(str.begin(), str.end(), AZ_WRONG_DATABASE_SEPARATOR, AZ_CORRECT_DATABASE_SEPARATOR); + AzFramework::StringFunc::Replace(str, AZ_DOUBLE_CORRECT_DATABASE_SEPARATOR, AZ_CORRECT_DATABASE_SEPARATOR_STRING); + } } PathDependencyManager::PathDependencyManager(AZStd::shared_ptr stateData, PlatformConfiguration* platformConfig) @@ -29,7 +35,97 @@ namespace AssetProcessor } - void PathDependencyManager::SaveUnresolvedDependenciesToDatabase(AssetBuilderSDK::ProductPathDependencySet& unresolvedDependencies, const AzToolsFramework::AssetDatabase::ProductDatabaseEntry& productEntry, const AZStd::string& platform) + void PathDependencyManager::QueueSourceForDependencyResolution(const AzToolsFramework::AssetDatabase::SourceDatabaseEntry& sourceEntry) + { + m_queuedForResolve.push_back(sourceEntry); + } + + void PathDependencyManager::ProcessQueuedDependencyResolves() + { + if (m_queuedForResolve.empty()) + { + return; + } + + auto queuedForResolve = m_queuedForResolve; + m_queuedForResolve.clear(); + + // Grab every product from the database and map to Source PK -> [products] + AZStd::unordered_map> productMap; + m_stateData->QueryCombined([&productMap](const AzToolsFramework::AssetDatabase::CombinedDatabaseEntry& entry) + { + productMap[entry.m_sourcePK].push_back(entry); + return true; + }); + + // Build up a list of all the paths we need to search for: products + 2 variations of the source path + AZStd::vector searches; + + for (const auto& entry : queuedForResolve) + { + // Search for each product + for (const auto& productEntry : productMap[entry.m_sourceID]) + { + const AZStd::string& productName = productEntry.m_productName; + + // strip path of the / + AZStd::string_view result = AssetUtilities::StripAssetPlatformNoCopy(productName); + searches.emplace_back(result, false, &entry, &productEntry); + } + + // Search for the source path + AZStd::string sourceNameWithScanFolder = + ToScanFolderPrefixedPath(aznumeric_cast(entry.m_scanFolderPK), entry.m_sourceName.c_str()); + AZStd::string sanitizedSourceName = entry.m_sourceName; + + SanitizeForDatabase(sourceNameWithScanFolder); + SanitizeForDatabase(sanitizedSourceName); + + searches.emplace_back(sourceNameWithScanFolder, true, &entry, nullptr); + searches.emplace_back(sanitizedSourceName, true, &entry, nullptr); + } + + AzToolsFramework::AssetDatabase::ProductDependencyDatabaseEntryContainer unresolvedDependencies; + m_stateData->GetUnresolvedProductDependencies(unresolvedDependencies); + + AZStd::recursive_mutex mapMutex; + // Map of Map of Product Dependency>> + AZStd::unordered_map>> sourceIdToMatchedSearchDependencies; + + // For every search path we created, we're going to see if it matches up against any of the unresolved dependencies + AZ::parallel_for_each( + searches.begin(), searches.end(), + [&sourceIdToMatchedSearchDependencies, &mapMutex, &unresolvedDependencies](const SearchEntry& search) + { + AZStd::unordered_set matches; + for (const auto& entry: unresolvedDependencies) + { + AZ::IO::PathView searchPath(search.m_path); + + if(((entry.m_dependencyType == AzToolsFramework::AssetDatabase::ProductDependencyDatabaseEntry::ProductDep_SourceFile && search.m_isSourcePath) + || (entry.m_dependencyType == AzToolsFramework::AssetDatabase::ProductDependencyDatabaseEntry::ProductDep_ProductFile && !search.m_isSourcePath)) + && searchPath.Match(entry.m_unresolvedPath)) + { + matches.insert(entry); + } + } + + if (!matches.empty()) + { + AZStd::scoped_lock lock(mapMutex); + auto& productDependencyDatabaseEntries = sourceIdToMatchedSearchDependencies[search.m_sourceEntry->m_sourceID][&search]; + productDependencyDatabaseEntries.insert(matches.begin(), matches.end()); + } + }); + + for (const auto& entry : queuedForResolve) + { + RetryDeferredDependencies(entry, sourceIdToMatchedSearchDependencies[entry.m_sourceID], productMap[entry.m_sourceID]); + } + } + + void PathDependencyManager::SaveUnresolvedDependenciesToDatabase(AssetBuilderSDK::ProductPathDependencySet& unresolvedDependencies, + const AzToolsFramework::AssetDatabase::ProductDatabaseEntry& productEntry, const AZStd::string& platform) { using namespace AzToolsFramework::AssetDatabase; @@ -206,9 +302,9 @@ namespace AssetProcessor } void PathDependencyManager::SaveResolvedDependencies(const AzToolsFramework::AssetDatabase::SourceDatabaseEntry& sourceEntry, const MapSet& exclusionMaps, const AZStd::string& sourceNameWithScanFolder, - const AZStd::vector& dependencyEntries, + const AZStd::unordered_set& dependencyEntries, AZStd::string_view matchedPath, bool isSourceDependency, const AzToolsFramework::AssetDatabase::ProductDatabaseEntryContainer& matchedProducts, - AzToolsFramework::AssetDatabase::ProductDependencyDatabaseEntryContainer& dependencyContainer) const + AZStd::vector& dependencyContainer) const { for (const auto& productDependencyDatabaseEntry : dependencyEntries) { @@ -267,8 +363,7 @@ namespace AssetProcessor } // All checks passed, this is a valid dependency we need to save to the db - dependencyContainer.push_back(); - auto& entry = dependencyContainer.back(); + AzToolsFramework::AssetDatabase::ProductDependencyDatabaseEntry entry; entry.m_productDependencyID = dependencyId; entry.m_productPK = productDependencyDatabaseEntry.m_productPK; @@ -276,62 +371,30 @@ namespace AssetProcessor entry.m_dependencySubID = matchedProduct.m_subID; entry.m_platform = productDependencyDatabaseEntry.m_platform; + dependencyContainer.push_back(AZStd::move(entry)); + // If there's more than 1 product, reset the ID so further products create new db entries dependencyId = AzToolsFramework::AssetDatabase::InvalidEntryId; } } } - void PathDependencyManager::RetryDeferredDependencies(const AzToolsFramework::AssetDatabase::SourceDatabaseEntry& sourceEntry) + void PathDependencyManager::RetryDeferredDependencies(const AzToolsFramework::AssetDatabase::SourceDatabaseEntry& sourceEntry, + const AZStd::unordered_map>& matches, + const AZStd::vector& products) { MapSet exclusionMaps = PopulateExclusionMaps(); - // Gather a list of all the products this source file produced - AzToolsFramework::AssetDatabase::ProductDatabaseEntryContainer products; - if (!m_stateData->GetProductsBySourceName(sourceEntry.m_sourceName.c_str(), products)) - { - AZ_TracePrintf(AssetProcessor::ConsoleChannel, "Source %s did not have any products. Skipping dependency processing.\n", sourceEntry.m_sourceName.c_str()); - return; - } - - AZStd::unordered_map> map; - - // Build up a list of all the paths we need to search for: products + 2 variations of the source path - AZStd::vector searchPaths; - - for (const auto& productEntry : products) - { - const AZStd::string& productName = productEntry.m_productName; - - // strip path of the / - AZStd::string strippedPath = AssetUtilities::StripAssetPlatform(productName).toUtf8().constData(); - SanitizeForDatabase(strippedPath); - - searchPaths.push_back(strippedPath); - } - AZStd::string sourceNameWithScanFolder = ToScanFolderPrefixedPath(aznumeric_cast(sourceEntry.m_scanFolderPK), sourceEntry.m_sourceName.c_str()); - AZStd::string sanitizedSourceName = sourceEntry.m_sourceName; - SanitizeForDatabase(sourceNameWithScanFolder); - SanitizeForDatabase(sanitizedSourceName); - - searchPaths.push_back(sourceNameWithScanFolder); - searchPaths.push_back(sanitizedSourceName); - - m_stateData->QueryProductDependenciesUnresolvedAdvanced(searchPaths, [&map](AzToolsFramework::AssetDatabase::ProductDependencyDatabaseEntry& entry, const AZStd::string& matchedPath) - { - map[matchedPath].push_back(AZStd::move(entry)); - return true; - }); - AzToolsFramework::AssetDatabase::ProductDependencyDatabaseEntryContainer dependencyContainer; + AzToolsFramework::AssetDatabase::ProductDependencyDatabaseEntryContainer dependencyVector; // Go through all the matched dependencies - for (const auto& pair : map) + for (const auto& pair : matches) { - AZStd::string_view matchedPath = pair.first; - const bool isSourceDependency = matchedPath == sanitizedSourceName || matchedPath == sourceNameWithScanFolder; + const SearchEntry* searchEntry = pair.first; + const bool isSourceDependency = searchEntry->m_isSourcePath; AzToolsFramework::AssetDatabase::ProductDatabaseEntryContainer matchedProducts; @@ -342,34 +405,24 @@ namespace AssetProcessor } else { - for (const auto& productEntry : products) - { - const AZStd::string& productName = productEntry.m_productName; - - // strip path of the leading asset platform / - AZStd::string strippedPath = AssetUtilities::StripAssetPlatform(productName).toUtf8().constData(); - SanitizeForDatabase(strippedPath); - - if (strippedPath == matchedPath) - { - matchedProducts.push_back(productEntry); - } - } + matchedProducts.push_back(*searchEntry->m_productEntry); } // Go through each dependency we're resolving and create a db entry for each product that resolved it (wildcard/source dependencies will generally create more than 1) - SaveResolvedDependencies(sourceEntry, exclusionMaps, sourceNameWithScanFolder, pair.second, matchedPath, isSourceDependency, matchedProducts, dependencyContainer); + SaveResolvedDependencies( + sourceEntry, exclusionMaps, sourceNameWithScanFolder, pair.second, searchEntry->m_path, isSourceDependency, matchedProducts, + dependencyVector); } - // Save everything to the db - if (!m_stateData->UpdateProductDependencies(dependencyContainer)) + // Save everything to the db, this will update matched non-wildcard dependencies and add new records for wildcard matches + if (!m_stateData->UpdateProductDependencies(dependencyVector)) { AZ_Error("PathDependencyManager", false, "Failed to update product dependencies"); } else { // Send a notification for each dependency that has been resolved - NotifyResolvedDependencies(dependencyContainer); + NotifyResolvedDependencies(dependencyVector); } } @@ -460,7 +513,7 @@ namespace AssetProcessor if (isExactDependency) { // Search for products in the cache platform folder - // Example: If a path dependency is "test1.asset" in AutomatedTesting on PC, this would search + // Example: If a path dependency is "test1.asset" in AutomatedTesting on PC, this would search // "AutomatedTesting/Cache/pc/test1.asset" m_stateData->GetProductsByProductName(productNameWithPlatform, productInfoContainer); diff --git a/Code/Tools/AssetProcessor/native/AssetManager/PathDependencyManager.h b/Code/Tools/AssetProcessor/native/AssetManager/PathDependencyManager.h index 3a207ec6a9..763ee2cd74 100644 --- a/Code/Tools/AssetProcessor/native/AssetManager/PathDependencyManager.h +++ b/Code/Tools/AssetProcessor/native/AssetManager/PathDependencyManager.h @@ -39,9 +39,27 @@ namespace AssetProcessor PathDependencyManager(AZStd::shared_ptr stateData, PlatformConfiguration* platformConfig); + void QueueSourceForDependencyResolution(const AzToolsFramework::AssetDatabase::SourceDatabaseEntry& sourceEntry); + + void ProcessQueuedDependencyResolves(); + + struct SearchEntry + { + SearchEntry(AZStd::string path, bool isSourcePath, const AzToolsFramework::AssetDatabase::SourceDatabaseEntry* sourceEntry, const AzToolsFramework::AssetDatabase::ProductDatabaseEntry* productEntry) + : m_path(std::move(path)), + m_isSourcePath(isSourcePath), + m_sourceEntry(sourceEntry), + m_productEntry(productEntry) {} + + AZStd::string m_path; + bool m_isSourcePath; + const AzToolsFramework::AssetDatabase::SourceDatabaseEntry* m_sourceEntry = nullptr; + const AzToolsFramework::AssetDatabase::ProductDatabaseEntry* m_productEntry = nullptr; + }; + /// This function is responsible for looking up existing, unresolved dependencies that the current asset satisfies. /// These can be dependencies on either the source asset or one of the product assets - void RetryDeferredDependencies(const AzToolsFramework::AssetDatabase::SourceDatabaseEntry& sourceEntry); + void RetryDeferredDependencies(const AzToolsFramework::AssetDatabase::SourceDatabaseEntry& sourceEntry, const AZStd::unordered_map>& matches, const AZStd::vector& products); /// This function is responsible for taking the path dependencies output by the current asset and trying to resolve them to AssetIds /// This does not look for dependencies that the current asset satisfies. @@ -66,7 +84,7 @@ namespace AssetProcessor MapSet PopulateExclusionMaps() const; void NotifyResolvedDependencies(const AzToolsFramework::AssetDatabase::ProductDependencyDatabaseEntryContainer& dependencyContainer) const; - void SaveResolvedDependencies(const AzToolsFramework::AssetDatabase::SourceDatabaseEntry& sourceEntry, const MapSet& exclusionMaps, const AZStd::string& sourceNameWithScanFolder, const AZStd::vector& dependencyEntries, AZStd::string_view matchedPath, bool isSourceDependency, const AzToolsFramework::AssetDatabase::ProductDatabaseEntryContainer& matchedProducts, AzToolsFramework::AssetDatabase::ProductDependencyDatabaseEntryContainer& dependencyContainer) const; + void SaveResolvedDependencies(const AzToolsFramework::AssetDatabase::SourceDatabaseEntry& sourceEntry, const MapSet& exclusionMaps, const AZStd::string& sourceNameWithScanFolder, const AZStd::unordered_set& dependencyEntries, AZStd::string_view matchedPath, bool isSourceDependency, const AzToolsFramework::AssetDatabase::ProductDatabaseEntryContainer& matchedProducts, AZStd::vector& dependencyContainer) const; static DependencyProductMap& SelectMap(MapSet& mapSet, bool wildcard, AzToolsFramework::AssetDatabase::ProductDependencyDatabaseEntry::DependencyType type); /// Returns false if a path contains wildcards, true otherwise @@ -93,5 +111,6 @@ namespace AssetProcessor AZStd::shared_ptr m_stateData; PlatformConfiguration* m_platformConfig{}; DependencyResolvedCallback m_dependencyResolvedCallback{}; + AZStd::vector m_queuedForResolve; }; } // namespace AssetProcessor diff --git a/Code/Tools/AssetProcessor/native/AssetManager/assetProcessorManager.cpp b/Code/Tools/AssetProcessor/native/AssetManager/assetProcessorManager.cpp index 8867f2379f..beb82dfcab 100644 --- a/Code/Tools/AssetProcessor/native/AssetManager/assetProcessorManager.cpp +++ b/Code/Tools/AssetProcessor/native/AssetManager/assetProcessorManager.cpp @@ -48,7 +48,7 @@ namespace AssetProcessor // note that this is not the first time we're opening the database - the main thread also opens it before this happens, // which allows it to upgrade it and check it for errors. If we get here, it means the database is already good to go. - m_stateData->OpenDatabase(); + m_stateData->OpenDatabase(); MigrateScanFolders(); @@ -70,7 +70,7 @@ namespace AssetProcessor m_excludedFolderCache = AZStd::make_unique(m_platformConfig); PopulateJobStateCache(); - + AssetProcessor::ProcessingJobInfoBus::Handler::BusConnect(); } @@ -126,7 +126,7 @@ namespace AssetProcessor { // capture scanning stats: AssetProcessor::StatsCapture::BeginCaptureStat("AssetScanning"); - + // Ensure that the source file list is populated before a scan begins m_sourceFilesInDatabase.clear(); m_fileModTimes.clear(); @@ -155,11 +155,11 @@ namespace AssetProcessor QString scanFolderPath; QString relativeToScanFolderPath = QString::fromUtf8(entry.m_fileName.c_str()); - + for (int i = 0; i < m_platformConfig->GetScanFolderCount(); ++i) { const auto& scanFolderInfo = m_platformConfig->GetScanFolderAt(i); - + if (scanFolderInfo.ScanFolderID() == entry.m_scanFolderPK) { scanFolderPath = scanFolderInfo.ScanPath(); @@ -181,7 +181,7 @@ namespace AssetProcessor { m_isCurrentlyScanning = false; AssetProcessor::StatsCapture::EndCaptureStat("AssetScanning"); - + // we cannot invoke this immediately - the scanner might be done, but we aren't actually ready until we've processed all remaining messages: QMetaObject::invokeMethod(this, "CheckMissingFiles", Qt::QueuedConnection); } @@ -216,7 +216,7 @@ namespace AssetProcessor else { QString statKey = QString("ProcessJob,%1,%2,%3").arg(jobEntry.m_databaseSourceName).arg(jobEntry.m_jobKey).arg(jobEntry.m_platformInfo.m_identifier.c_str()); - + if (status == JobStatus::InProgress) { //update to in progress status @@ -232,7 +232,7 @@ namespace AssetProcessor // without going thru the RC. // as such, all the code in this block should be crafted to work regardless of whether its double called. AssetProcessor::StatsCapture::EndCaptureStat(statKey.toUtf8().constData()); - + m_jobRunKeyToJobInfoMap.erase(jobEntry.m_jobRunKey); Q_EMIT SourceFinished(sourceUUID, legacySourceUUID); Q_EMIT JobComplete(jobEntry, status); @@ -308,7 +308,7 @@ namespace AssetProcessor //! A network request came in, Given a Job Run Key (from the above Job Request), asking for the actual log for that job. GetAbsoluteAssetDatabaseLocationResponse AssetProcessorManager::ProcessGetAbsoluteAssetDatabaseLocationRequest(MessageData messageData) - { + { GetAbsoluteAssetDatabaseLocationResponse response; AzToolsFramework::AssetDatabase::AssetDatabaseRequestsBus::Broadcast(&AzToolsFramework::AssetDatabase::AssetDatabaseRequests::GetAssetDatabaseLocation, response.m_absoluteAssetDatabaseLocation); @@ -525,7 +525,7 @@ namespace AssetProcessor { foundOne = true; return true; - }, + }, AZ::Uuid::CreateNull(), nullptr, platform.toUtf8().constData(), @@ -750,7 +750,7 @@ namespace AssetProcessor } OnJobStatusChanged(jobEntry, JobStatus::Failed); - + // note that we always print out the failed job status here in both batch and GUI mode. AZ_TracePrintf(AssetProcessor::ConsoleChannel, "Failed %s, (%s)... \n", jobEntry.m_pathRelativeToWatchFolder.toUtf8().constData(), @@ -868,7 +868,7 @@ namespace AssetProcessor && AzFramework::StringFunc::Equal(job.m_platform.c_str(), itProcessedAsset->m_entry.m_platformInfo.m_identifier.c_str())) { // If we are here it implies that for the same source file we have another job that outputs the same product. - // This is usually the case when two builders process the same source file and outputs the same product file. + // This is usually the case when two builders process the same source file and outputs the same product file. remove = true; AZStd::string consoleMsg = AZStd::string::format("Failing Job (source : %s , jobkey %s) because another job (source : %s , jobkey : %s ) outputted the same product %s.\n", itProcessedAsset->m_entry.m_pathRelativeToWatchFolder.toUtf8().constData(), itProcessedAsset->m_entry.m_jobKey.toUtf8().data(), source.m_sourceName.c_str(), job.m_jobKey.c_str(), newProductName.toUtf8().constData()); @@ -1128,7 +1128,7 @@ namespace AssetProcessor QString fullProductPath = m_cacheRootDir.absoluteFilePath(productName); // Strip the from the front of a relative product path - QString relativeProductPath = AssetUtilities::StripAssetPlatform(priorProduct.m_productName); + AZStd::string_view relativeProductPath = AssetUtilities::StripAssetPlatformNoCopy(priorProduct.m_productName); AZ::Data::AssetId assetId(source.m_sourceGuid, priorProduct.m_subID); @@ -1137,7 +1137,7 @@ namespace AssetProcessor AZ::Data::AssetId legacyAssetId(priorProduct.m_legacyGuid, 0); AZ::Data::AssetId legacySourceAssetId(AssetUtilities::CreateSafeSourceUUIDFromName(source.m_sourceName.c_str(), false), priorProduct.m_subID); - AssetNotificationMessage message(relativeProductPath.toUtf8().constData(), AssetNotificationMessage::AssetRemoved, priorProduct.m_assetType, processedAsset.m_entry.m_platformInfo.m_identifier.c_str()); + AssetNotificationMessage message(relativeProductPath, AssetNotificationMessage::AssetRemoved, priorProduct.m_assetType, processedAsset.m_entry.m_platformInfo.m_identifier.c_str()); message.m_assetId = assetId; if (legacyAssetId != assetId) @@ -1271,7 +1271,7 @@ namespace AssetProcessor [&](AzToolsFramework::AssetDatabase::ProductDependencyDatabaseEntry& dependencyEntry) { return dependencyEntry.m_dependencySubID == pair.first.m_subID - && dependencyEntry.m_dependencySourceGuid == source.m_sourceGuid; + && dependencyEntry.m_dependencySourceGuid == source.m_sourceGuid; }); if (conflictItr != dependencySet.end()) @@ -1312,14 +1312,14 @@ namespace AssetProcessor // relative file path is gotten by removing the platform and game from the product name // Strip the from the front of a relative product path - QString relativeProductPath = AssetUtilities::StripAssetPlatform(productName.toUtf8().constData()); + AZStd::string relativeProductPath = AssetUtilities::StripAssetPlatform(productName.toUtf8().constData()).toUtf8().constData(); - AssetNotificationMessage message(relativeProductPath.toUtf8().constData(), AssetNotificationMessage::AssetChanged, newProduct.m_assetType, processedAsset.m_entry.m_platformInfo.m_identifier.c_str()); + AssetNotificationMessage message(relativeProductPath, AssetNotificationMessage::AssetChanged, newProduct.m_assetType, processedAsset.m_entry.m_platformInfo.m_identifier.c_str()); AZ::Data::AssetId assetId(source.m_sourceGuid, newProduct.m_subID); AZ::Data::AssetId legacyAssetId(newProduct.m_legacyGuid, 0); AZ::Data::AssetId legacySourceAssetId(AssetUtilities::CreateSafeSourceUUIDFromName(source.m_sourceName.c_str(), false), newProduct.m_subID); - message.m_data = relativeProductPath.toUtf8().data(); + message.m_data = relativeProductPath; message.m_sizeBytes = QFileInfo(fullProductPath).size(); message.m_assetId = assetId; @@ -1350,7 +1350,7 @@ namespace AssetProcessor } Q_EMIT AssetMessage( message); - + AddKnownFoldersRecursivelyForFile(fullProductPath, m_cacheRootDir.absolutePath()); } @@ -1497,7 +1497,7 @@ namespace AssetProcessor // Record the modtime for the metadata file so we don't re-analyze this change again next time AP starts up QFileInfo metadataFileInfo(originalName); auto* scanFolder = m_platformConfig->GetScanFolderForFile(originalName); - + if (scanFolder) { QString databaseName; @@ -1534,7 +1534,7 @@ namespace AssetProcessor for (const QString& absolutePath : absoluteSourcePathList) { // we need to check if its already in the "active files" (things that we are looking over) - // or if its in the "currently being examined" list. The latter is likely to be the smaller list, + // or if its in the "currently being examined" list. The latter is likely to be the smaller list, // so we check it first. Both of those are absolute paths, so we convert to absolute path before // searching those lists: if (m_filesToExamine.find(absolutePath) != m_filesToExamine.end()) @@ -1633,9 +1633,6 @@ namespace AssetProcessor } } - // Strip the from the front of a relative product path - QString relativePath = AssetUtilities::StripAssetPlatform(relativeProductFile.toUtf8().constData()); - //set the fingerprint on the job that made this product for (auto& job : jobs) { @@ -1678,7 +1675,7 @@ namespace AssetProcessor } QString fullProductPath = m_cacheRootDir.absoluteFilePath(product.m_productName.c_str()); - QString relativeProductPath(AssetUtilities::StripAssetPlatform(product.m_productName)); + AZStd::string_view relativeProductPath = AssetUtilities::StripAssetPlatformNoCopy(product.m_productName); QFileInfo productFileInfo(fullProductPath); if (productFileInfo.exists()) { @@ -1725,7 +1722,7 @@ namespace AssetProcessor AZ::Data::AssetId legacyAssetId(product.m_legacyGuid, 0); AZ::Data::AssetId legacySourceAssetId(AssetUtilities::CreateSafeSourceUUIDFromName(source.m_sourceName.c_str(), false), product.m_subID); - AssetNotificationMessage message(relativeProductPath.toUtf8().constData(), AssetNotificationMessage::AssetRemoved, product.m_assetType, job.m_platform.c_str()); + AssetNotificationMessage message(relativeProductPath, AssetNotificationMessage::AssetRemoved, product.m_assetType, job.m_platform.c_str()); message.m_assetId = assetId; if (legacyAssetId != assetId) @@ -1759,7 +1756,7 @@ namespace AssetProcessor // and no overrides exist for it. // we must delete its products. using namespace AzToolsFramework::AssetDatabase; - + // If we fail to delete a product, the deletion event gets requeued // To avoid retrying forever, we keep track of the time of the first deletion failure and only retry // if less than this amount of time has passed. @@ -1832,7 +1829,7 @@ namespace AssetProcessor { return; } - + // Check if this file causes any file types to be re-evaluated CheckMetaDataRealFiles(normalizedPath); @@ -2046,7 +2043,7 @@ namespace AssetProcessor AZ_TracePrintf(AssetProcessor::DebugChannel, "Non-processed file: %s\n", databaseSourceFile.toUtf8().constData()); ++m_numSourcesNotHandledByAnyBuilder; - + // Record the modtime for the file so we know we've already processed it QString absolutePath = QDir(scanFolder->ScanPath()).absoluteFilePath(normalizedPath); @@ -2114,7 +2111,7 @@ namespace AssetProcessor // Check whether another job emitted this job as a job dependency and if true, queue the dependent job source file also JobDesc jobDesc(jobDetails.m_jobEntry.m_databaseSourceName.toUtf8().data(), jobDetails.m_jobEntry.m_jobKey.toUtf8().data(), jobDetails.m_jobEntry.m_platformInfo.m_identifier); - + shouldProcessAsset = true; QFileInfo file(jobDetails.m_jobEntry.GetAbsoluteSourcePath()); QDateTime dateTime(file.lastModified()); @@ -2347,7 +2344,7 @@ namespace AssetProcessor } QString canonicalRootDir = AssetUtilities::NormalizeFilePath(m_cacheRootDir.canonicalPath()); - + FileExamineContainer swapped; m_filesToExamine.swap(swapped); // makes it okay to call CheckSource(...) @@ -2470,7 +2467,7 @@ namespace AssetProcessor AZ_TracePrintf(AssetProcessor::DebugChannel, "ProcessFilesToExamineQueue: Unable to find the relative path.\n"); continue; } - + const ScanFolderInfo* scanFolderInfo = m_platformConfig->GetScanFolderForFile(normalizedPath); relativePathToFile = databasePathToFile; @@ -2494,9 +2491,9 @@ namespace AssetProcessor QString::fromUtf8(jobInfo.m_watchFolder.c_str()), relativePathToFile, databasePathToFile, - jobInfo.m_builderGuid, - *platformFromInfo, - jobInfo.m_jobKey.c_str(), 0, GenerateNewJobRunKey(), + jobInfo.m_builderGuid, + *platformFromInfo, + jobInfo.m_jobKey.c_str(), 0, GenerateNewJobRunKey(), AZ::Uuid::CreateNull()); job.m_autoFail = true; @@ -2597,7 +2594,7 @@ namespace AssetProcessor { // on the other hand, if we found a file it means that a deleted file revealed a file that // was previously overridden by it. - // Because the deleted file may have "revealed" a file with different case, + // Because the deleted file may have "revealed" a file with different case, // we have to actually correct its case here. This is rare, so it should be reasonable // to call the expensive function to discover correct case. QString pathRelativeToScanFolder; @@ -2674,6 +2671,7 @@ namespace AssetProcessor AZ_TracePrintf(ConsoleChannel, "Builder optimization: %i / %i files required full analysis, %i sources found but not processed by anyone\n", m_numSourcesNeedingFullAnalysis, m_numTotalSourcesFound, m_numSourcesNotHandledByAnyBuilder); } + m_pathDependencyManager->ProcessQueuedDependencyResolves(); QTimer::singleShot(20, this, SLOT(RemoveEmptyFolders())); } else @@ -2739,7 +2737,7 @@ namespace AssetProcessor } // over here we also want to invalidate the metafiles on disk map if it COULD Be a metafile - // note that there is no reason to do an expensive exacting computation here, it will be + // note that there is no reason to do an expensive exacting computation here, it will be // done later and cached when m_cachedMetaFilesExistMap is set to false, we just need to // know if its POSSIBLE that its a metafile, cheaply. // if its a metafile match, then invalidate the metafile table. @@ -2752,7 +2750,7 @@ namespace AssetProcessor m_metaFilesWhichActuallyExistOnDisk.clear(); // invalidate the map, force a recompuation later. } } - + } m_AssetProcessorIsBusy = true; @@ -2927,7 +2925,7 @@ namespace AssetProcessor } AZ::u64 fileHash = AssetUtilities::GetFileHash(fileInfo.m_filePath.toUtf8().constData()); - + if(fileHash != databaseHashValue) { // File contents have changed @@ -3149,7 +3147,7 @@ namespace AssetProcessor { AddMetadataFilesForFingerprinting(kvp.first.c_str(), job.m_fingerprintFiles); } - + // Check the current builder jobs with the previous ones in the database: job.m_jobEntry.m_computedFingerprint = AssetUtilities::GenerateFingerprint(job); JobIndentifier jobIndentifier(JobDesc(job.m_jobEntry.m_databaseSourceName.toUtf8().data(), job.m_jobEntry.m_jobKey.toUtf8().data(), job.m_jobEntry.m_platformInfo.m_identifier), job.m_jobEntry.m_builderGuid); @@ -3208,7 +3206,7 @@ namespace AssetProcessor { // If the database knows about the job than it implies that AP has processed it sucessfully at least once // and therefore the dependent job should not cause the job which depends on it to be processed again. - // If however we find a dependent job which is not known to AP then we know this job needs to be processed + // If however we find a dependent job which is not known to AP then we know this job needs to be processed // after all the dependent jobs have completed at least once. AzToolsFramework::AssetDatabase::JobDatabaseEntryContainer jobs; @@ -3238,7 +3236,7 @@ namespace AssetProcessor } else if(sourceFileDependency.m_sourceDependencyType != AssetBuilderSDK::SourceFileDependency::SourceFileDependencyType::Wildcards) { - AZ_TracePrintf(AssetProcessor::ConsoleChannel, "UpdateJobDependency: Failed to find builder dependency for %s job (%s, %s, %s)\n", + AZ_TracePrintf(AssetProcessor::ConsoleChannel, "UpdateJobDependency: Failed to find builder dependency for %s job (%s, %s, %s)\n", job.m_jobEntry.GetAbsoluteSourcePath().toUtf8().constData(), jobDependencyInternal->m_jobDependency.m_sourceFile.m_sourceFileDependencyPath.c_str(), jobDependencyInternal->m_jobDependency.m_jobKey.c_str(), @@ -3262,7 +3260,7 @@ namespace AssetProcessor ++jobDependencySlot; } - // sorting job dependencies as they can effect the fingerprint of the job + // sorting job dependencies as they can effect the fingerprint of the job AZStd::sort(job.m_jobDependencyList.begin(), job.m_jobDependencyList.end(), [](const AssetProcessor::JobDependencyInternal& lhs, const AssetProcessor::JobDependencyInternal& rhs) { @@ -3289,7 +3287,7 @@ namespace AssetProcessor for (const JobDependencyInternal& jobDependencyInternal : job.m_jobDependencyList) { // Loop over all the builderUuid and check whether the corresponding entry exists in the jobsFingerprint map. - // If an entry exists, it implies than we have already send the job over to the RCController + // If an entry exists, it implies than we have already send the job over to the RCController for (auto builderIter = jobDependencyInternal.m_builderUuidList.begin(); builderIter != jobDependencyInternal.m_builderUuidList.end(); ++builderIter) { JobIndentifier jobIdentifier(JobDesc(jobDependencyInternal.m_jobDependency.m_sourceFile.m_sourceFileDependencyPath, @@ -3299,7 +3297,7 @@ namespace AssetProcessor auto jobFound = m_jobFingerprintMap.find(jobIdentifier); if (jobFound == m_jobFingerprintMap.end()) { - // Job cannot be processed, since one of its dependent job hasn't been fingerprinted + // Job cannot be processed, since one of its dependent job hasn't been fingerprinted return false; } } @@ -3317,7 +3315,7 @@ namespace AssetProcessor // and call the CreateJobs function on the builder. // it bundles the results up in a JobToProcessEntry struct, while it is doing this: JobToProcessEntry entry; - + AZ::Uuid sourceUUID = AssetUtilities::CreateSafeSourceUUIDFromName(databasePathToFile.toUtf8().constData()); // first, we put the source UUID in the map so that its present for any other queries: @@ -3375,14 +3373,14 @@ namespace AssetProcessor builderInfo.m_createJobFunction(createJobsRequest, createJobsResponse); AssetProcessor::StatsCapture::EndCaptureStat(statKey.toUtf8().constData()); } - + AssetProcessor::SetThreadLocalJobId(0); bool isBuilderMissingFingerprint = (createJobsResponse.m_result == AssetBuilderSDK::CreateJobsResultCode::Success && !createJobsResponse.m_createJobOutputs.empty() && !createJobsResponse.m_createJobOutputs[0].m_additionalFingerprintInfo.empty() && builderInfo.m_analysisFingerprint.empty()); - + if (createJobsResponse.m_result == AssetBuilderSDK::CreateJobsResultCode::Failed || isBuilderMissingFingerprint) { AZ_TracePrintf(AssetProcessor::ConsoleChannel, "Createjobs Failed: %s.\n", normalizedPath.toUtf8().constData()); @@ -3393,7 +3391,7 @@ namespace AssetProcessor char resolvedBuffer[AZ_MAX_PATH_LEN] = { 0 }; AZ::IO::FileIOBase::GetInstance()->ResolvePath(fullPathToLogFile.c_str(), resolvedBuffer, AZ_MAX_PATH_LEN); - + JobDetails jobdetail; jobdetail.m_jobEntry = JobEntry( scanFolder->ScanPath(), @@ -3486,9 +3484,9 @@ namespace AssetProcessor scanFolder->ScanPath(), actualRelativePath, databasePathToFile, - builderInfo.m_busId, - *infoForPlatform, - jobDescriptor.m_jobKey.c_str(), 0, GenerateNewJobRunKey(), + builderInfo.m_busId, + *infoForPlatform, + jobDescriptor.m_jobKey.c_str(), 0, GenerateNewJobRunKey(), sourceUUID); newJob.m_jobEntry.m_checkExclusiveLock = jobDescriptor.m_checkExclusiveLock; newJob.m_jobParam = AZStd::move(jobDescriptor.m_jobParameters); @@ -3506,7 +3504,7 @@ namespace AssetProcessor newJob.m_jobDependencyList.push_back(JobDependencyInternal(jobDependency)); ++numJobDependencies; } - + // note that until analysis completes, the jobId is not set and neither is the destination pat JobDesc jobDesc(newJob.m_jobEntry.m_databaseSourceName.toUtf8().data(), newJob.m_jobEntry.m_jobKey.toUtf8().data(), newJob.m_jobEntry.m_platformInfo.m_identifier); m_jobDescToBuilderUuidMap[jobDesc].insert(builderInfo.m_busId); @@ -3515,7 +3513,7 @@ namespace AssetProcessor JobIndentifier jobIdentifier(jobDesc, builderInfo.m_busId); { AZStd::lock_guard lock(AssetProcessor::ProcessingJobInfoBus::GetOrCreateContext().m_contextMutex); - m_jobFingerprintMap.erase(jobIdentifier); + m_jobFingerprintMap.erase(jobIdentifier); } entry.m_jobsToAnalyze.push_back(AZStd::move(newJob)); @@ -3578,7 +3576,7 @@ namespace AssetProcessor // instead of a UUID, a path has been provided, prepare and use that. We need to turn it into a database path QString encodedFileData = QString::fromUtf8(sourceDependency.m_sourceFileDependencyPath.c_str()); encodedFileData = AssetUtilities::NormalizeFilePath(encodedFileData); - + if (sourceDependency.m_sourceDependencyType == AssetBuilderSDK::SourceFileDependency::SourceFileDependencyType::Wildcards) { int wildcardIndex = encodedFileData.indexOf("*"); @@ -3653,7 +3651,7 @@ namespace AssetProcessor } // Convert to relative paths - for (auto dependencyItr = resolvedDependencyList.begin(); dependencyItr != resolvedDependencyList.end();) + for (auto dependencyItr = resolvedDependencyList.begin(); dependencyItr != resolvedDependencyList.end();) { QString relativePath, scanFolder; if (m_platformConfig->ConvertToRelativePath(*dependencyItr, relativePath, scanFolder)) @@ -3705,7 +3703,7 @@ namespace AssetProcessor return (!resultDatabaseSourceName.isEmpty()); } - + void AssetProcessorManager::UpdateSourceFileDependenciesDatabase(JobToProcessEntry& entry) { using namespace AzToolsFramework::AssetDatabase; @@ -3738,7 +3736,7 @@ namespace AssetProcessor QString resolvedDatabaseName; if (!ResolveSourceFileDependencyPath(sourceDependency.second, resolvedDatabaseName,resolvedDependencyList)) { - // ResolveDependencyPath should only fail in a data error, otherwise it always outputs something, + // ResolveDependencyPath should only fail in a data error, otherwise it always outputs something, // even if that something starts with the placeholder. continue; } @@ -3759,7 +3757,7 @@ namespace AssetProcessor SourceFileDependencyEntry newDependencyEntry( sourceDependency.first, entry.m_sourceFileInfo.m_databasePath.toUtf8().constData(), - resolvedDatabaseName.toUtf8().constData(), + resolvedDatabaseName.toUtf8().constData(), sourceDependency.second.m_sourceDependencyType == AssetBuilderSDK::SourceFileDependency::SourceFileDependencyType::Wildcards ? SourceFileDependencyEntry::DEP_SourceLikeMatch : SourceFileDependencyEntry::DEP_SourceToSource, !sourceDependency.second.m_sourceFileDependencyUUID.IsNull()); // If the UUID is null, then record that this dependency came from a (resolved) path newDependencies.push_back(AZStd::move(newDependencyEntry)); @@ -3803,7 +3801,7 @@ namespace AssetProcessor } // get all the old dependencies and remove them. This function is comprehensive on all dependencies - // for a given source file so we can just eliminate all of them from that same source file and replace + // for a given source file so we can just eliminate all of them from that same source file and replace // them with all of the new ones for the given source file: AZStd::unordered_set oldDependencies; m_stateData->QueryDependsOnSourceBySourceDependency( @@ -4018,7 +4016,7 @@ namespace AssetProcessor result.m_watchFolder = QString::fromUtf8(scanFolder.m_scanFolder.c_str()); result.m_sourceRelativeToWatchFolder = result.m_sourceDatabaseName; - { + { // this scope exists to restrict the duration of the below lock. AZStd::lock_guard lock(m_sourceUUIDToSourceInfoMapMutex); m_sourceUUIDToSourceInfoMap.insert(AZStd::make_pair(sourceUuid, result)); @@ -4045,9 +4043,9 @@ namespace AssetProcessor auto jobPair = m_jobsToProcess.insert(AZStd::move(jobDetail)); if (!jobPair.second) { - // if we are here it means that this job was already found in the jobs to process list + // if we are here it means that this job was already found in the jobs to process list // and therefore insert failed, we will try to update the iterator manually here. - // Note that if insert fails the original object is not destroyed and therefore we can use move again. + // Note that if insert fails the original object is not destroyed and therefore we can use move again. // we just replaced a job, so we have to decrement its count. UpdateAnalysisTrackerForFile(jobPair.first->m_jobEntry, AnalysisTrackerUpdateType::JobFinished); @@ -4068,7 +4066,7 @@ namespace AssetProcessor QSet absoluteSourceFilePathQueue; QString databasePath; QString scanFolder; - + auto callbackFunction = [this, &absoluteSourceFilePathQueue](SourceFileDependencyEntry& entry) { QString relativeDatabaseName = QString::fromUtf8(entry.m_source.c_str()); @@ -4111,7 +4109,7 @@ namespace AssetProcessor sourceDatabaseEntry.m_sourceName = relativeSourceFilePath.toUtf8().constData(); sourceDatabaseEntry.m_sourceGuid = AssetUtilities::CreateSafeSourceUUIDFromName(sourceDatabaseEntry.m_sourceName.c_str()); - + if (!m_stateData->SetSource(sourceDatabaseEntry)) { AZ_Error(AssetProcessor::ConsoleChannel, false, "Failed to add source to the database!!!"); @@ -4260,7 +4258,7 @@ namespace AssetProcessor { AZ_TracePrintf(DebugChannel, "Builder %s analysis fingerprint changed. Files assigned to it will be re-analyzed.\n", priorBuilderUUID.ToString().c_str()); } - + if (builderIsDirty) { m_anyBuilderChange = true; @@ -4414,7 +4412,7 @@ namespace AssetProcessor source.m_analysisFingerprint.append(builderFP.ToString()); } - m_pathDependencyManager->RetryDeferredDependencies(source); + m_pathDependencyManager->QueueSourceForDependencyResolution(source); m_stateData->SetSource(source); databaseSourceName = source.m_sourceName.c_str(); @@ -4593,7 +4591,7 @@ namespace AssetProcessor { continue; } - + QString firstMatchingFile = m_platformConfig->FindFirstMatchingFile(dep); if (firstMatchingFile.isEmpty()) { @@ -4787,7 +4785,7 @@ namespace AssetProcessor assetIter->m_entry.m_sourceFileUUID); jobdetail.m_autoFail = true; jobdetail.m_critical = true; - jobdetail.m_priority = INT_MAX; // front of the queue. + jobdetail.m_priority = INT_MAX; // front of the queue. jobdetail.m_scanFolder = m_platformConfig->GetScanFolderForFile(assetIter->m_entry.GetAbsoluteSourcePath()); // the new lines make it easier to copy and paste the file names. jobdetail.m_jobParam[AZ_CRC(AutoFailReasonKey)] = autoFailReason; @@ -4859,6 +4857,6 @@ namespace AssetProcessor return filesFound; } - + } // namespace AssetProcessor diff --git a/Code/Tools/AssetProcessor/native/tests/BaseAssetProcessorTest.h b/Code/Tools/AssetProcessor/native/tests/BaseAssetProcessorTest.h index 71d3fa0f6d..be7e78d097 100644 --- a/Code/Tools/AssetProcessor/native/tests/BaseAssetProcessorTest.h +++ b/Code/Tools/AssetProcessor/native/tests/BaseAssetProcessorTest.h @@ -40,12 +40,14 @@ protected: void SetupEnvironment() override { // Setup code + AZ::Environment::Create(nullptr); qInstallMessageHandler(UnitTestMessageHandler); } void TeardownEnvironment() override { qInstallMessageHandler(nullptr); + AZ::Environment::Destroy(); } private: diff --git a/Code/Tools/AssetProcessor/native/tests/PathDependencyManagerTests.cpp b/Code/Tools/AssetProcessor/native/tests/PathDependencyManagerTests.cpp index b20d373307..1d6e92e47e 100644 --- a/Code/Tools/AssetProcessor/native/tests/PathDependencyManagerTests.cpp +++ b/Code/Tools/AssetProcessor/native/tests/PathDependencyManagerTests.cpp @@ -12,13 +12,23 @@ #include "AzToolsFramework/API/AssetDatabaseBus.h" #include "AssetDatabase/AssetDatabase.h" #include +#include +#include +#include +#include namespace UnitTests { class MockDatabaseLocationListener : public AzToolsFramework::AssetDatabase::AssetDatabaseRequests::Bus::Handler { public: - MOCK_METHOD1(GetAssetDatabaseLocation, bool(AZStd::string&)); + bool GetAssetDatabaseLocation(AZStd::string& location) override + { + location = m_databaseLocation; + return true; + } + + AZStd::string m_databaseLocation; }; namespace Util @@ -38,25 +48,45 @@ namespace UnitTests } } - struct PathDependencyDeletionTest - : UnitTest::ScopedAllocatorSetupFixture - , UnitTest::TraceBusRedirector + struct PathDependencyBase + : UnitTest::TraceBusRedirector { - void SetUp() override; - void TearDown() override; + void Init(); + void Destroy(); QTemporaryDir m_tempDir; AZStd::string m_databaseLocation; - ::testing::NiceMock m_databaseLocationListener; + MockDatabaseLocationListener m_databaseLocationListener; AZStd::shared_ptr m_stateData; AZStd::unique_ptr m_platformConfig; + AZStd::unique_ptr m_serializeContext; + AZ::Entity* m_jobManagerEntity{}; + AZ::ComponentDescriptor* m_descriptor{}; }; - void PathDependencyDeletionTest::SetUp() + struct PathDependencyDeletionTest + : UnitTest::ScopedAllocatorSetupFixture + , PathDependencyBase + { + void SetUp() override + { + PathDependencyBase::Init(); + } + + void TearDown() override + { + PathDependencyBase::Destroy(); + } + }; + + void PathDependencyBase::Init() { using namespace ::testing; using namespace AzToolsFramework::AssetDatabase; + ::UnitTest::TestRunner::Instance().m_suppressAsserts = false; + ::UnitTest::TestRunner::Instance().m_suppressErrors = false; + BusConnect(); QDir tempPath(m_tempDir.path()); @@ -68,21 +98,39 @@ namespace UnitTests // ":memory:" databases are one-instance-only, and even if another connection is opened to ":memory:" it would // not share with others created using ":memory:" and get a unique database instead. m_databaseLocation = tempPath.absoluteFilePath("test_database.sqlite").toUtf8().constData(); - - ON_CALL(m_databaseLocationListener, GetAssetDatabaseLocation(_)) - .WillByDefault( - DoAll( // set the 0th argument ref (string) to the database location and return true. - SetArgReferee<0>(m_databaseLocation), - Return(true))); + m_databaseLocationListener.m_databaseLocation = m_databaseLocation; m_stateData = AZStd::shared_ptr(new AssetProcessor::AssetDatabaseConnection()); m_stateData->OpenDatabase(); m_platformConfig = AZStd::make_unique(); + + AZ::AllocatorInstance::Create(); + AZ::AllocatorInstance::Create(); + + m_serializeContext = AZStd::make_unique(); + m_descriptor = AZ::JobManagerComponent::CreateDescriptor(); + m_descriptor->Reflect(m_serializeContext.get()); + + m_jobManagerEntity = aznew AZ::Entity{}; + m_jobManagerEntity->CreateComponent(); + m_jobManagerEntity->Init(); + m_jobManagerEntity->Activate(); } - void PathDependencyDeletionTest::TearDown() + void PathDependencyBase::Destroy() { + m_stateData = nullptr; + m_platformConfig = nullptr; + + m_jobManagerEntity->Deactivate(); + delete m_jobManagerEntity; + + delete m_descriptor; + + AZ::AllocatorInstance::Destroy(); + AZ::AllocatorInstance::Destroy(); + BusDisconnect(); } @@ -91,7 +139,7 @@ namespace UnitTests using namespace AzToolsFramework::AssetDatabase; // Add a product to the db with an unmet dependency - ScanFolderDatabaseEntry scanFolder("folder", "test", "test", ""); + ScanFolderDatabaseEntry scanFolder("folder", "test", "test", 0); m_stateData->SetScanFolder(scanFolder); SourceDatabaseEntry source1, source2; @@ -110,7 +158,8 @@ namespace UnitTests Util::CreateSourceJobAndProduct(m_stateData.get(), scanFolder.m_scanFolderID, source2, job2, product2, "source2.txt", "product2.jpg"); - manager.RetryDeferredDependencies(source2); + manager.QueueSourceForDependencyResolution(source2); + manager.ProcessQueuedDependencyResolves(); } TEST_F(PathDependencyDeletionTest, ExistingSourceWithUnmetDependency_RemovedFromDB_DependentProductCreatedWithoutError) @@ -118,7 +167,7 @@ namespace UnitTests using namespace AzToolsFramework::AssetDatabase; // Add a product to the db with an unmet dependency - ScanFolderDatabaseEntry scanFolder("folder", "test", "test", ""); + ScanFolderDatabaseEntry scanFolder("folder", "test", "test", 0); m_stateData->SetScanFolder(scanFolder); SourceDatabaseEntry source1, source2; @@ -137,7 +186,8 @@ namespace UnitTests Util::CreateSourceJobAndProduct(m_stateData.get(), scanFolder.m_scanFolderID, source2, job2, product2, "source2.txt", "product2.jpg"); - manager.RetryDeferredDependencies(source2); + manager.QueueSourceForDependencyResolution(source2); + manager.ProcessQueuedDependencyResolves(); } TEST_F(PathDependencyDeletionTest, NewSourceWithUnmetDependency_RemovedFromDB_DependentSourceCreatedWithoutError) @@ -147,7 +197,7 @@ namespace UnitTests AssetProcessor::PathDependencyManager manager(m_stateData, m_platformConfig.get()); // Add a product to the db with an unmet dependency - ScanFolderDatabaseEntry scanFolder("folder", "test", "test", ""); + ScanFolderDatabaseEntry scanFolder("folder", "test", "test", 0); m_stateData->SetScanFolder(scanFolder); SourceDatabaseEntry source1, source2; @@ -166,7 +216,8 @@ namespace UnitTests Util::CreateSourceJobAndProduct(m_stateData.get(), scanFolder.m_scanFolderID, source2, job2, product2, "source2.txt", "product2.jpg"); - manager.RetryDeferredDependencies(source2); + manager.QueueSourceForDependencyResolution(source2); + manager.ProcessQueuedDependencyResolves(); } TEST_F(PathDependencyDeletionTest, NewSourceWithUnmetDependency_RemovedFromDB_DependentProductCreatedWithoutError) @@ -176,7 +227,7 @@ namespace UnitTests AssetProcessor::PathDependencyManager manager(m_stateData, m_platformConfig.get()); // Add a product to the db with an unmet dependency - ScanFolderDatabaseEntry scanFolder("folder", "test", "test", ""); + ScanFolderDatabaseEntry scanFolder("folder", "test", "test", 0); m_stateData->SetScanFolder(scanFolder); SourceDatabaseEntry source1, source2; @@ -195,7 +246,8 @@ namespace UnitTests Util::CreateSourceJobAndProduct(m_stateData.get(), scanFolder.m_scanFolderID, source2, job2, product2, "source2.txt", "product2.jpg"); - manager.RetryDeferredDependencies(source2); + manager.QueueSourceForDependencyResolution(source2); + manager.ProcessQueuedDependencyResolves(); } TEST_F(PathDependencyDeletionTest, NewSourceWithUnmetDependency_Wildcard_RemovedFromDB_DependentSourceCreatedWithoutError) @@ -205,7 +257,7 @@ namespace UnitTests AssetProcessor::PathDependencyManager manager(m_stateData, m_platformConfig.get()); // Add a product to the db with an unmet dependency - ScanFolderDatabaseEntry scanFolder("folder", "test", "test", ""); + ScanFolderDatabaseEntry scanFolder("folder", "test", "test", 0); m_stateData->SetScanFolder(scanFolder); SourceDatabaseEntry source1, source2; @@ -224,6 +276,266 @@ namespace UnitTests Util::CreateSourceJobAndProduct(m_stateData.get(), scanFolder.m_scanFolderID, source2, job2, product2, "source2.txt", "product2.jpg"); - manager.RetryDeferredDependencies(source2); + manager.QueueSourceForDependencyResolution(source2); + manager.ProcessQueuedDependencyResolves(); + } + + using PathDependencyTests = PathDependencyDeletionTest; + + TEST_F(PathDependencyTests, SourceAndProductHaveSameName_SourceFileDependency_MatchesSource) + { + using namespace AzToolsFramework::AssetDatabase; + + AssetProcessor::PathDependencyManager manager(m_stateData, m_platformConfig.get()); + + ScanFolderDatabaseEntry scanFolder("folder", "test", "test", 0); + m_stateData->SetScanFolder(scanFolder); + + SourceDatabaseEntry source1, source2; + JobDatabaseEntry job1, job2; + ProductDatabaseEntry product1, product2, product3; + + Util::CreateSourceJobAndProduct( + m_stateData.get(), scanFolder.m_scanFolderID, source1, job1, product1, "source1.txt", "product1.jpg"); + + AssetBuilderSDK::ProductPathDependencySet set; + set.insert(AssetBuilderSDK::ProductPathDependency("*.xml", AssetBuilderSDK::ProductPathDependencyType::SourceFile)); + + manager.SaveUnresolvedDependenciesToDatabase(set, product1, "pc"); + + Util::CreateSourceJobAndProduct( + m_stateData.get(), scanFolder.m_scanFolderID, source2, job2, product2, "source2.xml", "source2.xml"); + + // Create a 2nd product for this source + product3 = ProductDatabaseEntry{ job2.m_jobID, product2.m_subID + 1, "source2.txt", AZ::Data::AssetType::CreateRandom() }; + ASSERT_TRUE(m_stateData->SetProduct(product3)); + + manager.QueueSourceForDependencyResolution(source2); + manager.ProcessQueuedDependencyResolves(); + + ProductDependencyDatabaseEntryContainer productDependencies; + m_stateData->GetProductDependencies(productDependencies); + + EXPECT_EQ(productDependencies.size(), 3); + } + + TEST_F(PathDependencyTests, SourceAndProductHaveSameName_ProductFileDependency_MatchesProduct) + { + using namespace AzToolsFramework::AssetDatabase; + + AssetProcessor::PathDependencyManager manager(m_stateData, m_platformConfig.get()); + + ScanFolderDatabaseEntry scanFolder("folder", "test", "test", 0); + m_stateData->SetScanFolder(scanFolder); + + SourceDatabaseEntry source1, source2; + JobDatabaseEntry job1, job2; + ProductDatabaseEntry product1, product2, product3; + + Util::CreateSourceJobAndProduct( + m_stateData.get(), scanFolder.m_scanFolderID, source1, job1, product1, "source1.txt", "product1.jpg"); + + AssetBuilderSDK::ProductPathDependencySet set; + set.insert(AssetBuilderSDK::ProductPathDependency("*.xml", AssetBuilderSDK::ProductPathDependencyType::ProductFile)); + + manager.SaveUnresolvedDependenciesToDatabase(set, product1, "pc"); + + Util::CreateSourceJobAndProduct( + m_stateData.get(), scanFolder.m_scanFolderID, source2, job2, product2, "source2.xml", "source2.xml"); + + // Create a 2nd product for this source + product3 = ProductDatabaseEntry{job2.m_jobID, product2.m_subID + 1, "source2.txt", AZ::Data::AssetType::CreateRandom()}; + ASSERT_TRUE(m_stateData->SetProduct(product3)); + + manager.QueueSourceForDependencyResolution(source2); + manager.ProcessQueuedDependencyResolves(); + + ProductDependencyDatabaseEntryContainer productDependencies; + m_stateData->GetProductDependencies(productDependencies); + + EXPECT_EQ(productDependencies.size(), 2); } + + struct PathDependencyBenchmarks + : UnitTest::ScopedAllocatorFixture + , PathDependencyBase + { + static inline constexpr int NumTestDependencies = 4; // Must be a multiple of 4 + static inline constexpr int NumTestProducts = 2; // Must be a multiple of 2 + + AzToolsFramework::AssetDatabase::ProductDatabaseEntryContainer m_products; + AzToolsFramework::AssetDatabase::SourceDatabaseEntry m_source1, m_source2, m_source4; + AzToolsFramework::AssetDatabase::JobDatabaseEntry m_job1, m_job2, m_job4; + AzToolsFramework::AssetDatabase::ProductDatabaseEntry m_product1, m_product2, m_product4; + AzToolsFramework::AssetDatabase::ProductDependencyDatabaseEntryContainer m_dependencies; + + void SetupTestData() + { + using namespace AzToolsFramework::AssetDatabase; + + ScanFolderDatabaseEntry scanFolder("folder", "test", "test", 0); + ASSERT_TRUE(m_stateData->SetScanFolder(scanFolder)); + + Util::CreateSourceJobAndProduct( + m_stateData.get(), scanFolder.m_scanFolderID, m_source1, m_job1, m_product1, "source1.txt", "product1.jpg"); + + Util::CreateSourceJobAndProduct( + m_stateData.get(), scanFolder.m_scanFolderID, m_source4, m_job4, m_product4, "source4.txt", "product4.jpg"); + + for (int i = 0; i < NumTestDependencies / 2; ++i) + { + m_dependencies.emplace_back( + m_product1.m_productID, AZ::Uuid::CreateNull(), 0, 0, "pc", 0, + AZStd::string::format("folder/folder2/%d_*2.jpg", i).c_str()); + ++i; + m_dependencies.emplace_back( + m_product1.m_productID, AZ::Uuid::CreateNull(), 0, 0, "mac", 0, + AZStd::string::format("folder/folder2/%d_*2.jpg", i).c_str()); + } + + for (int i = 0; i < NumTestDependencies / 2; ++i) + { + m_dependencies.emplace_back( + m_product4.m_productID, AZ::Uuid::CreateNull(), 0, 0, "pc", 0, + AZStd::string::format("folder/folder2/%d_*2.jpg", i).c_str()); + ++i; + m_dependencies.emplace_back( + m_product4.m_productID, AZ::Uuid::CreateNull(), 0, 0, "mac", 0, + AZStd::string::format("folder/folder2/%d_*2.jpg", i).c_str()); + } + + ASSERT_TRUE(m_stateData->SetProductDependencies(m_dependencies)); + + Util::CreateSourceJobAndProduct( + m_stateData.get(), scanFolder.m_scanFolderID, m_source2, m_job2, m_product2, "source2.txt", "product2.jpg"); + + auto job3 = JobDatabaseEntry( + m_source2.m_sourceID, "jobkey", 1111, "mac", AZ::Uuid::CreateRandom(), AzToolsFramework::AssetSystem::JobStatus::Completed, + 4444); + ASSERT_TRUE(m_stateData->SetJob(job3)); + + for (int i = 0; i < NumTestProducts; ++i) + { + m_products.emplace_back( + m_job2.m_jobID, i, AZStd::string::format("pc/folder/folder2/%d_product2.jpg", i).c_str(), + AZ::Data::AssetType::CreateRandom()); + ++i; + m_products.emplace_back( + job3.m_jobID, i, AZStd::string::format("mac/folder/folder2/%d_product2.jpg", i).c_str(), + AZ::Data::AssetType::CreateRandom()); + } + + ASSERT_TRUE(m_stateData->SetProducts(m_products)); + } + + void DoTest() + { + AssetProcessor::PathDependencyManager manager(m_stateData, m_platformConfig.get()); + + manager.QueueSourceForDependencyResolution(m_source2); + manager.ProcessQueuedDependencyResolves(); + } + + void VerifyResult() + { + using namespace AzToolsFramework::AssetDatabase; + + ProductDependencyDatabaseEntryContainer productDependencies; + m_stateData->GetProductDependencies(productDependencies); + + for (int i = 0; i < NumTestDependencies / 2 && i < NumTestProducts; ++i) + { + const auto& product = m_products[i]; + int found = 0; + + for (const auto& unresolvedProductDependency : productDependencies) + { + if (unresolvedProductDependency.m_dependencySourceGuid == m_source2.m_sourceGuid && + unresolvedProductDependency.m_dependencySubID == product.m_subID && + unresolvedProductDependency.m_productPK == m_product1.m_productID) + { + ++found; + } + + if (unresolvedProductDependency.m_dependencySourceGuid == m_source2.m_sourceGuid && + unresolvedProductDependency.m_dependencySubID == product.m_subID && + unresolvedProductDependency.m_productPK == m_product4.m_productID) + { + ++found; + } + + if (found == 2) + break; + } + + EXPECT_TRUE(found == 2) << product.m_productName.c_str() << " was not found"; + } + + EXPECT_EQ(productDependencies.size(), NumTestDependencies * 2); + } + }; + + // For some reason, BENCHMARK_F doesn't seem to call the destructor + // So we'll wrap the class and handle the new/delete ourselves + struct PathDependencyBenchmarksWrapperClass : public ::benchmark::Fixture + { + void SetUp([[maybe_unused]] const benchmark::State& st) override + { + m_benchmarks = new PathDependencyBenchmarks(); + m_benchmarks->Init(); + m_benchmarks->SetupTestData(); + } + + void SetUp([[maybe_unused]] benchmark::State& st) override + { + m_benchmarks = new PathDependencyBenchmarks(); + m_benchmarks->Init(); + m_benchmarks->SetupTestData(); + } + + void TearDown([[maybe_unused]] benchmark::State& st) override + { + m_benchmarks->Destroy(); + delete m_benchmarks; + } + + void TearDown([[maybe_unused]] const benchmark::State& st) override + { + m_benchmarks->Destroy(); + delete m_benchmarks; + } + + PathDependencyBenchmarks* m_benchmarks = {}; + }; + + struct PathDependencyTestValidation + : PathDependencyBenchmarks, ::testing::Test + { + void SetUp() override + { + PathDependencyBase::Init(); + } + void TearDown() override + { + PathDependencyBase::Destroy(); + } + }; + + TEST_F(PathDependencyTestValidation, DeferredWildcardDependencyResolution) + { + SetupTestData(); + DoTest(); + VerifyResult(); + } + + BENCHMARK_F(PathDependencyBenchmarksWrapperClass, BM_DeferredWildcardDependencyResolution)(benchmark::State& state) + { + for (auto _ : state) + { + m_benchmarks->m_stateData->SetProductDependencies(m_benchmarks->m_dependencies); + + m_benchmarks->DoTest(); + } + } + } diff --git a/Code/Tools/AssetProcessor/native/tests/assetmanager/AssetProcessorManagerTest.cpp b/Code/Tools/AssetProcessor/native/tests/assetmanager/AssetProcessorManagerTest.cpp index 58f76f8da6..0908d0fdb9 100644 --- a/Code/Tools/AssetProcessor/native/tests/assetmanager/AssetProcessorManagerTest.cpp +++ b/Code/Tools/AssetProcessor/native/tests/assetmanager/AssetProcessorManagerTest.cpp @@ -15,6 +15,10 @@ #include #include +#include +#include +#include +#include using namespace AssetProcessor; @@ -26,6 +30,7 @@ public: friend class GTEST_TEST_CLASS_NAME_(MultiplatformPathDependencyTest, AssetProcessed_Impl_MultiplatformDependencies); friend class GTEST_TEST_CLASS_NAME_(MultiplatformPathDependencyTest, AssetProcessed_Impl_MultiplatformDependencies_DeferredResolution); + friend class GTEST_TEST_CLASS_NAME_(MultiplatformPathDependencyTest, SameFilenameForAllPlatforms); friend class GTEST_TEST_CLASS_NAME_(MultiplatformPathDependencyTest, AssetProcessed_Impl_MultiplatformDependencies_SourcePath); @@ -176,8 +181,21 @@ void AssetProcessorManagerTest::SetUp() AssetProcessorTest::SetUp(); + AZ::AllocatorInstance::Create(); + AZ::AllocatorInstance::Create(); + m_data = AZStd::make_unique(); + m_data->m_serializeContext = AZStd::make_unique(); + + m_data->m_descriptor = AZ::JobManagerComponent::CreateDescriptor(); + m_data->m_descriptor->Reflect(m_data->m_serializeContext.get()); + + m_data->m_jobManagerEntity = aznew AZ::Entity{}; + m_data->m_jobManagerEntity->CreateComponent(); + m_data->m_jobManagerEntity->Init(); + m_data->m_jobManagerEntity->Activate(); + m_config.reset(new AssetProcessor::PlatformConfiguration()); m_mockApplicationManager.reset(new AssetProcessor::MockApplicationManager()); @@ -256,6 +274,10 @@ void AssetProcessorManagerTest::SetUp() void AssetProcessorManagerTest::TearDown() { + m_data->m_jobManagerEntity->Deactivate(); + delete m_data->m_jobManagerEntity; + delete m_data->m_descriptor; + m_data = nullptr; QObject::disconnect(m_idleConnection); @@ -271,6 +293,9 @@ void AssetProcessorManagerTest::TearDown() m_qApp.reset(); m_scopeDir.reset(); + AZ::AllocatorInstance::Destroy(); + AZ::AllocatorInstance::Destroy(); + AssetProcessor::AssetProcessorTest::TearDown(); } @@ -1270,7 +1295,8 @@ TEST_F(AbsolutePathProductDependencyTest, AbsolutePathProductDependency_RetryDef AZ::Data::AssetType::CreateNull()); m_assetProcessorManager->m_stateData->SetProduct(matchingProductForDependency); - m_assetProcessorManager->m_pathDependencyManager->RetryDeferredDependencies(matchingSource); + m_assetProcessorManager->m_pathDependencyManager->QueueSourceForDependencyResolution(matchingSource); + m_assetProcessorManager->m_pathDependencyManager->ProcessQueuedDependencyResolves(); // The product dependency ID shouldn't change when it goes from unresolved to resolved. AZStd::vector resolvedProductDependencies; @@ -1405,6 +1431,7 @@ bool PathDependencyTest::ProcessAsset(TestAsset& asset, const OutputAssetSet& ou // tell the APM that the asset has been processed and allow it to bubble through its event queue: m_isIdling = false; m_assetProcessorManager->AssetProcessed(capturedDetails[jobSet].m_jobEntry, processJobResponse); + m_assetProcessorManager->CheckForIdle(); jobSet++; } @@ -2646,6 +2673,44 @@ TEST_F(MultiplatformPathDependencyTest, AssetProcessed_Impl_MultiplatformDepende ASSERT_NE(SearchDependencies(dependencyContainer, asset1.m_products[0]), SearchDependencies(dependencyContainer, asset1.m_products[1])); } +TEST_F(MultiplatformPathDependencyTest, SameFilenameForAllPlatforms) +{ + TestAsset asset2("asset2"); + bool result = ProcessAsset( + asset2, { { ".output" }, { ".output" } }, { { "*1.output", AssetBuilderSDK::ProductPathDependencyType::ProductFile } }, "subfolder1/", + ".txt"); + + ASSERT_TRUE(result); + + TestAsset asset1("asset1"); + result = ProcessAsset(asset1, { { ".output" }, { ".output" } }, {}); + + ASSERT_TRUE(result); + + AssetDatabaseConnection* sharedConnection = m_assetProcessorManager->m_stateData.get(); + ASSERT_TRUE(sharedConnection); + + AzToolsFramework::AssetDatabase::ProductDependencyDatabaseEntryContainer dependencyContainer; + + sharedConnection->GetProductDependencies(dependencyContainer); + int resolvedCount = 0; + int unresolvedCount = 0; + for (const auto& dep : dependencyContainer) + { + if (dep.m_unresolvedPath.empty()) + { + resolvedCount++; + } + else + { + unresolvedCount++; + } + } + ASSERT_EQ(resolvedCount, 2); + ASSERT_EQ(unresolvedCount, 2); + VerifyDependencies(dependencyContainer, { asset1.m_products[0], asset1.m_products[1] }, { "*1.output", "*1.output" }); +} + TEST_F(MultiplatformPathDependencyTest, AssetProcessed_Impl_MultiplatformDependencies_SourcePath) { // One product will be pc, one will be console (order is non-deterministic) @@ -4185,7 +4250,7 @@ TEST_F(LockedFileTest, DeleteFile_LockedProduct_DeleteFails) auto theFile = m_data->m_absolutePath[1].toUtf8(); const char* theFileString = theFile.constData(); auto [sourcePath, productPath] = *m_data->m_productPaths.find(theFileString); - + { QFile file(theFileString); file.remove(); @@ -4257,7 +4322,7 @@ TEST_F(LockedFileTest, DeleteFile_LockedProduct_DeletesWhenReleased) EXPECT_FALSE(QFile::exists(productPath)); EXPECT_EQ(m_data->m_deletedSources.size(), 1); - + EXPECT_GT(m_deleteCounter, 1); // Make sure the AP tried more than once to delete the file m_errorAbsorber->ExpectAsserts(0); } @@ -5364,7 +5429,7 @@ AZStd::vector WildcardSourceDependencyTest::FileAddedTest(const Q void WildcardSourceDependencyTest::SetUp() { AssetProcessorManagerTest::SetUp(); - + QDir tempPath(m_tempDir.path()); // Add a non-recursive scan folder. Only files directly inside of this folder should be picked up, subfolders are ignored @@ -5454,7 +5519,7 @@ TEST_F(WildcardSourceDependencyTest, Relative_Broad) { // Expect all files except for the 2 invalid ones (e and f) AZStd::vector resolvedPaths; - + ASSERT_TRUE(Test("*.foo", resolvedPaths)); ASSERT_THAT(resolvedPaths, ::testing::UnorderedElementsAre("a.foo", "b.foo", "folder/one/c.foo", "folder/one/d.foo", "1a.foo", "1b.foo")); } @@ -5585,7 +5650,7 @@ TEST_F(WildcardSourceDependencyTest, Relative_CacheFolder) { AZStd::vector resolvedPaths; QDir tempPath(m_tempDir.path()); - + ASSERT_TRUE(Test("*cache.foo", resolvedPaths)); ASSERT_THAT(resolvedPaths, ::testing::UnorderedElementsAre()); } @@ -5635,7 +5700,7 @@ TEST_F(WildcardSourceDependencyTest, FilesRemovedAfterInitialCache) ASSERT_EQ(excludedFolders.size(), 3); } - + m_fileStateCache->SignalDeleteEvent(tempPath.absoluteFilePath("subfolder2/redirected/folder/two/ignored")); const auto& excludedFolders = excludedFolderCacheInterface->GetExcludedFolders(); diff --git a/Code/Tools/AssetProcessor/native/tests/assetmanager/AssetProcessorManagerTest.h b/Code/Tools/AssetProcessor/native/tests/assetmanager/AssetProcessorManagerTest.h index 7bbef41f44..c532b08016 100644 --- a/Code/Tools/AssetProcessor/native/tests/assetmanager/AssetProcessorManagerTest.h +++ b/Code/Tools/AssetProcessor/native/tests/assetmanager/AssetProcessorManagerTest.h @@ -23,6 +23,8 @@ #include #include +#include +#include #include #include #include "resourcecompiler/rccontroller.h" @@ -39,7 +41,7 @@ class AssetProcessorManagerTest : public AssetProcessor::AssetProcessorTest { public: - + AssetProcessorManagerTest(); virtual ~AssetProcessorManagerTest() @@ -67,16 +69,19 @@ protected: { AZStd::string m_databaseLocation; ::testing::NiceMock m_databaseLocationListener; + AZ::Entity* m_jobManagerEntity{}; + AZ::ComponentDescriptor* m_descriptor{}; + AZStd::unique_ptr m_serializeContext; }; AZStd::unique_ptr m_data; - + private: int m_argc; char** m_argv; AZStd::unique_ptr m_scopeDir; - AZStd::unique_ptr m_qApp; + AZStd::unique_ptr m_qApp; }; struct AbsolutePathProductDependencyTest diff --git a/Code/Tools/AssetProcessor/native/utilities/PlatformConfiguration.cpp b/Code/Tools/AssetProcessor/native/utilities/PlatformConfiguration.cpp index 7543e9ec8b..c62d822171 100644 --- a/Code/Tools/AssetProcessor/native/utilities/PlatformConfiguration.cpp +++ b/Code/Tools/AssetProcessor/native/utilities/PlatformConfiguration.cpp @@ -51,7 +51,7 @@ namespace AssetProcessor case AZ::SettingsRegistryInterface::VisitAction::Begin: { // Only continue traversal if the path is exactly the AssetProcessorSettingsKey (which indicates the start of traversal) - // or if a "Platform *" object and it's children are being traversed + // or if a "Platform *" object and it's children are being traversed if (jsonPath == AssetProcessorSettingsKey) { return AZ::SettingsRegistryInterface::VisitResponse::Continue; @@ -631,7 +631,7 @@ namespace AssetProcessor for (const AssetBuilderSDK::PlatformInfo& platform : m_enabledPlatforms) { AZStd::string_view currentRCParams = assetRecognizer.m_defaultParams; - // The "/Amazon/AssetProcessor/Settings/RC */" entry will be queried + // The "/Amazon/AssetProcessor/Settings/RC */" entry will be queried AZ::IO::Path overrideParamsKey = AZ::IO::Path(AZ::IO::PosixPathSeparator); overrideParamsKey /= path; overrideParamsKey /= platform.m_identifier; @@ -644,7 +644,7 @@ namespace AssetProcessor } else { - // otherwise check for tags associated with the platform + // otherwise check for tags associated with the platform for (const AZStd::string& tag : platform.m_tags) { overrideParamsKey.ReplaceFilename(AZ::IO::PathView(tag)); @@ -1416,6 +1416,8 @@ namespace AssetProcessor return QString(); } + auto* fileStateInterface = AZ::Interface::Get(); + for (int pathIdx = 0; pathIdx < m_scanFolders.size(); ++pathIdx) { AssetProcessor::ScanFolderInfo scanFolderInfo = m_scanFolders[pathIdx]; @@ -1430,7 +1432,7 @@ namespace AssetProcessor QDir rooted(scanFolderInfo.ScanPath()); QString absolutePath = rooted.absoluteFilePath(tempRelativeName); AssetProcessor::FileStateInfo fileStateInfo; - auto* fileStateInterface = AZ::Interface::Get(); + if (fileStateInterface) { if (fileStateInterface->GetFileInfo(absolutePath, &fileStateInfo)) @@ -1499,7 +1501,7 @@ namespace AssetProcessor QRegExp nameMatch{ posixRelativeName, Qt::CaseInsensitive, QRegExp::Wildcard }; AZStd::stack dirs; dirs.push(sourceFolderDir.absolutePath()); - + while (!dirs.empty()) { QString absolutePath = dirs.top(); @@ -1528,7 +1530,7 @@ namespace AssetProcessor continue; } } - + QString pathMatch{ sourceFolderDir.relativeFilePath(dirIterator.filePath()) }; if (nameMatch.exactMatch(pathMatch)) { diff --git a/Code/Tools/AssetProcessor/native/utilities/assetUtils.cpp b/Code/Tools/AssetProcessor/native/utilities/assetUtils.cpp index 6bcd0dec01..a6cbfd8077 100644 --- a/Code/Tools/AssetProcessor/native/utilities/assetUtils.cpp +++ b/Code/Tools/AssetProcessor/native/utilities/assetUtils.cpp @@ -131,7 +131,7 @@ namespace AssetUtilsInternal } } } while (!timer.hasExpired(waitTimeInSeconds * 1000)); //We will keep retrying until the timer has expired the inputted timeout - + // once we're done, regardless of success or failure, we 'unlock' those files for further process. // if we failed, also re-trigger them to rebuild (the bool param at the end of the ebus call) QString normalized = AssetUtilities::NormalizeFilePath(outputFile); @@ -870,23 +870,27 @@ namespace AssetUtilities return true; } - QString StripAssetPlatform(AZStd::string_view relativeProductPath) + AZStd::string_view StripAssetPlatformNoCopy(AZStd::string_view relativeProductPath) { // Skip over the assetPlatform path segment if it is matches one of the platform defaults // Otherwise return the path unchanged - AZStd::string_view strippedProductPath{ relativeProductPath }; - if (AZStd::optional pathSegment = AZ::StringFunc::TokenizeNext(strippedProductPath, AZ_CORRECT_AND_WRONG_FILESYSTEM_SEPARATOR); - pathSegment.has_value()) + + AZStd::string_view originalPath = relativeProductPath; + AZStd::optional firstPathSegment = AZ::StringFunc::TokenizeNext(relativeProductPath, AZ_CORRECT_AND_WRONG_FILESYSTEM_SEPARATOR); + + if (firstPathSegment && AzFramework::PlatformHelper::GetPlatformIdFromName(*firstPathSegment) != AzFramework::PlatformId::Invalid) { - AZ::IO::FixedMaxPathString assetPlatformSegmentLower{ *pathSegment }; - AZStd::to_lower(assetPlatformSegmentLower.begin(), assetPlatformSegmentLower.end()); - if (AzFramework::PlatformHelper::GetPlatformIdFromName(assetPlatformSegmentLower) != AzFramework::PlatformId::Invalid) - { - return QString::fromUtf8(strippedProductPath.data(), aznumeric_cast(strippedProductPath.size())); - } + return relativeProductPath; } - return QString::fromUtf8(relativeProductPath.data(), aznumeric_cast(relativeProductPath.size())); + return originalPath; + } + + QString StripAssetPlatform(AZStd::string_view relativeProductPath) + { + AZStd::string_view result = StripAssetPlatformNoCopy(relativeProductPath); + + return QString::fromUtf8(result.data(), aznumeric_cast(result.size())); } QString NormalizeFilePath(const QString& filePath) diff --git a/Code/Tools/AssetProcessor/native/utilities/assetUtils.h b/Code/Tools/AssetProcessor/native/utilities/assetUtils.h index 46ec5d6145..18cf5422de 100644 --- a/Code/Tools/AssetProcessor/native/utilities/assetUtils.h +++ b/Code/Tools/AssetProcessor/native/utilities/assetUtils.h @@ -145,7 +145,7 @@ namespace AssetUtilities //! Strips the first "asset platform" from the first path segment of a relative product path //! This is meant for removing the asset platform for paths such as "pc/MyAssetFolder/MyAsset.asset" //! Therefore the result here becomes "MyAssetFolder/MyAsset" - //! + //! //! Similarly invoking this function on relative path that begins with the "server" platform //! "server/AssetFolder/Server.asset2" -> "AssetFolder/Server.asset2" //! This function does not strip an asset platform from anywhere, but the first path segment @@ -153,6 +153,10 @@ namespace AssetUtilities //! would return a copy of the relative path QString StripAssetPlatform(AZStd::string_view relativeProductPath); + //! Same as StripAssetPlatform, but does not perform any string copies + //! The return result is only valid for as long as the original input is valid + AZStd::string_view StripAssetPlatformNoCopy(AZStd::string_view relativeProductPath); + //! Converts all slashes to forward slashes, removes double slashes, //! replaces all indirections such as '.' or '..' as appropriate. //! On windows, the drive letter (if present) is converted to uppercase.