birkeh/o3de
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

AP: product dependency optimization (#6619)

Browse Source 
* Initial pass at optimizing product path dependency resolution

Signed-off-by: amzn-mike <80125227+amzn-mike@users.noreply.github.com>

* Add version of StripAssetPlatform that doesn't allocate or copy strings.  Re-add missing test and fix up compile errors.  Add benchmark test

Signed-off-by: amzn-mike <80125227+amzn-mike@users.noreply.github.com>

* Change UpdateProductDependencies to directly call s_InsertProductDependencyQuery.BindAndStep

Signed-off-by: amzn-mike <80125227+amzn-mike@users.noreply.github.com>

* Add test for same filename on multiple platforms

Signed-off-by: amzn-mike <80125227+amzn-mike@users.noreply.github.com>

* Rework search logic to keep track of the source of a search path (source vs product) and keep track of which search matches which dependency to avoid doing another search through every product later on

Signed-off-by: amzn-mike <80125227+amzn-mike@users.noreply.github.com>

* Clean up code, expand test

Signed-off-by: amzn-mike <80125227+amzn-mike@users.noreply.github.com>

* Fix paths not being lowercased by SanitizeForDatabase.  Fix UpdateProductDependencies not updating existing dependencies

Signed-off-by: amzn-mike <80125227+amzn-mike@users.noreply.github.com>

* Add test for duplicate dependency matches.  Fix saving duplicates

Signed-off-by: amzn-mike <80125227+amzn-mike@users.noreply.github.com>

* Clean up code

Signed-off-by: amzn-mike <80125227+amzn-mike@users.noreply.github.com>

* Separate test into test and benchmark versions

Signed-off-by: amzn-mike <80125227+amzn-mike@users.noreply.github.com>

* Cleanup include

Signed-off-by: amzn-mike <80125227+amzn-mike@users.noreply.github.com>

* Fix includes, switch hardcoded job manager setup to use JobManagerComponent instead

Signed-off-by: amzn-mike <80125227+amzn-mike@users.noreply.github.com>

* Replaced wildcard_match with PathView::Match.  Changed StripAssetPlatformNoCopy to use TokenizeNext.  Removed Environment Create/Destroy calls.  Made ScopedAllocatorFixture a base class of ScopedAllocatorSetupFixture

Signed-off-by: amzn-mike <80125227+amzn-mike@users.noreply.github.com>

* Add AZ Environment create/destroy on AP test environment

Signed-off-by: amzn-mike <80125227+amzn-mike@users.noreply.github.com>

* Add missing asserts on database functions

Signed-off-by: amzn-mike <80125227+amzn-mike@users.noreply.github.com>

* Fix incorrect usage of StripAssetPlatformNoCopy

Signed-off-by: amzn-mike <80125227+amzn-mike@users.noreply.github.com>

* Fix source/product dependency type being ignored.  Removed need for unordered_set for list of resolved dependencies.  Updated unit tests

Signed-off-by: amzn-mike <80125227+amzn-mike@users.noreply.github.com>

* Better variable names

Signed-off-by: amzn-mike <80125227+amzn-mike@users.noreply.github.com>

* Remove testing code

Signed-off-by: amzn-mike <80125227+amzn-mike@users.noreply.github.com>

* Fix missing includes and namespaces

Signed-off-by: amzn-mike <80125227+amzn-mike@users.noreply.github.com>
This commit is contained in:
amzn-mike
2022-01-13 08:56:24 -06:00
committed by GitHub
parent 6d1a2382e8
commit fed1278fe6
14 changed files with 742 additions and 237 deletions
Download Patch File Download Diff File Expand all files Collapse all files
Code/Tools/AssetProcessor/native/AssetDatabase/AssetDatabase.cpp
+53 -31
View File
@@ -172,7 +172,7 @@ namespace AssetProcessor
static const char* CREATEINDEX_BUILDERGUID_SOURCE_SOURCEDEPENDENCY_STATEMENT =
"CREATE INDEX IF NOT EXISTS BuilderGuid_Source_SourceDependency ON SourceDependency (BuilderGuid, Source);";
static const char* CREATEINDEX_TYPEOFDEPENDENCY_SOURCEDEPENDENCY = "AssetProcessor::CreateIndexTypeOfDependency_SourceDependency";
static const char* CREATEINDEX_TYPEOFDEPENDENCY_SOURCEDEPENDENCY_STATEMENT =
static const char* CREATEINDEX_TYPEOFDEPENDENCY_SOURCEDEPENDENCY_STATEMENT =
"CREATE INDEX IF NOT EXISTS TypeOfDependency_SourceDependency ON SourceDependency (TypeOfDependency);";
static const char* CREATEINDEX_SCANFOLDERS_SOURCES_SCANFOLDER = "AssetProcesser::CreateIndexScanFoldersSourcesScanFolder";
@@ -611,7 +611,7 @@ namespace AssetProcessor
SqlParam<const char*>(":missingDependencyString"),
SqlParam<const char*>(":lastScanTime"),
SqlParam<AZ::u64>(":scanTimeSecondsSinceEpoch"));
static const auto s_DeleteMissingProductDependencyByProductIdQuery = MakeSqlQuery(
DELETE_MISSING_PRODUCT_DEPENDENCY_BY_PRODUCTID,
@@ -643,7 +643,7 @@ namespace AssetProcessor
SqlParam<const char*>(":analysisFingerprint"));
static const char* INSERT_COLUMN_ANALYSISFINGERPRINT = "AssetProcessor::AddColumnAnalysisFingerprint";
static const char* INSERT_COLUMN_ANALYSISFINGERPRINT_STATEMENT =
static const char* INSERT_COLUMN_ANALYSISFINGERPRINT_STATEMENT =
"ALTER TABLE Sources "
"ADD AnalysisFingerprint TEXT NOT NULL collate nocase default('');";
@@ -653,7 +653,7 @@ namespace AssetProcessor
"ADD TypeOfDependency INTEGER NOT NULL DEFAULT 0;";
static const char* INSERT_COLUMN_FILE_MODTIME = "AssetProcessor::AddFiles_ModTime";
static const char* INSERT_COLUMN_FILE_MODTIME_STATEMENT =
static const char* INSERT_COLUMN_FILE_MODTIME_STATEMENT =
"ALTER TABLE Files "
"ADD ModTime INTEGER NOT NULL DEFAULT 0;";
@@ -673,7 +673,7 @@ namespace AssetProcessor
"ADD UnresolvedDependencyType INTEGER NOT NULL DEFAULT 0;";
static const char* INSERT_COLUMN_PRODUCTDEPENDENCY_PLATFORM = "AssetProcessor::AddProductDependency_Platform";
static const char* INSERT_COLUMN_PRODUCTDEPENDENCY_PLATFORM_STATEMENT =
static const char* INSERT_COLUMN_PRODUCTDEPENDENCY_PLATFORM_STATEMENT =
"ALTER TABLE ProductDependencies "
"ADD Platform TEXT NOT NULL collate nocase default('');";
@@ -721,7 +721,7 @@ namespace AssetProcessor
SqlParam<AZ::s64>(":isfolder"),
SqlParam<AZ::u64>(":modtime"),
SqlParam<AZ::u64>(":hash"));
static const char* UPDATE_FILE = "AssetProcessor::UpdateFile";
static const char* UPDATE_FILE_STATEMENT =
"UPDATE Files SET "
@@ -961,7 +961,7 @@ namespace AssetProcessor
AZ_TracePrintf(AssetProcessor::ConsoleChannel, "Upgraded Asset Database to version %i (AddedTypeOfDependencyIndex)\n", foundVersion)
}
}
if (foundVersion == AssetDatabase::DatabaseVersion::AddedTypeOfDependencyIndex)
{
if (m_databaseConnection->ExecuteOneOffStatement(INSERT_COLUMN_PRODUCTDEPENDENCY_PLATFORM))
@@ -1178,7 +1178,7 @@ namespace AssetProcessor
AddStatement(m_databaseConnection, s_InsertJobQuery);
AddStatement(m_databaseConnection, s_UpdateJobQuery);
AddStatement(m_databaseConnection, s_DeleteJobQuery);
// ---------------------------------------------------------------------------------------------
// Builder Info Table
// ---------------------------------------------------------------------------------------------
@@ -1207,7 +1207,7 @@ namespace AssetProcessor
m_databaseConnection->AddStatement(CREATE_SOURCE_DEPENDENCY_TABLE, CREATE_SOURCE_DEPENDENCY_TABLE_STATEMENT);
m_databaseConnection->AddStatement(INSERT_COLUMN_SOURCEDEPENDENCY_TYPEOFDEPENDENCY, INSERT_COLUMN_SOURCEDEPENDENCY_TYPEOFDEPENDENCY_STATEMENT);
m_databaseConnection->AddStatement(INSERT_COLUMNS_SOURCEDEPENDENCY_FROM_ASSETID, INSERT_COLUMNS_SOURCEDEPENDENCY_FROM_ASSETID_STATEMENT);
m_createStatements.push_back(CREATE_SOURCE_DEPENDENCY_TABLE);
AddStatement(m_databaseConnection, s_InsertSourceDependencyQuery);
@@ -1242,7 +1242,7 @@ namespace AssetProcessor
AddStatement(m_databaseConnection, s_InsertProductDependencyQuery);
AddStatement(m_databaseConnection, s_UpdateProductDependencyQuery);
AddStatement(m_databaseConnection, s_DeleteProductDependencyByProductIdQuery);
// ---------------------------------------------------------------------------------------------
// Missing Product Dependency table
// ---------------------------------------------------------------------------------------------
@@ -1253,7 +1253,7 @@ namespace AssetProcessor
AddStatement(m_databaseConnection, s_InsertMissingProductDependencyQuery);
AddStatement(m_databaseConnection, s_UpdateMissingProductDependencyQuery);
AddStatement(m_databaseConnection, s_DeleteMissingProductDependencyByProductIdQuery);
// ---------------------------------------------------------------------------------------------
// Files table
// ---------------------------------------------------------------------------------------------
@@ -1344,7 +1344,7 @@ namespace AssetProcessor
bool AssetDatabaseConnection::GetScanFolderByScanFolderID(AZ::s64 scanfolderID, ScanFolderDatabaseEntry& entry)
{
bool found = false;
QueryScanFolderByScanFolderID( scanfolderID,
QueryScanFolderByScanFolderID( scanfolderID,
[&](ScanFolderDatabaseEntry& scanFolderEntry)
{
entry = scanFolderEntry;
@@ -1357,7 +1357,7 @@ namespace AssetProcessor
bool AssetDatabaseConnection::GetScanFolderBySourceID(AZ::s64 sourceID, ScanFolderDatabaseEntry& entry)
{
bool found = false;
QueryScanFolderBySourceID( sourceID,
QueryScanFolderBySourceID( sourceID,
[&](ScanFolderDatabaseEntry& scanFolderEntry)
{
entry = scanFolderEntry;
@@ -1370,7 +1370,7 @@ namespace AssetProcessor
bool AssetDatabaseConnection::GetScanFolderByJobID(AZ::s64 jobID, ScanFolderDatabaseEntry& entry)
{
bool found = false;
QueryScanFolderByJobID( jobID,
QueryScanFolderByJobID( jobID,
[&](ScanFolderDatabaseEntry& scanFolderEntry)
{
entry = scanFolderEntry;
@@ -1383,7 +1383,7 @@ namespace AssetProcessor
bool AssetDatabaseConnection::GetScanFolderByProductID(AZ::s64 productID, ScanFolderDatabaseEntry& entry)
{
bool found = false;
QueryScanFolderByProductID( productID,
QueryScanFolderByProductID( productID,
[&](ScanFolderDatabaseEntry& scanFolderEntry)
{
entry = scanFolderEntry;
@@ -2105,7 +2105,7 @@ namespace AssetProcessor
bool AssetDatabaseConnection::GetProductsLikeProductName(QString likeProductName, LikeType likeType, ProductDatabaseEntryContainer& container, AZ::Uuid builderGuid, QString jobKey, QString platform, JobStatus status)
{
bool found = false;
if (likeProductName.isEmpty())
{
return false;
@@ -2198,7 +2198,7 @@ namespace AssetProcessor
bool AssetDatabaseConnection::GetProductByJobIDSubId(AZ::s64 jobID, AZ::u32 subID, AzToolsFramework::AssetDatabase::ProductDatabaseEntry& result)
{
bool found = false;
QueryProductByJobIDSubID(jobID, subID,
QueryProductByJobIDSubID(jobID, subID,
[&](ProductDatabaseEntry& resultFromDB)
{
found = true;
@@ -2312,13 +2312,19 @@ namespace AssetProcessor
{
return false;
}
bool succeeded = true;
ScopedTransaction transaction(m_databaseConnection);
for (auto& entry : container)
{
succeeded &= SetProduct(entry);
if(!SetProduct(entry))
{
return false;
}
}
return succeeded;
transaction.Commit();
return true;
}
//! Clear the products for a given source. This removes the entry entirely, not just sets it to empty.
@@ -2407,7 +2413,7 @@ namespace AssetProcessor
if(!platform.isEmpty())
{
AZStd::string platformStr = platform.toUtf8().constData();
if (!s_DeleteProductsBySourceidPlatformQuery.BindAndStep(*m_databaseConnection, sourceID, platformStr.c_str()))
{
return false;
@@ -2521,7 +2527,7 @@ namespace AssetProcessor
{
succeeded = succeeded && RemoveSourceFileDependency(entry);
}
if (succeeded)
{
transaction.Commit();
@@ -2577,7 +2583,7 @@ namespace AssetProcessor
}
bool AssetDatabaseConnection::GetDependsOnSourceBySource(
const char* source,
const char* source,
AzToolsFramework::AssetDatabase::SourceFileDependencyEntry::TypeOfDependency typeOfDependency,
AzToolsFramework::AssetDatabase::SourceFileDependencyEntryContainer& container)
{
@@ -2596,7 +2602,7 @@ namespace AssetProcessor
bool AssetDatabaseConnection::GetSourceFileDependencyBySourceDependencyId(AZ::s64 sourceDependencyId, SourceFileDependencyEntry& sourceDependencyEntry)
{
bool found = false;
QuerySourceDependencyBySourceDependencyId(sourceDependencyId,
QuerySourceDependencyBySourceDependencyId(sourceDependencyId,
[&](SourceFileDependencyEntry& entry)
{
found = true;
@@ -2624,7 +2630,7 @@ namespace AssetProcessor
return false;
}
if (creatingNew)
{
AZ::s64 rowID = m_databaseConnection->GetLastRowID();
@@ -2958,9 +2964,25 @@ namespace AssetProcessor
for(auto& entry : container)
{
if(!SetProductDependency(entry))
if(entry.m_productDependencyID == InvalidEntryId)
{
return false;
if (!s_InsertProductDependencyQuery.BindAndStep(
*m_databaseConnection, entry.m_productPK, entry.m_dependencySourceGuid, entry.m_dependencySubID,
entry.m_dependencyFlags.to_ullong(), entry.m_platform.c_str(), entry.m_unresolvedPath.c_str(),
entry.m_dependencyType, entry.m_fromAssetId))
{
return false;
}
}
else
{
if(!s_UpdateProductDependencyQuery.BindAndStep(
*m_databaseConnection, entry.m_productPK, entry.m_dependencySourceGuid, entry.m_dependencySubID,
entry.m_dependencyFlags.to_ullong(), entry.m_platform.c_str(), entry.m_unresolvedPath.c_str(),
entry.m_productDependencyID, entry.m_dependencyType, entry.m_fromAssetId))
{
return false;
}
}
}
@@ -2989,7 +3011,7 @@ namespace AssetProcessor
}
// now insert the new ones since we know there's no collisions:
for (auto& entry : container)
{
@@ -3109,7 +3131,7 @@ namespace AssetProcessor
}
Statement* statement = autoFinal.Get();
if (statement->Step() == Statement::SqlError)
{
AZ_Warning(LOG_NAME, false, "Failed to write the new source into the database. %s", entry.m_fileName.c_str());
@@ -3126,7 +3148,7 @@ namespace AssetProcessor
return UpdateFile(entry, entryAlreadyExists);
}
bool AssetDatabaseConnection::UpdateFile(FileDatabaseEntry& entry, bool& entryAlreadyExists)
bool AssetDatabaseConnection::UpdateFile(FileDatabaseEntry& entry, bool& entryAlreadyExists)
{
entryAlreadyExists = false;