birkeh/o3de
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Asset Processor: Zero analysis dependency fingerprint verification (#7292)

Browse Source 
* Move modtime scanning tests out of APM tests file and into its own file.

Changes were kept to a minimum to get things compiling, this is just a move of code

Signed-off-by: amzn-mike <80125227+amzn-mike@users.noreply.github.com>

* Fix rebase compile errors

Signed-off-by: amzn-mike <80125227+amzn-mike@users.noreply.github.com>

* Verify dependency fingerprints during Zero Analysis.

This fixes an issue where dependencies that weren't finished processing when AP shuts down would not resume when AP is started back up due to Zero Analysis ignoring dependencies when determining files to skip.
Added unit test for verification

Signed-off-by: amzn-mike <80125227+amzn-mike@users.noreply.github.com>

* Fix compile error, make 17 a constexpr

Signed-off-by: amzn-mike <80125227+amzn-mike@users.noreply.github.com>

* Fix compile error

Signed-off-by: amzn-mike <80125227+amzn-mike@users.noreply.github.com>
This commit is contained in:
amzn-mike
2022-02-03 08:50:35 -06:00
committed by GitHub
parent c3e0830586
commit 9168fe07f1
3 changed files with 112 additions and 23 deletions
Download Patch File Download Diff File Expand all files Collapse all files
Code/Tools/AssetProcessor/native/AssetManager/assetProcessorManager.cpp
+38 -23
View File
@@ -2942,11 +2942,20 @@ namespace AssetProcessor
// File is a source file that has been processed before
AZStd::string fingerprintFromDatabase = sourceFileItr->m_analysisFingerprint.toUtf8().data();
AZStd::string_view builderEntries(fingerprintFromDatabase.begin() + s_lengthOfUuid + 1, fingerprintFromDatabase.end());
AZStd::string_view dependencyFingerprint(fingerprintFromDatabase.begin(), fingerprintFromDatabase.begin() + s_lengthOfUuid);
int numBuildersEmittingSourceDependencies = 0;
if (!fingerprintFromDatabase.empty() && AreBuildersUnchanged(builderEntries, numBuildersEmittingSourceDependencies))
{
// Builder(s) have not changed since last time
AZStd::string currentFingerprint = ComputeRecursiveDependenciesFingerprint(
fileInfo.m_filePath.toUtf8().constData(), sourceFileItr->m_sourceDatabaseName.toUtf8().constData());
if(dependencyFingerprint != currentFingerprint)
{
// Dependencies have changed
return false;
}
// Success - we can skip this file, nothing has changed!
// Remove it from the list of to-be-processed files, otherwise the AP will assume the file was deleted
@@ -4311,6 +4320,32 @@ namespace AssetProcessor
}
}
AZStd::string AssetProcessorManager::ComputeRecursiveDependenciesFingerprint(const AZStd::string& fileAbsolutePath, const AZStd::string& fileDatabaseName)
{
AZStd::string concatenatedFingerprints;
// QSet is not ordered.
SourceFilesForFingerprintingContainer knownDependenciesAbsolutePaths;
// this automatically adds the input file to the list:
QueryAbsolutePathDependenciesRecursive(QString::fromUtf8(fileDatabaseName.c_str()), knownDependenciesAbsolutePaths,
AzToolsFramework::AssetDatabase::SourceFileDependencyEntry::DEP_Any, false);
AddMetadataFilesForFingerprinting(QString::fromUtf8(fileAbsolutePath.c_str()), knownDependenciesAbsolutePaths);
// reserve 17 chars for each since its a 64 bit hex number, and then one more for the dash inbetween each.
constexpr int bytesPerFingerprint = (sizeof(AZ::u64) * 2) + 1; // 2 HEX characters per byte +1 for the `-` we will add between each fingerprint
concatenatedFingerprints.reserve((knownDependenciesAbsolutePaths.size() * bytesPerFingerprint));
for (const auto& element : knownDependenciesAbsolutePaths)
{
// if its a placeholder then don't bother hitting the disk to find it.
concatenatedFingerprints.append(AssetUtilities::GetFileFingerprint(element.first, element.second));
concatenatedFingerprints.append("-");
}
// to keep this from growing out of hand, we don't use the full string, we use a hash of it:
return AZ::Uuid::CreateName(concatenatedFingerprints.c_str()).ToString<AZStd::string>();
}
void AssetProcessorManager::FinishAnalysis(AZStd::string fileToCheck)
{
using namespace AzToolsFramework::AssetDatabase;
@@ -4378,29 +4413,9 @@ namespace AssetProcessor
if (found)
{
// construct the analysis fingerprint
// the format for this data is "modtimefingerprint:builder0:builder1:builder2:...:buildern"
source.m_analysisFingerprint.clear();
// compute mod times:
// get the appropriate modtimes:
AZStd::string modTimeArray;
// the format for this data is "hashfingerprint:builder0:builder1:builder2:...:buildern"
source.m_analysisFingerprint = ComputeRecursiveDependenciesFingerprint(fileToCheck, analysisTracker->m_databaseSourceName);
// QSet is not ordered.
SourceFilesForFingerprintingContainer knownDependenciesAbsolutePaths;
// this automatically adds the input file to the list:
QueryAbsolutePathDependenciesRecursive(QString::fromUtf8(analysisTracker->m_databaseSourceName.c_str()), knownDependenciesAbsolutePaths, SourceFileDependencyEntry::DEP_Any, false);
AddMetadataFilesForFingerprinting(QString::fromUtf8(fileToCheck.c_str()), knownDependenciesAbsolutePaths);
// reserve 17 chars for each since its a 64 bit hex number, and then one more for the dash inbetween each.
modTimeArray.reserve((knownDependenciesAbsolutePaths.size() * 17));
for (const auto& element : knownDependenciesAbsolutePaths)
{
// if its a placeholder then don't bother hitting the disk to find it.
modTimeArray.append(AssetUtilities::GetFileFingerprint(element.first, element.second));
modTimeArray.append("-");
}
// to keep this from growing out of hand, we don't use the full string, we use a hash of it:
source.m_analysisFingerprint = AZ::Uuid::CreateName(modTimeArray.c_str()).ToString<AZStd::string>();
for (const AZ::Uuid& builderID : analysisTracker->m_buildersInvolved)
{
source.m_analysisFingerprint.append(":");
@@ -4423,7 +4438,7 @@ namespace AssetProcessor
const ScanFolderInfo* scanFolder = m_platformConfig->GetScanFolderForFile(fileToCheck.c_str());
scanFolderPk = aznumeric_cast<int>(scanFolder->ScanFolderID());
m_platformConfig->ConvertToRelativePath(fileToCheck.c_str(), scanFolder, databaseSourceName);
PlatformConfiguration::ConvertToRelativePath(fileToCheck.c_str(), scanFolder, databaseSourceName);
}
// Record the modtime for the file so we know we processed it