birkeh/o3de
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Integrating up through commit 90f050496

Browse Source
This commit is contained in:
alexpete
2021-04-07 14:03:29 -07:00
parent 8f2ed080a9
commit c2cbd430fe
2694 changed files with 285622 additions and 176874 deletions
Download Patch File Download Diff File Expand all files Collapse all files
Code/Tools/AssetProcessor/AssetBuilderSDK/AssetBuilderSDK/AssetBuilderSDK.cpp
+2 -1
View File
@@ -1302,7 +1302,8 @@ namespace AssetBuilderSDK
->Field("Patterns", &AssetBuilderDesc::m_patterns)
->Field("BusId", &AssetBuilderDesc::m_busId)
->Field("Version", &AssetBuilderDesc::m_version)
->Field("AnalysisFingerprint", &AssetBuilderDesc::m_analysisFingerprint);
->Field("AnalysisFingerprint", &AssetBuilderDesc::m_analysisFingerprint)
->Field("ProductsToKeepOnFailure", &AssetBuilderDesc::m_productsToKeepOnFailure);
serializeContext->RegisterGenericType<AZStd::vector<AssetBuilderDesc>>();
}
Code/Tools/AssetProcessor/AssetBuilderSDK/AssetBuilderSDK/AssetBuilderSDK.h
+5 -1
View File
@@ -272,7 +272,7 @@ namespace AssetBuilderSDK
* If your analysis fingerprint DOES change, then all source files will be sent to your CreateJobs function regardless of modtime changes.
* This does not necessarily mean that the jobs will need doing, just that CreateJobs will be called.
* For best results, make sure your analysis fingerprint only changes when its likely that you need to re-analyze source files for changes, which
* may result in job fingerprints to be diffent (for example, if you have changed your logic inside your builder).
* may result in job fingerprints to be different (for example, if you have changed your logic inside your builder).
**/
AZStd::string m_analysisFingerprint;
@@ -282,6 +282,10 @@ namespace AssetBuilderSDK
AZStd::unordered_map<AZStd::string, AZ::u8> m_flagsByJobKey;
// If BF_DeleteLastKnownGoodProductOnFailure is raised, ALL specified product keys will be deleted on failure
// use this set to keep specific products in the job, if necessary
AZStd::unordered_map<AZStd::string, AZStd::unordered_set<AZ::u32>> m_productsToKeepOnFailure;
void AddFlags(AZ::u8 flag, const AZStd::string& jobKey);
bool HasFlag(AZ::u8 flag, const AZStd::string& jobKey) const;
Code/Tools/AssetProcessor/AssetBuilderSDK/AssetBuilderSDK/SerializationDependencies.cpp
+2 -2
View File
@@ -151,7 +151,7 @@ namespace AssetBuilderSDK
bool GatherProductDependencies(
AZ::SerializeContext& serializeContext,
void* obj,
const void* obj,
AZ::TypeId typeId,
AZStd::vector<ProductDependency>& productDependencies,
ProductPathDependencySet& productPathDependencySet,
@@ -176,7 +176,7 @@ namespace AssetBuilderSDK
return enumerateResult;
}
bool OutputObject(void* obj, AZ::TypeId typeId, AZStd::string_view outputPath, AZ::Data::AssetType assetType, AZ::u32 subId, JobProduct& jobProduct, AZ::SerializeContext* serializeContext, const DependencyHandler& handler)
bool OutputObject(const void* obj, AZ::TypeId typeId, AZStd::string_view outputPath, AZ::Data::AssetType assetType, AZ::u32 subId, JobProduct& jobProduct, AZ::SerializeContext* serializeContext, const DependencyHandler& handler)
{
if (!serializeContext)
{
Code/Tools/AssetProcessor/AssetBuilderSDK/AssetBuilderSDK/SerializationDependencies.h
+6 -6
View File
@@ -57,7 +57,7 @@ namespace AssetBuilderSDK
bool GatherProductDependencies(
AZ::SerializeContext& serializeContext,
void* obj,
const void* obj,
AZ::TypeId typeId,
AZStd::vector<ProductDependency>& productDependencies,
ProductPathDependencySet& productPathDependencySet,
@@ -66,7 +66,7 @@ namespace AssetBuilderSDK
template<class T>
bool GatherProductDependencies(
AZ::SerializeContext& serializeContext,
AZ::Data::Asset<T>* obj,
const AZ::Data::Asset<T>* obj,
AZ::TypeId typeId,
AZStd::vector<ProductDependency>& productDependencies,
ProductPathDependencySet& productPathDependencySet,
@@ -79,7 +79,7 @@ namespace AssetBuilderSDK
template<class T>
bool GatherProductDependencies(
AZ::SerializeContext& serializeContext,
T* obj,
const T* obj,
AZStd::vector<ProductDependency>& productDependencies,
ProductPathDependencySet& productPathDependencySet,
const DependencyHandler& handler = &UpdateDependenciesFromClassData)
@@ -87,18 +87,18 @@ namespace AssetBuilderSDK
return GatherProductDependencies(serializeContext, obj, azrtti_typeid<T>(), productDependencies, productPathDependencySet, handler);
}
bool OutputObject(void* obj, AZ::TypeId typeId, AZStd::string_view outputPath, AZ::Data::AssetType assetType, AZ::u32 subId, JobProduct& jobProduct, AZ::SerializeContext* serializeContext = nullptr,
bool OutputObject(const void* obj, AZ::TypeId typeId, AZStd::string_view outputPath, AZ::Data::AssetType assetType, AZ::u32 subId, JobProduct& jobProduct, AZ::SerializeContext* serializeContext = nullptr,
const DependencyHandler& handler = &UpdateDependenciesFromClassData);
template<class T>
bool OutputObject(T* obj, AZStd::string_view outputPath, AZ::Data::AssetType assetType, AZ::u32 subId, JobProduct& jobProduct, AZ::SerializeContext* serializeContext = nullptr,
bool OutputObject(const T* obj, AZStd::string_view outputPath, AZ::Data::AssetType assetType, AZ::u32 subId, JobProduct& jobProduct, AZ::SerializeContext* serializeContext = nullptr,
const DependencyHandler& handler = &UpdateDependenciesFromClassData)
{
return OutputObject(obj, azrtti_typeid<T>(), outputPath, assetType, subId, jobProduct, serializeContext, handler);
}
template<class T>
bool OutputObject(AZ::Data::Asset<T>* obj, AZStd::string_view outputPath, AZ::Data::AssetType assetType, AZ::u32 subId, JobProduct& jobProduct, AZ::SerializeContext* serializeContext = nullptr,
bool OutputObject(const AZ::Data::Asset<T>* obj, AZStd::string_view outputPath, AZ::Data::AssetType assetType, AZ::u32 subId, JobProduct& jobProduct, AZ::SerializeContext* serializeContext = nullptr,
const DependencyHandler& handler = &UpdateDependenciesFromClassData)
{
AZ_Error("AssetBuilderSDK", false, "Can't output dependencies for AZ::Data::Asset<T>* - Use T* or another underlying type");
Code/Tools/AssetProcessor/native/AssetManager/PathDependencyManager.cpp
+2 -2
View File
@@ -464,8 +464,8 @@ namespace AssetProcessor
if (isExactDependency)
{
// Search for products in the cache platform folder
// Example: If a path dependency is "test1.asset" in SamplesProject on PC, this would search
// "SamplesProject/Cache/pc/test1.asset"
// Example: If a path dependency is "test1.asset" in AutomatedTesting on PC, this would search
// "AutomatedTesting/Cache/pc/test1.asset"
m_stateData->GetProductsByProductName(productNameWithPlatform, productInfoContainer);
}
Code/Tools/AssetProcessor/native/AssetManager/assetProcessorManager.cpp
+25 -6
View File
@@ -725,6 +725,14 @@ namespace AssetProcessor
{
AzToolsFramework::AssetDatabase::ProductDatabaseEntryContainer products;
m_stateData->GetProductsByJobID(job.m_jobID, products);
auto keepProductsIter = description.m_productsToKeepOnFailure.find(jobEntry.m_jobKey.toUtf8().constData());
if (keepProductsIter != description.m_productsToKeepOnFailure.end())
{
// keep some products
AZStd::erase_if(products, [&](const auto& entry) { return !keepProductsIter->second.contains(entry.m_subID); });
}
DeleteProducts(products);
}
}
@@ -1569,7 +1577,7 @@ namespace AssetProcessor
// this might be interesting, but only if its a known product!
// the dictionary in statedata stores only the relative path, not the platform.
// which means right now we have, for example
// d:/SamplesProject/Cache/ios/textures/favorite.tga
// d:/AutomatedTesting/Cache/ios/textures/favorite.tga
// ^^^^^^^^^ projectroot
// ^^^^^^^^^^^^^^^^^^^^^ cache root
// ^^^^^^^^^^^^^^^^^^^^^^^^^ platform root
@@ -1701,7 +1709,19 @@ namespace AssetProcessor
AZ_TracePrintf(AssetProcessor::ConsoleChannel, "Deleting file %s because either its source file %s was removed or the builder did not emit this job.\n", fullProductPath.toUtf8().constData(), source.m_sourceName.c_str());
AssetProcessor::ProcessingJobInfoBus::Broadcast(&AssetProcessor::ProcessingJobInfoBus::Events::BeginCacheFileUpdate, fullProductPath.toUtf8().data());
successfullyRemoved &= QFile::remove(fullProductPath);
bool wasRemoved = QFile::remove(fullProductPath);
// Another process may be holding on to the file currently, so wait for a very brief period and then retry deleting
// once in case we were just too quick to delete the file before.
if (!wasRemoved)
{
constexpr int DeleteRetryDelay = 10;
AZStd::this_thread::sleep_for(AZStd::chrono::milliseconds(DeleteRetryDelay));
wasRemoved = QFile::remove(fullProductPath);
}
successfullyRemoved &= wasRemoved;
AssetProcessor::ProcessingJobInfoBus::Broadcast(&AssetProcessor::ProcessingJobInfoBus::Events::EndCacheFileUpdate, fullProductPath.toUtf8().data(), false);
if(productFileInfo.absoluteDir().entryList(QDir::Files | QDir::Dirs | QDir::NoDotAndDotDot).empty())
@@ -1805,10 +1825,9 @@ namespace AssetProcessor
{
if (!DeleteProducts(products))
{
// try again in a while. Achieve this by recycling the item back into
// the queue as if it had been deleted again.
CheckSource(FileEntry(normalizedPath, true));
AZ_TracePrintf(AssetProcessor::ConsoleChannel, "Delete failed on %s. Will retry!. \n", normalizedPath.toUtf8().constData());
// DeleteProducts will make an attempt to retry deleting each product
// We can't just re-queue the whole file with CheckSource because we're deleting bits from the database as we go
AZ_TracePrintf(AssetProcessor::ConsoleChannel, "Delete failed on %s.\n", normalizedPath.toUtf8().constData());
}
}
else
Code/Tools/AssetProcessor/native/AssetManager/assetScannerWorker.cpp
+11 -3
View File
@@ -105,6 +105,16 @@ void AssetScannerWorker::ScanForSourceFiles(const ScanFolderInfo& scanFolderInfo
AZ::u64 fileSize = isDirectory ? 0 : entry.size();
AssetFileInfo assetFileInfo(absPath, modTime, fileSize, &rootScanFolder, isDirectory);
// Skip over the Cache folder if the file entry is the project cache root
QDir projectCacheRoot;
AssetUtilities::ComputeProjectCacheRoot(projectCacheRoot);
QString relativeToProjectCacheRoot = projectCacheRoot.relativeFilePath(absPath);
if (QDir::isRelativePath(relativeToProjectCacheRoot) && !relativeToProjectCacheRoot.startsWith(".."))
{
// The Cache folder should not be scanned
continue;
}
// Filtering out excluded files
if (m_platformConfiguration->IsFileExcluded(absPath))
{
@@ -136,7 +146,5 @@ void AssetScannerWorker::EmitFiles()
m_folderList.clear();
Q_EMIT ExcludedFound(m_excludedList);
m_excludedList.clear();
}
Code/Tools/AssetProcessor/native/InternalBuilders/SettingsRegistryBuilder.cpp
+8 -4
View File
@@ -157,7 +157,7 @@ namespace AssetProcessor
{
AssetBuilderSDK::AssetBuilderDesc builderDesc;
builderDesc.m_name = "Settings Registry Builder";
builderDesc.m_patterns.emplace_back("*/bootstrap.cfg", AssetBuilderSDK::AssetBuilderPattern::PatternType::Wildcard);
builderDesc.m_patterns.emplace_back("*/engine.json", AssetBuilderSDK::AssetBuilderPattern::PatternType::Wildcard);
builderDesc.m_builderType = AssetBuilderSDK::AssetBuilderDesc::AssetBuilderType::Internal;
builderDesc.m_busId = m_builderId;
builderDesc.m_createJobFunction = AZStd::bind(&SettingsRegistryBuilder::CreateJobs, this, AZStd::placeholders::_1, AZStd::placeholders::_2);
@@ -195,7 +195,9 @@ namespace AssetProcessor
response.m_createJobOutputs.push_back(AZStd::move(job));
}
response.m_sourceFileDependencyList.emplace_back("*.setreg", AZ::Uuid::CreateNull(),
AZ::IO::Path settingsRegistryWildcard = AZ::SettingsRegistryInterface::RegistryFolder;
settingsRegistryWildcard /= "*.setreg";
response.m_sourceFileDependencyList.emplace_back(AZStd::move(settingsRegistryWildcard.Native()), AZ::Uuid::CreateNull(),
AssetBuilderSDK::SourceFileDependency::SourceFileDependencyType::Wildcards);
response.m_result = AssetBuilderSDK::CreateJobsResultCode::Success;
}
@@ -294,10 +296,11 @@ namespace AssetProcessor
AZ::SettingsRegistryMergeUtils::MergeSettingsToRegistry_GemRegistries(registry, platform, specialization, &scratchBuffer);
AZ::SettingsRegistryMergeUtils::MergeSettingsToRegistry_ProjectRegistry(registry, platform, specialization, &scratchBuffer);
// Merge the Developer User settings registry only in non-release builds
// Merge the Project User and User home settings registry only in non-release builds
if (!specialization.Contains("release"))
{
AZ::SettingsRegistryMergeUtils::MergeSettingsToRegistry_UserRegistry(registry, platform, specialization, &scratchBuffer);
AZ::SettingsRegistryMergeUtils::MergeSettingsToRegistry_O3deUserRegistry(registry, platform, specialization, &scratchBuffer);
AZ::SettingsRegistryMergeUtils::MergeSettingsToRegistry_ProjectUserRegistry(registry, platform, specialization, &scratchBuffer);
}
AZ::ComponentApplicationBus::Broadcast([&registry](AZ::ComponentApplicationRequests* appRequests)
@@ -336,6 +339,7 @@ namespace AssetProcessor
file.Close();
response.m_outputProducts.emplace_back(outputPath, m_assetType, productSubID + aznumeric_cast<AZ::u32>(i));
response.m_outputProducts.back().m_dependenciesHandled = true;
outputPath.erase(extensionOffset);
}
Code/Tools/AssetProcessor/native/tests/AssetCatalog/AssetCatalogUnitTests.cpp
+1 -1
View File
@@ -120,7 +120,7 @@ namespace AssetProcessor
QString canonicalTempDirPath = AssetUtilities::NormalizeDirectoryPath(m_data->m_temporarySourceDir.canonicalPath());
m_data->m_temporarySourceDir = QDir(canonicalTempDirPath);
m_data->m_scopedDir.Setup(m_data->m_temporarySourceDir.path());
m_data->m_gameName = AssetUtilities::ComputeProjectName("SamplesProject"); // uses the above file.
m_data->m_gameName = AssetUtilities::ComputeProjectName("AutomatedTesting"); // uses the above file.
AssetUtilities::ResetAssetRoot();
QDir newRoot; // throwaway dummy var - we just want to invoke the below function
Code/Tools/AssetProcessor/native/tests/AssetProcessorMessagesTests.cpp
+3 -3
View File
@@ -105,8 +105,8 @@ namespace AssetProcessorMessagesTests
m_batchApplicationManager = AZStd::make_unique<UnitTestBatchApplicationManager>(&argC, nullptr, nullptr);
m_batchApplicationManager->BeforeRun();
// Override Game Name to be "SamplesProject"
AssetUtilities::ComputeProjectName("SamplesProject", true);
// Override Game Name to be "AutomatedTesting"
AssetUtilities::ComputeProjectName("AutomatedTesting", true);
m_batchApplicationManager->m_platformConfiguration = new PlatformConfiguration();
m_batchApplicationManager->InitAssetProcessorManager();
@@ -150,7 +150,7 @@ namespace AssetProcessorMessagesTests
connectionSettings.m_assetProcessorIp = "127.0.0.1";
connectionSettings.m_assetProcessorPort = AssetProcessorPort;
connectionSettings.m_branchToken = appBranchToken;
connectionSettings.m_projectName = "SamplesProject";
connectionSettings.m_projectName = "AutomatedTesting";
connectionSettings.m_assetPlatform = "pc";
connectionSettings.m_connectionIdentifier = "UNITTEST";
connectionSettings.m_connectTimeout = AZStd::chrono::seconds(15);
Code/Tools/AssetProcessor/native/tests/assetmanager/AssetProcessorManagerTest.cpp
+2 -2
View File
@@ -227,7 +227,7 @@ void AssetProcessorManagerTest::SetUp()
SetArgReferee<0>(m_data->m_databaseLocation),
Return(true)));
m_gameName = AssetUtilities::ComputeProjectName("SamplesProject", true);
m_gameName = AssetUtilities::ComputeProjectName("AutomatedTesting", true);
AssetUtilities::ResetAssetRoot();
QDir newRoot;
@@ -1665,7 +1665,7 @@ TEST_F(PathDependencyTest, NoLongerProcessedFile_IsRemoved)
m_sharedConnection->GetProductsBySourceName("test1.txt", products);
ASSERT_EQ(products.size(), 0);
ASSERT_FALSE(QFile::exists(m_normalizedCacheRootDir.absoluteFilePath("pc/samplesproject/test1.asset1").toUtf8().constData()));
ASSERT_FALSE(QFile::exists(m_normalizedCacheRootDir.absoluteFilePath("pc/automatedtesting/test1.asset1").toUtf8().constData()));
}
TEST_F(PathDependencyTest, AssetProcessed_Impl_SelfReferrentialProductDependency_DependencyIsRemoved)
Code/Tools/AssetProcessor/native/tests/platformconfiguration/platformconfigurationtests.cpp
+2 -2
View File
@@ -586,7 +586,7 @@ TEST_F(PlatformConfigurationUnitTests, Test_GemHandling)
QTemporaryDir tempEngineRoot;
QDir tempPath(tempEngineRoot.path());
AssetUtilities::ResetAssetRoot();
AssetUtilities::ComputeProjectName("SamplesProject", true);
AssetUtilities::ComputeProjectName("AutomatedTesting", true);
QDir computedEngineRoot;
ASSERT_TRUE(AssetUtilities::ComputeAssetRoot(computedEngineRoot, &tempPath));
@@ -666,7 +666,7 @@ TEST_F(PlatformConfigurationUnitTests, PlatformConfigFile_IsPresent_Found)
QTemporaryDir tempEngineRoot;
QDir tempPath(tempEngineRoot.path());
AssetUtilities::ResetAssetRoot();
AssetUtilities::ComputeProjectName("SamplesProject", true);
AssetUtilities::ComputeProjectName("AutomatedTesting", true);
auto settingsRegistry = AZ::SettingsRegistry::Get();
ASSERT_NE(nullptr, settingsRegistry);
Code/Tools/AssetProcessor/native/unittests/AssetProcessorManagerUnitTests.cpp
+1 -1
View File
@@ -70,7 +70,7 @@ namespace AssetProcessor
namespace
{
constexpr const char* AssetProcessorManagerTestGameProject = "SamplesProject";
constexpr const char* AssetProcessorManagerTestGameProject = "AutomatedTesting";
/// This functions sorts the processed result list by platform name
/// if platform is same than it sorts by job description
void sortAssetToProcessResultList(QList<JobDetails>& processResults)
Code/Tools/AssetProcessor/native/utilities/GUIApplicationManager.cpp
+1
View File
@@ -669,6 +669,7 @@ void GUIApplicationManager::FileChanged(QString path)
AZ::SettingsRegistryInterface* settingsRegistry = AZ::SettingsRegistry::Get();
AZ_Assert(settingsRegistry, "Unable to retrieve global SettingsRegistry, it should be available now");
AZ::SettingsRegistryMergeUtils::MergeSettingsToRegistry_Bootstrap(*settingsRegistry);
AZ::SettingsRegistryMergeUtils::MergeSettingsToRegistry_O3deUserRegistry(*settingsRegistry, AZ_TRAIT_OS_PLATFORM_CODENAME, {});
AZ::SettingsRegistryMergeUtils::MergeSettingsToRegistry_CommandLine(*settingsRegistry, *m_frameworkApp.GetAzCommandLine(), false);
AZ::SettingsRegistryMergeUtils::MergeSettingsToRegistry_AddRuntimeFilePaths(*settingsRegistry);
}
Code/Tools/AssetProcessor/native/utilities/PlatformConfiguration.cpp
+2
View File
@@ -1011,6 +1011,7 @@ namespace AssetProcessor
}
AZ::IO::FixedMaxPathString projectPath = AZ::Utils::GetProjectPath();
AZ::IO::FixedMaxPathString projectName = AZ::Utils::GetProjectName();
AZ::IO::FixedMaxPath engineRoot(AZ::IO::PosixPathSeparator);
settingsRegistry->Get(engineRoot.Native(), AZ::SettingsRegistryMergeUtils::FilePathKey_EngineRootFolder);
@@ -1061,6 +1062,7 @@ namespace AssetProcessor
AZ::StringFunc::Replace(scanFolderEntry.m_scanFolderDisplayName, "@ROOT@", assetRootPath.c_str());
AZ::StringFunc::Replace(scanFolderEntry.m_scanFolderDisplayName, "@PROJECTROOT@", projectPath.c_str());
AZ::StringFunc::Replace(scanFolderEntry.m_scanFolderDisplayName, "@PROJECTNAME@", projectName.c_str());
AZ::StringFunc::Replace(scanFolderEntry.m_scanFolderDisplayName, "@ENGINEROOT@", engineRoot.c_str());
QStringList includeIdentifiers;
Code/Tools/AssetProcessor/native/utilities/assetUtils.h
+1 -1
View File
@@ -97,7 +97,7 @@ namespace AssetUtilities
bool ShouldUseFileHashing();
//! Determine the name of the current project - for example, SamplesProject
//! Determine the name of the current project - for example, AutomatedTesting
//! Can be overridden by passing in a non-empty projectNameOverride
//! The override will persist if the project name wasn't set previously or
//! force=true is supplied