b7f6b57e16
This removes mostly benign (but noisy) messages about "Failed to find builder dependency". Changed AssetUtils::GetPossibleDepenencyPaths to return all possible source paths, rather than stopping when one is found. This function is now used to report a list of all possible source dependencies, so that CreateJobs will get called by the AP whenever a file shows up at one of those locations. If a file was missing before and then appears, this will cause the builders to wake up and add the appropriate job dependencies on the new files. In ShaderVariantAssetBuilder and MaterialBuilder, we now use GetPossibleDepenencyPaths to report source dependencies rather than job dependencies. We only report a job dependency when the actual source file has been identified. This should all now be consistent with the intended design of the AP's dependency systems (the prior approach was a hack based on misunderstanding of what source dependencies are). SrgLayoutBuilder's change is a bit tricky. The above changes did not fix all of the "Failed to find builder dependency" messages because AzslBuilder sometimes skips particular files in CreateJobs. When this happens, it is invalid to report an AzslBuilder job dependency on that file. So I copied the same conditional code that is used to skip an azsli file in CreateJobs, and used that to skip AddAzslBuilderJobDependency() in SrgLayoutBuilder as well. With all these changes combined, I do think we've solved the issue where jobs fail to evict outdated jobs, as described in ATOM-15134. However, we are not yet seeing the iteration time improvements we were hoping for. Before these changes I was seeing roughly a 0.5 minute delay for the initial change, and a 2 minute delay for a subsequent change. With these changes it's more like 0.5 mibutes and 1.5 minutes. It appears that the AP scan is being starved by all the AssetBuilder processing going on, and perhaps IO contention. I suspect that this will be greatly improved on the development branch where we no longer have AzslBuilder and SrgLayoutBuilder slowing things down. ATOM-15136 Builder dependency errors reported in mainline ATOM-15134 Replace GetPossibleDepenencyPaths Approach with Source Dependencies
152 lines
7.3 KiB
C++
152 lines
7.3 KiB
C++
/*
|
|
* All or portions of this file Copyright(c) Amazon.com, Inc.or its affiliates or
|
|
* its licensors.
|
|
*
|
|
* For complete copyright and license terms please see the LICENSE at the root of this
|
|
* distribution(the "License").All use of this software is governed by the License,
|
|
*or, if provided, by the license below or the license accompanying this file.Do not
|
|
* remove or modify any license notices.This file is distributed on an "AS IS" BASIS,
|
|
*WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
*
|
|
*/
|
|
|
|
#include <Atom/RPI.Edit/Common/AssetUtils.h>
|
|
#include <AzFramework/StringFunc/StringFunc.h>
|
|
#include <AzToolsFramework/API/EditorAssetSystemAPI.h>
|
|
#include <AzCore/IO/IOUtils.h>
|
|
|
|
namespace AZ
|
|
{
|
|
namespace RPI
|
|
{
|
|
namespace AssetUtils
|
|
{
|
|
uint32_t CalcSrgProductSubId(const Name& srgName)
|
|
{
|
|
//[GFX TODO] Make SrgLayoutBuilder share this code after the gems reorg is complete
|
|
return static_cast<uint32_t>(AZStd::hash<AZStd::string>()(srgName.GetStringView()) & 0xFFFFFFFF);
|
|
}
|
|
|
|
AZStd::string GetProductPathByAssetId(const AZ::Data::AssetId& assetId)
|
|
{
|
|
AZStd::string productPath;
|
|
if (assetId.IsValid())
|
|
{
|
|
Data::AssetCatalogRequestBus::BroadcastResult(productPath, &Data::AssetCatalogRequests::GetAssetPathById, assetId);
|
|
AZ::StringFunc::Path::Normalize(productPath);
|
|
}
|
|
return productPath;
|
|
}
|
|
|
|
AZStd::string GetSourcePathByAssetId(const AZ::Data::AssetId& assetId)
|
|
{
|
|
AZStd::string sourcePath;
|
|
if (assetId.IsValid())
|
|
{
|
|
const AZStd::string& productPath = GetProductPathByAssetId(assetId);
|
|
if (!productPath.empty())
|
|
{
|
|
bool sourceFileFound = false;
|
|
AzToolsFramework::AssetSystemRequestBus::BroadcastResult(sourceFileFound, &AzToolsFramework::AssetSystem::AssetSystemRequest::GetFullSourcePathFromRelativeProductPath, productPath, sourcePath);
|
|
AZ::StringFunc::Path::Normalize(sourcePath);
|
|
}
|
|
}
|
|
return sourcePath;
|
|
}
|
|
|
|
AZStd::string ResolvePathReference(const AZStd::string& originatingSourceFilePath, const AZStd::string& referencedSourceFilePath)
|
|
{
|
|
AZStd::string normalizedReferencedPath = referencedSourceFilePath;
|
|
AzFramework::StringFunc::Path::Normalize(normalizedReferencedPath);
|
|
|
|
AZStd::string originatingSourceFolder = originatingSourceFilePath;
|
|
AzFramework::StringFunc::Path::StripFullName(originatingSourceFolder);
|
|
|
|
AZStd::string pathFromOriginatingFolder;
|
|
AzFramework::StringFunc::Path::Join(originatingSourceFolder.c_str(), referencedSourceFilePath.c_str(), pathFromOriginatingFolder);
|
|
|
|
bool assetFound = false;
|
|
AZ::Data::AssetInfo sourceInfo;
|
|
AZStd::string watchFolder;
|
|
|
|
// Try to find the source file starting at the originatingSourceFilePath, and return the full path
|
|
AzToolsFramework::AssetSystemRequestBus::BroadcastResult(assetFound, &AzToolsFramework::AssetSystem::AssetSystemRequest::GetSourceInfoBySourcePath, pathFromOriginatingFolder.c_str(), sourceInfo, watchFolder);
|
|
if (assetFound)
|
|
{
|
|
AZStd::string fullSourcePath;
|
|
// Construct fails if either of the watchFolder (root) or the pathFromOriginatingFolder is empty.
|
|
// For some testing purposes, root can be empty.
|
|
if (AzFramework::StringFunc::Path::ConstructFull(watchFolder.c_str(), pathFromOriginatingFolder.c_str(), fullSourcePath, true))
|
|
{
|
|
return fullSourcePath;
|
|
}
|
|
else
|
|
{
|
|
return pathFromOriginatingFolder;
|
|
}
|
|
}
|
|
|
|
// Try to find the source file starting at the asset root, and return the full path
|
|
AzToolsFramework::AssetSystemRequestBus::BroadcastResult(assetFound, &AzToolsFramework::AssetSystem::AssetSystemRequest::GetSourceInfoBySourcePath, normalizedReferencedPath.c_str(), sourceInfo, watchFolder);
|
|
if (assetFound)
|
|
{
|
|
AZStd::string fullSourcePath;
|
|
// Construct fails if either of the watchFolder (root) or the normalizedReferencedPath is empty.
|
|
// For some testing purposes, root can be empty.
|
|
if (AzFramework::StringFunc::Path::ConstructFull(watchFolder.c_str(), normalizedReferencedPath.c_str(), fullSourcePath, true))
|
|
{
|
|
return fullSourcePath;
|
|
}
|
|
else
|
|
{
|
|
return normalizedReferencedPath;
|
|
}
|
|
}
|
|
|
|
// If no source file could be found, just return the original reference path. Something else will probably fail and report errors.
|
|
return normalizedReferencedPath;
|
|
}
|
|
|
|
AZStd::vector<AZStd::string> GetPossibleDepenencyPaths(const AZStd::string& originatingSourceFilePath, const AZStd::string& referencedSourceFilePath)
|
|
{
|
|
AZStd::vector<AZStd::string> results;
|
|
|
|
// Use the referencedSourceFilePath as a relative path starting at originatingSourceFilePath
|
|
AZStd::string combinedPath = originatingSourceFilePath;
|
|
AzFramework::StringFunc::Path::StripFullName(combinedPath);
|
|
AzFramework::StringFunc::Path::Join(combinedPath.c_str(), referencedSourceFilePath.c_str(), combinedPath);
|
|
results.push_back(combinedPath);
|
|
|
|
// Use the referencedSourceFilePath as a standard asset path
|
|
results.push_back(referencedSourceFilePath);
|
|
|
|
return results;
|
|
}
|
|
|
|
Outcome<Data::AssetId> MakeAssetId(const AZStd::string& sourcePath, uint32_t productSubId)
|
|
{
|
|
bool assetFound = false;
|
|
AZ::Data::AssetInfo sourceInfo;
|
|
AZStd::string watchFolder;
|
|
AzToolsFramework::AssetSystemRequestBus::BroadcastResult(assetFound, &AzToolsFramework::AssetSystem::AssetSystemRequest::GetSourceInfoBySourcePath, sourcePath.c_str(), sourceInfo, watchFolder);
|
|
|
|
if (!assetFound)
|
|
{
|
|
AZ_Error("AssetUtils", false, "Could not find asset [%s]", sourcePath.c_str());
|
|
return AZ::Failure();
|
|
}
|
|
else
|
|
{
|
|
return AZ::Success(AZ::Data::AssetId(sourceInfo.m_assetId.m_guid, productSubId));
|
|
}
|
|
}
|
|
|
|
Outcome<Data::AssetId> MakeAssetId(const AZStd::string& originatingSourcePath, const AZStd::string& referencedSourceFilePath, uint32_t productSubId)
|
|
{
|
|
AZStd::string resolvedPath = ResolvePathReference(originatingSourcePath, referencedSourceFilePath);
|
|
return MakeAssetId(resolvedPath, productSubId);
|
|
}
|
|
} // namespace AssetUtils
|
|
} // namespace RPI
|
|
} // namespace AZ
|