Merge branch 'main' into sceneapi_script_autotest

This commit is contained in:
jackalbe
2021-04-14 08:32:59 -05:00
2497 changed files with 45024 additions and 527853 deletions
@@ -28,7 +28,7 @@ AZ_POP_DISABLE_WARNING
namespace AWSNativeSDKInit
{
// Entry point for Lumberyard managing the AWSNativeSDK's initialization and shutdown requirements
// Entry point for Open 3D Engine managing the AWSNativeSDK's initialization and shutdown requirements
// Use an AZ::Environment variable to enforce only one init and shutdown
class InitializationManager
{
@@ -38,7 +38,7 @@ namespace AWSNativeSDKInit
InitializationManager();
~InitializationManager();
// Call to guarantee that the API is initialized with proper Lumberyard settings.
// Call to guarantee that the API is initialized with proper Open 3D Engine settings.
// It's fine to call this from every module which needs to use the NativeSDK
// Creates a static shared pointer using the AZ EnvironmentVariable system.
// This will prevent a the AWS SDK from going through the shutdown routine until all references are gone, or
@@ -11,4 +11,5 @@
set(FILES
source/main.cpp
source/AssetBundlerBatch.rc
)
@@ -0,0 +1,3 @@
version https://git-lfs.github.com/spec/v1
oid sha256:fe5b63adf64ca5b940bb9ff35a198e41f956e248b0c9bff349d36885a8e6bfcf
size 103824
@@ -0,0 +1 @@
IDI_ICON1 ICON DISCARDABLE "AssetBundlerBatch.ico"
@@ -22,6 +22,7 @@
#include <AzCore/Module/ModuleManagerBus.h>
#include <AzCore/Slice/SliceSystemComponent.h>
#include <AzCore/std/string/conversions.h>
#include <AzCore/StringFunc/StringFunc.h>
#include <AzCore/UserSettings/UserSettingsComponent.h>
#include <AzCore/Utils/Utils.h>
@@ -31,7 +32,6 @@
#include <AzFramework/FileTag/FileTagComponent.h>
#include <AzFramework/Input/System/InputSystemComponent.h>
#include <AzFramework/Platform/PlatformDefaults.h>
#include <AzFramework/StringFunc/StringFunc.h>
#include <AzFramework/Components/AzFrameworkConfigurationSystemComponent.h>
#include <AzToolsFramework/Archive/ArchiveComponent.h>
@@ -66,7 +66,7 @@ namespace AssetBundler
AzToolsFramework::AssetFileInfoListComparison::Reflect(context);
AzToolsFramework::AssetBundleSettings::Reflect(context);
AZ::IO::FileIOBase* fileIO = AZ::IO::FileIOBase::GetInstance();
[[maybe_unused]] AZ::IO::FileIOBase* fileIO = AZ::IO::FileIOBase::GetInstance();
AZ_Assert(fileIO != nullptr, "AZ::IO::FileIOBase must be ready for use.\n");
m_assetSeedManager = AZStd::make_unique<AzToolsFramework::AssetSeedManager>();
@@ -86,7 +86,7 @@ namespace AssetBundler
bool ApplicationManager::Run()
{
const AzFramework::CommandLine* parser = GetCommandLine();
const AZ::CommandLine* parser = GetCommandLine();
bool shouldPrintHelp = ShouldPrintHelp(parser);
@@ -196,7 +196,7 @@ namespace AssetBundler
// Get Generic Command Info
////////////////////////////////////////////////////////////////////////////////////////////
CommandType ApplicationManager::GetCommandType(const AzFramework::CommandLine* parser, [[maybe_unused]] bool suppressErrors)
CommandType ApplicationManager::GetCommandType(const AZ::CommandLine* parser, [[maybe_unused]] bool suppressErrors)
{
// Verify that the user has only typed in one sub-command
size_t numMiscValues = parser->GetNumMiscValues();
@@ -247,12 +247,12 @@ namespace AssetBundler
}
}
bool ApplicationManager::ShouldPrintHelp(const AzFramework::CommandLine* parser)
bool ApplicationManager::ShouldPrintHelp(const AZ::CommandLine* parser)
{
return parser->HasSwitch(AssetBundler::HelpFlag) || parser->HasSwitch(AssetBundler::HelpFlagAlias);
}
bool ApplicationManager::ShouldPrintVerbose(const AzFramework::CommandLine* parser)
bool ApplicationManager::ShouldPrintVerbose(const AZ::CommandLine* parser)
{
return parser->HasSwitch(AssetBundler::VerboseFlag);
}
@@ -368,17 +368,10 @@ namespace AssetBundler
// Store Detailed Command Info
////////////////////////////////////////////////////////////////////////////////////////////
AZ::Outcome<SeedsParams, AZStd::string> ApplicationManager::ParseSeedsCommandData(const AzFramework::CommandLine* parser)
AZ::Outcome<SeedsParams, AZStd::string> ApplicationManager::ParseSeedsCommandData(const AZ::CommandLine* parser)
{
using namespace AzToolsFramework;
auto validateArgsOutcome = ValidateInputArgs(parser, m_allSeedsArgs);
if (!validateArgsOutcome.IsSuccess())
{
OutputHelpSeeds();
return AZ::Failure(validateArgsOutcome.TakeError());
}
SeedsParams params;
params.m_ignoreFileCase = parser->HasSwitch(IgnoreFileCaseFlag);
@@ -467,15 +460,8 @@ namespace AssetBundler
return AZStd::string::format(FailureMessage, arg1, arg2);
}
AZ::Outcome<AssetListsParams, AZStd::string> ApplicationManager::ParseAssetListsCommandData(const AzFramework::CommandLine* parser)
AZ::Outcome<AssetListsParams, AZStd::string> ApplicationManager::ParseAssetListsCommandData(const AZ::CommandLine* parser)
{
auto validateArgsOutcome = ValidateInputArgs(parser, m_allAssetListsArgs);
if (!validateArgsOutcome.IsSuccess())
{
OutputHelpAssetLists();
return AZ::Failure(validateArgsOutcome.TakeError());
}
AssetListsParams params;
// Read in Platform arg
@@ -537,15 +523,8 @@ namespace AssetBundler
return AZ::Success(params);
}
AZ::Outcome<ComparisonRulesParams, AZStd::string> ApplicationManager::ParseComparisonRulesCommandData(const AzFramework::CommandLine* parser)
AZ::Outcome<ComparisonRulesParams, AZStd::string> ApplicationManager::ParseComparisonRulesCommandData(const AZ::CommandLine* parser)
{
auto validateArgsOutcome = ValidateInputArgs(parser, m_allComparisonRulesArgs);
if (!validateArgsOutcome.IsSuccess())
{
OutputHelpComparisonRules();
return AZ::Failure(validateArgsOutcome.TakeError());
}
ScopedTraceHandler traceHandler;
ComparisonRulesParams params;
@@ -572,7 +551,7 @@ namespace AssetBundler
break;
case 1:
params.m_comparisonRulesStepAction = ComparisonRulesStepAction::Add;
params.m_destinationLine = static_cast<size_t>(AzFramework::StringFunc::ToInt(parser->GetSwitchValue(AddComparisonStepArg, 0).c_str()));
params.m_destinationLine = static_cast<size_t>(AZ::StringFunc::ToInt(parser->GetSwitchValue(AddComparisonStepArg, 0).c_str()));
break;
default:
return AZ::Failure(AZStd::string::format("Invalid command: \"--%s\" cannot have more than one input value.", AddComparisonStepArg));
@@ -604,7 +583,7 @@ namespace AssetBundler
}
params.m_comparisonRulesStepAction = ComparisonRulesStepAction::Remove;
params.m_initialLine = static_cast<size_t>(AzFramework::StringFunc::ToInt(parser->GetSwitchValue(RemoveComparisonStepArg, 0).c_str()));
params.m_initialLine = static_cast<size_t>(AZ::StringFunc::ToInt(parser->GetSwitchValue(RemoveComparisonStepArg, 0).c_str()));
}
// Read in Move Comparison Step arg
@@ -625,8 +604,8 @@ namespace AssetBundler
}
params.m_comparisonRulesStepAction = ComparisonRulesStepAction::Move;
params.m_initialLine = static_cast<size_t>(AzFramework::StringFunc::ToInt(parser->GetSwitchValue(MoveComparisonStepArg, 0).c_str()));
params.m_destinationLine = static_cast<size_t>(AzFramework::StringFunc::ToInt(parser->GetSwitchValue(MoveComparisonStepArg, 1).c_str()));
params.m_initialLine = static_cast<size_t>(AZ::StringFunc::ToInt(parser->GetSwitchValue(MoveComparisonStepArg, 0).c_str()));
params.m_destinationLine = static_cast<size_t>(AZ::StringFunc::ToInt(parser->GetSwitchValue(MoveComparisonStepArg, 1).c_str()));
}
// Read in Edit Comparison Step arg
@@ -642,12 +621,12 @@ namespace AssetBundler
if (parser->GetNumSwitchValues(EditComparisonStepArg) != 1)
{
return AZ::Failure(AZStd::string::format(
"Invalid command: \"--%s\" requires exatly one input value (the line number of the Comparison Step you wish to edit)",
"Invalid command: \"--%s\" requires exactly one input value (the line number of the Comparison Step you wish to edit)",
EditComparisonStepArg));
}
params.m_comparisonRulesStepAction = ComparisonRulesStepAction::Edit;
params.m_initialLine = static_cast<size_t>(AzFramework::StringFunc::ToInt(parser->GetSwitchValue(EditComparisonStepArg, 0).c_str()));
params.m_initialLine = static_cast<size_t>(AZ::StringFunc::ToInt(parser->GetSwitchValue(EditComparisonStepArg, 0).c_str()));
// When editing a Comparison Step, we can only accept one input for every value type
auto parseComparisonTypesForEditOutcome = ParseComparisonTypesAndPatternsForEditCommand(parser, params);
@@ -686,7 +665,7 @@ namespace AssetBundler
return AZ::Success(params);
}
AZ::Outcome<void, AZStd::string> ApplicationManager::ParseComparisonTypesAndPatterns(const AzFramework::CommandLine* parser, ComparisonRulesParams& params)
AZ::Outcome<void, AZStd::string> ApplicationManager::ParseComparisonTypesAndPatterns(const AZ::CommandLine* parser, ComparisonRulesParams& params)
{
int filePatternsConsumed = 0;
size_t numComparisonTypes = parser->GetNumSwitchValues(ComparisonTypeArg);
@@ -770,7 +749,7 @@ namespace AssetBundler
return AZ::Success();
}
AZ::Outcome<void, AZStd::string> ApplicationManager::ParseComparisonTypesAndPatternsForEditCommand(const AzFramework::CommandLine* parser, ComparisonRulesParams& params)
AZ::Outcome<void, AZStd::string> ApplicationManager::ParseComparisonTypesAndPatternsForEditCommand(const AZ::CommandLine* parser, ComparisonRulesParams& params)
{
if (parser->HasSwitch(ComparisonTypeArg))
{
@@ -855,7 +834,7 @@ namespace AssetBundler
return AZ::Success();
}
AZ::Outcome<void, AZStd::string> ApplicationManager::ParseComparisonRulesFirstAndSecondInputArgs(const AzFramework::CommandLine* parser, ComparisonRulesParams& params)
AZ::Outcome<void, AZStd::string> ApplicationManager::ParseComparisonRulesFirstAndSecondInputArgs(const AZ::CommandLine* parser, ComparisonRulesParams& params)
{
if (params.m_comparisonTypeList.size() > 1 && (parser->HasSwitch(ComparisonFirstInputArg) || parser->HasSwitch(ComparisonSecondInputArg)))
{
@@ -926,15 +905,8 @@ namespace AssetBundler
return AZ::Success();
}
AZ::Outcome<ComparisonParams, AZStd::string> ApplicationManager::ParseCompareCommandData(const AzFramework::CommandLine* parser)
AZ::Outcome<ComparisonParams, AZStd::string> ApplicationManager::ParseCompareCommandData(const AZ::CommandLine* parser)
{
auto validateArgsOutcome = ValidateInputArgs(parser, m_allCompareArgs);
if (!validateArgsOutcome.IsSuccess())
{
OutputHelpCompare();
return AZ::Failure(validateArgsOutcome.TakeError());
}
ComparisonParams params;
// Read in Platform arg
@@ -1024,15 +996,8 @@ namespace AssetBundler
return AZ::Success(params);
}
AZ::Outcome<BundleSettingsParams, AZStd::string> ApplicationManager::ParseBundleSettingsCommandData(const AzFramework::CommandLine* parser)
AZ::Outcome<BundleSettingsParams, AZStd::string> ApplicationManager::ParseBundleSettingsCommandData(const AZ::CommandLine* parser)
{
auto validateArgsOutcome = ValidateInputArgs(parser, m_allBundleSettingsArgs);
if (!validateArgsOutcome.IsSuccess())
{
OutputHelpBundleSettings();
return AZ::Failure(validateArgsOutcome.TakeError());
}
BundleSettingsParams params;
// Read in Platform arg
@@ -1099,7 +1064,7 @@ namespace AssetBundler
return AZ::Success(params);
}
AZ::Outcome<BundlesParamsList, AZStd::string> ApplicationManager::ParseBundleSettingsAndOverrides(const AzFramework::CommandLine* parser, const char* commandName)
AZ::Outcome<BundlesParamsList, AZStd::string> ApplicationManager::ParseBundleSettingsAndOverrides(const AZ::CommandLine* parser, const char* commandName)
{
// Read in Bundle Settings File args
auto bundleSettingsOutcome = GetArgsList<FilePath>(parser, BundleSettingsFileArg, commandName);
@@ -1235,15 +1200,8 @@ namespace AssetBundler
return AZ::Success(bundleParamsList);
}
AZ::Outcome<BundlesParamsList, AZStd::string> ApplicationManager::ParseBundlesCommandData(const AzFramework::CommandLine* parser)
AZ::Outcome<BundlesParamsList, AZStd::string> ApplicationManager::ParseBundlesCommandData(const AZ::CommandLine* parser)
{
auto validateArgsOutcome = ValidateInputArgs(parser, m_allBundlesArgs);
if (!validateArgsOutcome.IsSuccess())
{
OutputHelpBundles();
return AZ::Failure(validateArgsOutcome.TakeError());
}
auto parseSettingsOutcome = ParseBundleSettingsAndOverrides(parser, BundlesCommand);
if (!parseSettingsOutcome.IsSuccess())
{
@@ -1253,16 +1211,8 @@ namespace AssetBundler
return AZ::Success(parseSettingsOutcome.TakeValue());
}
AZ::Outcome<BundleSeedParams, AZStd::string> ApplicationManager::ParseBundleSeedCommandData(const AzFramework::CommandLine* parser)
AZ::Outcome<BundleSeedParams, AZStd::string> ApplicationManager::ParseBundleSeedCommandData(const AZ::CommandLine* parser)
{
auto validateArgsOutcome = ValidateInputArgs(parser, m_allBundleSeedArgs);
if (!validateArgsOutcome.IsSuccess())
{
OutputHelpBundles();
return AZ::Failure(validateArgsOutcome.TakeError());
}
BundleSeedParams params;
params.m_addSeedList = GetAddSeedArgList(parser);
@@ -1278,7 +1228,7 @@ namespace AssetBundler
return AZ::Success(params);
}
AZ::Outcome<void, AZStd::string> ApplicationManager::ValidateInputArgs(const AzFramework::CommandLine* parser, const AZStd::vector<const char*>& validArgList)
AZ::Outcome<void, AZStd::string> ApplicationManager::ValidateInputArgs(const AZ::CommandLine* parser, const AZStd::vector<const char*>& validArgList)
{
for (const auto& paramInfo : *parser)
{
@@ -1291,7 +1241,7 @@ namespace AssetBundler
for (const auto& validArg : validArgList)
{
if (AzFramework::StringFunc::Equal(paramInfo.m_option, validArg))
if (AZ::StringFunc::Equal(paramInfo.m_option, validArg))
{
isValidArg = true;
break;
@@ -1300,14 +1250,14 @@ namespace AssetBundler
if (!isValidArg)
{
return AZ::Failure(AZStd::string::format("Invalid command: \"--%s\" is not a valid argument for this sub-command.", paramInfo.m_option.c_str()));
return AZ::Failure(AZStd::string::format("Unknown argument: \"--%s\" is not an unknown argument for this sub-command.", paramInfo.m_option.c_str()));
}
}
return AZ::Success();
}
AZ::Outcome<AZStd::string, AZStd::string> ApplicationManager::GetFilePathArg(const AzFramework::CommandLine* parser, const char* argName, const char* subCommandName, bool isRequired)
AZ::Outcome<AZStd::string, AZStd::string> ApplicationManager::GetFilePathArg(const AZ::CommandLine* parser, const char* argName, const char* subCommandName, bool isRequired)
{
if (!parser->HasSwitch(argName))
{
@@ -1328,7 +1278,7 @@ namespace AssetBundler
template <typename T>
AZ::Outcome<AZStd::vector<T>, AZStd::string> ApplicationManager::GetArgsList(const AzFramework::CommandLine* parser, const char* argName, const char* subCommandName, bool isRequired)
AZ::Outcome<AZStd::vector<T>, AZStd::string> ApplicationManager::GetArgsList(const AZ::CommandLine* parser, const char* argName, const char* subCommandName, bool isRequired)
{
AZStd::vector<T> args;
@@ -1352,7 +1302,7 @@ namespace AssetBundler
return AZ::Success(args);
}
AZ::Outcome<AzFramework::PlatformFlags, AZStd::string> ApplicationManager::GetPlatformArg(const AzFramework::CommandLine* parser)
AZ::Outcome<AzFramework::PlatformFlags, AZStd::string> ApplicationManager::GetPlatformArg(const AZ::CommandLine* parser)
{
using namespace AzFramework;
PlatformFlags platform = AzFramework::PlatformFlags::Platform_NONE;
@@ -1395,7 +1345,7 @@ namespace AssetBundler
return platformFlags;
}
AZStd::vector<AZStd::string> ApplicationManager::GetAddSeedArgList(const AzFramework::CommandLine* parser)
AZStd::vector<AZStd::string> ApplicationManager::GetAddSeedArgList(const AZ::CommandLine* parser)
{
AZStd::vector<AZStd::string> addSeedList;
size_t numAddSeedArgs = parser->GetNumSwitchValues(AddSeedArg);
@@ -1406,7 +1356,7 @@ namespace AssetBundler
return addSeedList;
}
AZStd::vector<AZStd::string> ApplicationManager::GetSkipArgList(const AzFramework::CommandLine* parser)
AZStd::vector<AZStd::string> ApplicationManager::GetSkipArgList(const AZ::CommandLine* parser)
{
AZStd::vector<AZStd::string> skipList;
size_t numArgs = parser->GetNumSwitchValues(SkipArg);
@@ -1920,10 +1870,12 @@ namespace AssetBundler
AZStd::vector<AZStd::string> destructiveOverwriteFilePaths = comparisonOperations.GetDestructiveOverwriteFilePaths();
if (!destructiveOverwriteFilePaths.empty())
{
#if defined(AZ_ENABLE_TRACING)
for (const AZStd::string& path : destructiveOverwriteFilePaths)
{
AZ_Error(AssetBundler::AppWindowName, false, "Asset List file ( %s ) already exists, running this command would perform a destructive overwrite.", path.c_str());
}
#endif
AZ_Printf(AssetBundler::AppWindowName, "\nRun your command again with the ( --%s ) arg if you want to save over the existing file.\n\n", AllowOverwritesFlag)
hasError = true;
continue;
@@ -2028,7 +1980,7 @@ namespace AssetBundler
AZStd::string assetListFilePath = FilePath(params.m_assetListFile.AbsolutePath(), platformName).AbsolutePath();
if (!assetListFilePath.empty())
{
if (!AzFramework::StringFunc::EndsWith(assetListFilePath, AssetSeedManager::GetAssetListFileExtension()))
if (!AZ::StringFunc::EndsWith(assetListFilePath, AssetSeedManager::GetAssetListFileExtension()))
{
AZ_Error(AppWindowName, false, "Cannot set Asset List file to ( %s ): file extension must be ( %s ).", assetListFilePath.c_str(), AssetSeedManager::GetAssetListFileExtension());
return false;
@@ -2041,7 +1993,7 @@ namespace AssetBundler
}
// Make the path relative to the engine root folder before saving
AzFramework::StringFunc::Replace(assetListFilePath, GetEngineRoot(), "");
AZ::StringFunc::Replace(assetListFilePath, GetEngineRoot(), "");
bundleSettings.m_assetFileInfoListPath = assetListFilePath;
}
@@ -2050,14 +2002,14 @@ namespace AssetBundler
AZStd::string outputBundlePath = FilePath(params.m_outputBundlePath.AbsolutePath(), platformName).AbsolutePath();
if (!outputBundlePath.empty())
{
if (!AzFramework::StringFunc::EndsWith(outputBundlePath, AssetBundleSettings::GetBundleFileExtension()))
if (!AZ::StringFunc::EndsWith(outputBundlePath, AssetBundleSettings::GetBundleFileExtension()))
{
AZ_Error(AppWindowName, false, "Cannot set Output Bundle Path to ( %s ): file extension must be ( %s ).", outputBundlePath.c_str(), AssetBundleSettings::GetBundleFileExtension());
return false;
}
// Make the path relative to the engine root folder before saving
AzFramework::StringFunc::Replace(outputBundlePath, GetEngineRoot(), "");
AZ::StringFunc::Replace(outputBundlePath, GetEngineRoot(), "");
bundleSettings.m_bundleFilePath = outputBundlePath;
}
@@ -2341,7 +2293,7 @@ namespace AssetBundler
AZStd::string platformSpecificAssetCatalogPath;
if (assetCatalogFile.empty())
{
AzFramework::StringFunc::Path::ConstructFull(
AZ::StringFunc::Path::ConstructFull(
PlatformAddressedAssetCatalog::GetAssetRootForPlatform(platformId).c_str(),
AssetBundler::AssetCatalogFilename,
platformSpecificAssetCatalogPath);
@@ -2489,7 +2441,7 @@ namespace AssetBundler
if (params.m_generateDebugFile)
{
debugListFileAbsolutePath = assetListFileAbsolutePath;
AzFramework::StringFunc::Path::ReplaceExtension(debugListFileAbsolutePath, AssetFileDebugInfoList::GetAssetListDebugFileExtension());
AZ::StringFunc::Path::ReplaceExtension(debugListFileAbsolutePath, AssetFileDebugInfoList::GetAssetListDebugFileExtension());
AZ_TracePrintf(AssetBundler::AppWindowName, "Saving Asset List Debug file to ( %s )...\n", debugListFileAbsolutePath.c_str());
}
@@ -2667,9 +2619,9 @@ namespace AssetBundler
AZ_Printf(AppWindowName, "\n");
AZ_Printf(AppWindowName, "Some args in this tool take paths as arguments, and there are two main types:\n");
AZ_Printf(AppWindowName, " \"path\" - This refers to an Engine-Root-Relative path.\n");
AZ_Printf(AppWindowName, " - Example: \"C:\\Lumberyard\\dev\\AutomatedTesting\\test.txt\" can be represented as \"AutomatedTesting\\test.txt\".\n");
AZ_Printf(AppWindowName, " - Example: \"C:\\O3DE\\dev\\SamplesProject\\test.txt\" can be represented as \"SamplesProject\\test.txt\".\n");
AZ_Printf(AppWindowName, " \"cache path\" - This refers to a Cache-Relative path.\n");
AZ_Printf(AppWindowName, " - Example: \"C:\\Lumberyard\\dev\\AutomatedTesting\\Cache\\pc\\animations\\skeletonlist.xml\" is represented as \"animations\\skeletonlist.xml\".\n");
AZ_Printf(AppWindowName, " - Example: \"C:\\O3DE\\dev\\Cache\\SamplesProject\\pc\\samplesproject\\animations\\skeletonlist.xml\" is represented as \"animations\\skeletonlist.xml\".\n");
AZ_Printf(AppWindowName, "\n");
OutputHelpSeeds();
@@ -2706,7 +2658,7 @@ namespace AssetBundler
AZ_Printf(AppWindowName, " --%-25s-Outputs the contents of the Seed List file after performing any specified operations.\n", PrintFlag);
AZ_Printf(AppWindowName, " --%-25s-Specifies the platform(s) referenced by all Seed operations.\n", PlatformArg);
AZ_Printf(AppWindowName, "%-31s---Requires an existing cache of assets for the input platform(s).\n", "");
AZ_Printf(AppWindowName, "%-31s---Defaults to all enabled platforms. Platforms can be changed by modifying AssetProcessorPlatformConfig.ini.\n", "");
AZ_Printf(AppWindowName, "%-31s---Defaults to all enabled platforms. Platforms can be changed by modifying AssetProcessorPlatformConfig.setreg.\n", "");
AZ_Printf(AppWindowName, " --%-25s-Updates the path hints stored in the Seed List file.\n", UpdateSeedPathArg);
AZ_Printf(AppWindowName, " --%-25s-Removes the path hints stored in the Seed List file.\n", RemoveSeedPathArg);
AZ_Printf(AppWindowName, " --%-25s-Allows input file path to still match if the file path case is different than on disk.\n", IgnoreFileCaseFlag);
@@ -2726,10 +2678,10 @@ namespace AssetBundler
AZ_Printf(AppWindowName, " --%-25s-The specified files and all dependencies will be ignored when generating the Asset List file.\n", SkipArg);
AZ_Printf(AppWindowName, "%-31s---Takes in a comma-separated list of either: cache paths to pre-processed assets, or wildcard patterns.\n", "");
AZ_Printf(AppWindowName, " --%-25s-Automatically include all default Seed List files in generated Asset List File.\n", AddDefaultSeedListFilesFlag);
AZ_Printf(AppWindowName, "%-31s---This will include Seed List files for the Lumberyard Engine and all enabled Gems.\n", "");
AZ_Printf(AppWindowName, "%-31s---This will include Seed List files for the Open 3D Engine Engine and all enabled Gems.\n", "");
AZ_Printf(AppWindowName, " --%-25s-Specifies the platform(s) to generate an Asset List file for.\n", PlatformArg);
AZ_Printf(AppWindowName, "%-31s---Requires an existing cache of assets for the input platform(s).\n", "");
AZ_Printf(AppWindowName, "%-31s---Defaults to all enabled platforms. Platforms can be changed by modifying AssetProcessorPlatformConfig.ini.\n", "");
AZ_Printf(AppWindowName, "%-31s---Defaults to all enabled platforms. Platforms can be changed by modifying AssetProcessorPlatformConfig.setreg.\n", "");
AZ_Printf(AppWindowName, " --%-25s-[Testing] Specifies the Asset Catalog file referenced by all Asset List operations.\n", AssetCatalogFileArg);
AZ_Printf(AppWindowName, "%-31s---Designed to be used in Unit Tests.\n", "");
AZ_Printf(AppWindowName, " --%-25s-Outputs the contents of the Asset List file after adding any specified seed files.\n", PrintFlag);
@@ -2793,7 +2745,7 @@ namespace AssetBundler
AZ_Printf(AppWindowName, "%-31s---Leave list blank to just print the final comparison result.\n", "");
AZ_Printf(AppWindowName, " --%-25s-Specifies the platform(s) referenced when determining which Asset List files to compare.\n", PlatformArg);
AZ_Printf(AppWindowName, "%-31s---All input Asset List files must exist for all specified platforms\n", "");
AZ_Printf(AppWindowName, "%-31s---Defaults to all enabled platforms. Platforms can be changed by modifying AssetProcessorPlatformConfig.ini.\n", "");
AZ_Printf(AppWindowName, "%-31s---Defaults to all enabled platforms. Platforms can be changed by modifying AssetProcessorPlatformConfig.setreg.\n", "");
AZ_Printf(AppWindowName, " --%-25s-Allow destructive overwrites of files. Include this arg in automation.\n", AllowOverwritesFlag);
AZ_Printf(AppWindowName, " --%-25s-Specifies the game project to use rather than the current default project set in bootstrap.cfg's project_path.\n", ProjectArg);
}
@@ -2805,11 +2757,11 @@ namespace AssetBundler
AZ_Printf(AppWindowName, " --%-25s-[Required] Specifies the Bundle Settings file to operate on by path. Must include (.%s) file extension.\n", BundleSettingsFileArg, AssetBundleSettings::GetBundleSettingsFileExtension());
AZ_Printf(AppWindowName, " --%-25s-Sets the Asset List file to use for Bundle generation. Must include (.%s) file extension.\n", AssetListFileArg, AssetSeedManager::GetAssetListFileExtension());
AZ_Printf(AppWindowName, " --%-25s-Sets the path where generated Bundles will be stored. Must include (.%s) file extension.\n", OutputBundlePathArg, AssetBundleSettings::GetBundleFileExtension());
AZ_Printf(AppWindowName, " --%-25s-Determines which version of Lumberyard Bundles to generate. Current version is (%i).\n", BundleVersionArg, AzFramework::AssetBundleManifest::CurrentBundleVersion);
AZ_Printf(AppWindowName, " --%-25s-Determines which version of Open 3D Engine Bundles to generate. Current version is (%i).\n", BundleVersionArg, AzFramework::AssetBundleManifest::CurrentBundleVersion);
AZ_Printf(AppWindowName, " --%-25s-Sets the maximum size for a single Bundle (in MB). Default size is (%i MB).\n", MaxBundleSizeArg, AssetBundleSettings::GetMaxBundleSizeInMB());
AZ_Printf(AppWindowName, "%-31s---Bundles larger than this limit will be divided into a series of smaller Bundles and named accordingly.\n", "");
AZ_Printf(AppWindowName, " --%-25s-Specifies the platform(s) referenced by all Bundle Settings operations.\n", PlatformArg);
AZ_Printf(AppWindowName, "%-31s---Defaults to all enabled platforms. Platforms can be changed by modifying AssetProcessorPlatformConfig.ini.\n", "");
AZ_Printf(AppWindowName, "%-31s---Defaults to all enabled platforms. Platforms can be changed by modifying AssetProcessorPlatformConfig.setreg.\n", "");
AZ_Printf(AppWindowName, " --%-25s-Outputs the contents of the Bundle Settings file after modifying any specified values.\n", PrintFlag);
AZ_Printf(AppWindowName, " --%-25s-Specifies the game project to use rather than the current default project set in bootstrap.cfg's project_path.\n", ProjectArg);
}
@@ -2822,7 +2774,7 @@ namespace AssetBundler
AZ_Printf(AppWindowName, "%-31s---If any other args are specified, they will override the values stored inside this file.\n", "");
AZ_Printf(AppWindowName, " --%-25s-Sets the Asset List files to use for Bundle generation. Must include (.%s) file extension.\n", AssetListFileArg, AssetSeedManager::GetAssetListFileExtension());
AZ_Printf(AppWindowName, " --%-25s-Sets the paths where generated Bundles will be stored. Must include (.%s) file extension.\n", OutputBundlePathArg, AssetBundleSettings::GetBundleFileExtension());
AZ_Printf(AppWindowName, " --%-25s-Determines which versions of Lumberyard Bundles to generate. Current version is (%i).\n", BundleVersionArg, AzFramework::AssetBundleManifest::CurrentBundleVersion);
AZ_Printf(AppWindowName, " --%-25s-Determines which versions of Open 3D Engine Bundles to generate. Current version is (%i).\n", BundleVersionArg, AzFramework::AssetBundleManifest::CurrentBundleVersion);
AZ_Printf(AppWindowName, " --%-25s-Sets the maximum size for Bundles (in MB). Default size is (%i MB).\n", MaxBundleSizeArg, AssetBundleSettings::GetMaxBundleSizeInMB());
AZ_Printf(AppWindowName, "%-31s---Bundles larger than this limit will be divided into a series of smaller Bundles and named accordingly.\n", "");
AZ_Printf(AppWindowName, " --%-25s-Specifies the platform(s) that will be referenced when generating Bundles.\n", PlatformArg);
@@ -2839,7 +2791,7 @@ namespace AssetBundler
AZ_Printf(AppWindowName, "%-31s---Takes in a cache path to a pre-processed asset. A cache path is a path relative to \"ProjectPath\\Cache\\platform\\\"\n", "");
AZ_Printf(AppWindowName, " --%-25s-Specifies the Bundle Settings file to operate on by path. Must include (.%s) file extension.\n", BundleSettingsFileArg, AssetBundleSettings::GetBundleSettingsFileExtension());
AZ_Printf(AppWindowName, " --%-25s-Sets the path where generated Bundles will be stored. Must include (.%s) file extension.\n", OutputBundlePathArg, AssetBundleSettings::GetBundleFileExtension());
AZ_Printf(AppWindowName, " --%-25s-Determines which version of Lumberyard Bundles to generate. Current version is (%i).\n", BundleVersionArg, AzFramework::AssetBundleManifest::CurrentBundleVersion);
AZ_Printf(AppWindowName, " --%-25s-Determines which version of Open 3D Engine Bundles to generate. Current version is (%i).\n", BundleVersionArg, AzFramework::AssetBundleManifest::CurrentBundleVersion);
AZ_Printf(AppWindowName, " --%-25s-Sets the maximum size for a single Bundle (in MB). Default size is (%i MB).\n", MaxBundleSizeArg, AssetBundleSettings::GetMaxBundleSizeInMB());
AZ_Printf(AppWindowName, "%-31s---Bundles larger than this limit will be divided into a series of smaller Bundles and named accordingly.\n", "");
AZ_Printf(AppWindowName, " --%-25s-Specifies the platform(s) that will be referenced when generating Bundles.\n", PlatformArg);
+24 -45
View File
@@ -47,7 +47,7 @@ namespace AssetBundler
const char* AssetCatalogFileArg = "overrideAssetCatalogFile";
const char* AllowOverwritesFlag = "allowOverwrites";
const char* IgnoreFileCaseFlag = "ignoreFileCase";
const char* ProjectArg = "project";
const char* ProjectArg = "project-path";
// Seeds
const char* SeedsCommand = "seeds";
@@ -485,7 +485,7 @@ namespace AssetBundler
return AZ::Failure(AZStd::string::format(
"Unable to locate the Project Cache path from Settings Registry at key %s."
" Please run the Lumberyard Asset Processor to generate a Cache and build assets.",
" Please run the Open 3D Engine Asset Processor to generate a Cache and build assets.",
AZ::SettingsRegistryMergeUtils::FilePathKey_CacheProjectRootFolder));
}
@@ -503,7 +503,7 @@ namespace AssetBundler
if (tempPlatformList.empty())
{
return AZ::Failure(AZStd::string("Cache is empty. Please run the Lumberyard Asset Processor to generate a Cache and build assets."));
return AZ::Failure(AZStd::string("Cache is empty. Please run the Open 3D Engine Asset Processor to generate a Cache and build assets."));
}
for (const QString& platform : tempPlatformList)
@@ -520,7 +520,7 @@ namespace AssetBundler
if (assetCatalogFilePath.empty())
{
return AZ::Failure(AZStd::string::format(
"Unable to retrieve cache platform path from Settings Registry at key: %s. Please run the Lumberyard Asset Processor to generate platform-specific cache folders and build assets.",
"Unable to retrieve cache platform path from Settings Registry at key: %s. Please run the Open 3D Engine Asset Processor to generate platform-specific cache folders and build assets.",
AZ::SettingsRegistryMergeUtils::FilePathKey_CacheProjectRootFolder));
}
@@ -607,9 +607,9 @@ namespace AssetBundler
if (!filePath.empty())
{
m_validPath = true;
m_originalPath = m_absolutePath = filePath;
AzFramework::StringFunc::Path::Normalize(m_originalPath);
ComputeAbsolutePath(m_absolutePath, platform, checkFileCase, ignoreFileCase);
m_absolutePath = AZ::IO::PathView(filePath).LexicallyNormal();
m_originalPath = m_absolutePath;
ComputeAbsolutePath(platform, checkFileCase, ignoreFileCase);
}
}
@@ -621,12 +621,12 @@ namespace AssetBundler
const AZStd::string& FilePath::AbsolutePath() const
{
return m_absolutePath;
return m_absolutePath.Native();
}
const AZStd::string& FilePath::OriginalPath() const
{
return m_originalPath;
return m_originalPath.Native();
}
bool FilePath::IsValid() const
@@ -639,60 +639,37 @@ namespace AssetBundler
return m_errorString;
}
void FilePath::ComputeAbsolutePath(AZStd::string& filePath, const AZStd::string& platformIdentifier, bool checkFileCase, bool ignoreFileCase)
void FilePath::ComputeAbsolutePath(const AZStd::string& platformIdentifier, bool checkFileCase, bool ignoreFileCase)
{
if (AzToolsFramework::AssetFileInfoListComparison::IsTokenFile(filePath))
if (AzToolsFramework::AssetFileInfoListComparison::IsTokenFile(m_absolutePath.Native()))
{
return;
}
if (!platformIdentifier.empty())
{
AssetBundler::AddPlatformIdentifier(filePath, platformIdentifier);
}
const char* appRoot = nullptr;
AzFramework::ApplicationRequests::Bus::BroadcastResult(appRoot, &AzFramework::ApplicationRequests::GetAppRoot);
#if AZ_TRAIT_OS_USE_WINDOWS_FILE_PATHS
AZStd::string driveString;
AzFramework::StringFunc::Path::GetDrive(appRoot, driveString);
if (AzFramework::StringFunc::FirstCharacter(filePath.c_str()) == AZ_CORRECT_FILESYSTEM_SEPARATOR)
{
AzFramework::StringFunc::Path::ConstructFull(driveString.c_str(), filePath.c_str(), filePath, true);
}
#endif
if (!AzFramework::StringFunc::Path::IsRelative(filePath.c_str()))
{
// it is already an absolute path
AzFramework::StringFunc::Path::Normalize(filePath);
}
else
{
AzFramework::StringFunc::Path::ConstructFull(appRoot, m_absolutePath.c_str(), m_absolutePath, true);
AssetBundler::AddPlatformIdentifier(m_absolutePath.Native(), platformIdentifier);
}
AZ::IO::Path enginePath = AZ::IO::PathView(AZ::Utils::GetEnginePath());
m_absolutePath = enginePath / m_absolutePath;
if (checkFileCase)
{
QDir rootDir(appRoot);
QString relFilePath = rootDir.relativeFilePath(m_absolutePath.c_str());
if (AzToolsFramework::AssetUtils::UpdateFilePathToCorrectCase(QString(appRoot), relFilePath))
AZ::IO::Path relFilePath = m_absolutePath.LexicallyProximate(enginePath);
if (AzToolsFramework::AssetUtils::UpdateFilePathToCorrectCase(enginePath.Native(), relFilePath.Native()))
{
if (ignoreFileCase)
{
AzFramework::StringFunc::Path::ConstructFull(appRoot, relFilePath.toUtf8().data(), m_absolutePath, true);
m_absolutePath = (enginePath / relFilePath).String();
}
else
{
AZStd::string absfilePath(rootDir.filePath(relFilePath).toUtf8().data());
AzFramework::StringFunc::Path::Normalize(absfilePath);
if (!AZ::StringFunc::Equal(absfilePath.c_str(), m_absolutePath.c_str(), true))
AZ::IO::Path absfilePath = (enginePath / relFilePath).LexicallyNormal();
if (absfilePath != AZ::IO::PathView(m_absolutePath))
{
m_errorString = AZStd::string::format("File case mismatch, file ( %s ) does not exist on disk, did you mean file ( %s ). \
Please run the command again with the correct file path or use ( --%s ) arg if you want to allow case insensitive file match.\n",
m_absolutePath.c_str(), rootDir.filePath(relFilePath.toUtf8().data()).toUtf8().data(), IgnoreFileCaseFlag);
m_errorString = AZStd::string::format("File case mismatch, file ( %s ) does not exist on disk, did you mean file ( %s )."
" Please run the command again with the correct file path or use ( --%s ) arg if you want to allow case insensitive file match.\n",
m_absolutePath.c_str(), absfilePath.c_str(), IgnoreFileCaseFlag);
m_validPath = false;
}
}
@@ -731,10 +708,12 @@ m_absolutePath.c_str(), rootDir.filePath(relFilePath.toUtf8().data()).toUtf8().d
void ScopedTraceHandler::ReportErrors()
{
m_reportingError = true;
#if defined(AZ_ENABLE_TRACING)
for (const AZStd::string& error : m_errors)
{
AZ_Error(AssetBundler::AppWindowName, false, error.c_str());
}
#endif
ClearErrors();
m_reportingError = false;
+4 -4
View File
@@ -14,7 +14,7 @@
#include <AzCore/std/string/string.h>
#include <AzCore/Debug/TraceMessageBus.h>
#include <AzCore/IO/SystemFile.h> //AZ_MAX_PATH_LEN
#include <AzCore/IO/Path/Path.h>
#include <AzCore/Outcome/Outcome.h>
#include <AzFramework/Platform/PlatformDefaults.h>
#include <AzToolsFramework/Asset/AssetBundler.h>
@@ -246,10 +246,10 @@ namespace AssetBundler
AZStd::string ErrorString() const;
bool IsValid() const;
private:
void ComputeAbsolutePath(AZStd::string& filePath, const AZStd::string& platformIdentifier, bool checkFileCase, bool ignoreFileCase);
void ComputeAbsolutePath(const AZStd::string& platformIdentifier, bool checkFileCase, bool ignoreFileCase);
AZStd::string m_absolutePath;
AZStd::string m_originalPath;
AZ::IO::Path m_absolutePath;
AZ::IO::Path m_originalPath;
AZStd::string m_errorString;
bool m_validPath = false;
};
+51 -35
View File
@@ -12,8 +12,9 @@
#include <AzFramework/API/ApplicationAPI.h>
#include <source/utils/utils.h>
#include <AzCore/Settings/SettingsRegistryImpl.h>
#include <AzCore/Settings/SettingsRegistryMergeUtils.h>
#include <AzCore/UnitTest/TestTypes.h>
#include <AzFramework/StringFunc/StringFunc.h>
#include <Utils/Utils.h>
#include <AzFramework/IO/LocalFileIO.h>
@@ -36,10 +37,27 @@ namespace AssetBundler
AZ::IO::FileIOBase::SetInstance(nullptr);
AZ::IO::FileIOBase::SetInstance(m_localFileIO);
m_tempDir = new UnitTest::ScopedTemporaryDirectory();
auto settingsRegistry = AZ::SettingsRegistry::Get();
if (settingsRegistry == nullptr)
{
m_settingsRegistry = AZStd::make_unique<AZ::SettingsRegistryImpl>();
settingsRegistry = m_settingsRegistry.get();
AZ::SettingsRegistry::Register(settingsRegistry);
}
settingsRegistry->Get(m_oldEngineRoot.Native(), AZ::SettingsRegistryMergeUtils::FilePathKey_EngineRootFolder);
settingsRegistry->Set(AZ::SettingsRegistryMergeUtils::FilePathKey_EngineRootFolder, m_tempDir->GetDirectory());
}
void TearDown() override
{
// Reset Engine Path if there was an existing Settings Registry from before
auto settingsRegistry = AZ::SettingsRegistry::Get();
settingsRegistry->Set(AZ::SettingsRegistryMergeUtils::FilePathKey_EngineRootFolder, m_oldEngineRoot.Native());
if(settingsRegistry == m_settingsRegistry.get())
{
AZ::SettingsRegistry::Unregister(settingsRegistry);
m_settingsRegistry.reset();
}
delete m_tempDir;
AZ::IO::FileIOBase::SetInstance(nullptr);
delete m_localFileIO;
@@ -53,7 +71,7 @@ namespace AssetBundler
void NormalizePathKeepCase(AZStd::string& /*path*/) override {}
void CalculateBranchTokenForEngineRoot(AZStd::string& /*token*/) const override {}
const char* GetAppRoot() const override
const char* GetEngineRoot() const override
{
return m_tempDir->GetDirectory();
}
@@ -61,56 +79,55 @@ namespace AssetBundler
AZ::IO::FileIOBase* m_priorFileIO = nullptr;
AZ::IO::FileIOBase* m_localFileIO = nullptr;
UnitTest::ScopedTemporaryDirectory* m_tempDir = nullptr;
AZStd::unique_ptr<AZ::SettingsRegistryInterface> m_settingsRegistry;
AZ::IO::Path m_oldEngineRoot;
};
TEST_F(MockUtilsTest, DISABLED_TestFilePath_StartsWithAFileSeparator_Valid)
{
AZStd::string relFilePath = "Foo/foo.xml";
AzFramework::StringFunc::Prepend(relFilePath, AZ_CORRECT_FILESYSTEM_SEPARATOR);
AZStd::string absoluteFilePath;
#if AZ_TRAIT_OS_USE_WINDOWS_FILE_PATHS
AZStd::string driveString;
AzFramework::StringFunc::Path::GetDrive(GetAppRoot(), driveString);
AzFramework::StringFunc::Path::ConstructFull(driveString.c_str(), relFilePath.c_str(), absoluteFilePath, true);
#else
absoluteFilePath = relFilePath;
#endif
FilePath filePath(relFilePath);
AZ::IO::Path relFilePath = "Foo/foo.xml";
AZ::IO::Path absoluteFilePath = AZ::IO::PathView(GetEngineRoot()).RootPath();
absoluteFilePath /= relFilePath;
absoluteFilePath = absoluteFilePath.LexicallyNormal();
FilePath filePath(relFilePath.Native());
EXPECT_STREQ(filePath.AbsolutePath().c_str(), absoluteFilePath.c_str());
}
TEST_F(MockUtilsTest, TestFilePath_RelativePath_Valid)
{
AZStd::string relFilePath = "Foo\\foo.xml";
AZStd::string absoluteFilePath;
AzFramework::StringFunc::Path::ConstructFull(GetAppRoot(), relFilePath.c_str(), absoluteFilePath, true);
FilePath filePath(relFilePath);
EXPECT_EQ(filePath.AbsolutePath(), absoluteFilePath);
AZ::IO::Path relFilePath = "Foo\\foo.xml";
AZ::IO::Path absoluteFilePath = (AZ::IO::Path(GetEngineRoot()) / relFilePath).LexicallyNormal();
FilePath filePath(relFilePath.Native());
EXPECT_EQ(AZ::IO::PathView{ filePath.AbsolutePath() }, absoluteFilePath);
}
#if !AZ_TRAIT_USE_WINDOWS_FILE_API
// When using Windows file API the the AZ::IO::Path comparisons are case insensitive
TEST_F(MockUtilsTest, TestFilePath_CasingMismatch_Error_valid)
{
AZStd::string relFilePath = "Foo\\Foo.xml";
AZStd::string wrongCaseRelFilePath = "Foo\\foo.xml";
AZStd::string correctAbsoluteFilePath;
AZStd::string wrongCaseAbsoluteFilePath;
AzFramework::StringFunc::Path::ConstructFull(GetAppRoot(), relFilePath.c_str(), correctAbsoluteFilePath, true);
AzFramework::StringFunc::Path::ConstructFull(GetAppRoot(), wrongCaseRelFilePath.c_str(), wrongCaseAbsoluteFilePath, true);
AZ::IO::Path relFilePath = "Foo\\Foo.xml";
AZ::IO::Path wrongCaseRelFilePath = "Foo\\foo.xml";
AZ::IO::Path correctAbsoluteFilePath = (AZ::IO::Path(GetEngineRoot()) / relFilePath).LexicallyNormal();
AZ::IO::Path wrongCaseAbsoluteFilePath = (AZ::IO::Path(GetEngineRoot()) / wrongCaseRelFilePath).LexicallyNormal();
AZ::IO::HandleType fileHandle = AZ::IO::InvalidHandle;
AZ::IO::FileIOBase::GetInstance()->Open(correctAbsoluteFilePath.c_str(), AZ::IO::OpenMode::ModeWrite | AZ::IO::OpenMode::ModeCreatePath, fileHandle);
FilePath filePath(wrongCaseAbsoluteFilePath, true, false);
FilePath filePath(wrongCaseAbsoluteFilePath.Native(), true, false);
EXPECT_FALSE(filePath.IsValid());
EXPECT_TRUE(filePath.ErrorString().find("File case mismatch") != AZStd::string::npos);
EXPECT_TRUE(filePath.ErrorString().contains("File case mismatch"));
}
#endif
TEST_F(MockUtilsTest, TestFilePath_NoFileExists_NoError_valid)
{
AZStd::string relFilePath = "Foo\\Foo.xml";
AZStd::string absoluteFilePath;
AzFramework::StringFunc::Path::ConstructFull(GetAppRoot(), relFilePath.c_str(), absoluteFilePath, true);
FilePath filePath(absoluteFilePath, true, false);
AZ::IO::Path relFilePath = "Foo\\Foo.xml";
AZ::IO::Path absoluteFilePath = (AZ::IO::Path(GetEngineRoot()) / relFilePath).LexicallyNormal();
FilePath filePath(absoluteFilePath.Native(), true, false);
EXPECT_TRUE(filePath.IsValid());
EXPECT_TRUE(filePath.ErrorString().empty());
}
@@ -119,13 +136,12 @@ namespace AssetBundler
{
AZStd::string relFilePath = "Foo\\Foo.xml";
AZStd::string wrongCaseRelFilePath = "Foo\\foo.xml";
AZStd::string correctAbsoluteFilePath;
AZStd::string wrongCaseAbsoluteFilePath;
AzFramework::StringFunc::Path::ConstructFull(GetAppRoot(), relFilePath.c_str(), correctAbsoluteFilePath, true);
AzFramework::StringFunc::Path::ConstructFull(GetAppRoot(), wrongCaseRelFilePath.c_str(), wrongCaseAbsoluteFilePath, true);
AZ::IO::Path correctAbsoluteFilePath = (AZ::IO::Path(GetEngineRoot()) / relFilePath).LexicallyNormal();
AZ::IO::Path wrongCaseAbsoluteFilePath = (AZ::IO::Path(GetEngineRoot()) / wrongCaseRelFilePath).LexicallyNormal();
AZ::IO::HandleType fileHandle = AZ::IO::InvalidHandle;
AZ::IO::FileIOBase::GetInstance()->Open(correctAbsoluteFilePath.c_str(), AZ::IO::OpenMode::ModeWrite | AZ::IO::OpenMode::ModeCreatePath, fileHandle);
FilePath filePath(wrongCaseAbsoluteFilePath, true, true);
FilePath filePath(wrongCaseAbsoluteFilePath.Native(), true, true);
EXPECT_TRUE(filePath.IsValid());
EXPECT_STREQ(filePath.AbsolutePath().c_str(), correctAbsoluteFilePath.c_str());
}
@@ -0,0 +1,3 @@
version https://git-lfs.github.com/spec/v1
oid sha256:5bb66cade9f4f42c6db0a4e43f0c2be87b185d788353330c4165ace4f8c44289
size 107101
@@ -0,0 +1 @@
IDI_ICON1 ICON DISCARDABLE "AssetBuilder.ico"
@@ -19,4 +19,5 @@ set(FILES
AssetBuilderInfo.cpp
TraceMessageHook.h
TraceMessageHook.cpp
AssetBuilder.rc
)
+1 -3
View File
@@ -18,8 +18,6 @@ endif()
add_subdirectory(AssetBuilderSDK)
add_subdirectory(AssetBuilder)
include(${CMAKE_CURRENT_SOURCE_DIR}/Platform/${PAL_PLATFORM_NAME}/PAL_${PAL_PLATFORM_NAME_LOWERCASE}.cmake)
ly_add_target(
NAME AssetProcessor.Static STATIC
NAMESPACE AZ
@@ -63,7 +61,7 @@ ly_add_source_properties(
)
ly_add_target(
NAME AssetProcessor ${PAL_TRAIT_BUILD_ASSETPROCESSOR_APPLICATION_TYPE}
NAME AssetProcessor APPLICATION
NAMESPACE AZ
AUTOMOC
AUTOUIC
@@ -11,5 +11,5 @@
*/
#pragma once
#define ASSETPROCESSOR_TRAIT_LEGACY_RC_RELATIVE_PATH "/rc"
#define ASSETPROCESSOR_TRAIT_CASE_SENSITIVE_FILESYSTEM true
#define ASSETPROCESSOR_TRAIT_LEGACY_RC_RELATIVE_PATH "rc"
#define ASSETPROCESSOR_TRAIT_CASE_SENSITIVE_FILESYSTEM true
@@ -11,5 +11,5 @@
*/
#pragma once
#define ASSETPROCESSOR_TRAIT_LEGACY_RC_RELATIVE_PATH "/rc"
#define ASSETPROCESSOR_TRAIT_CASE_SENSITIVE_FILESYSTEM false
#define ASSETPROCESSOR_TRAIT_LEGACY_RC_RELATIVE_PATH "rc"
#define ASSETPROCESSOR_TRAIT_CASE_SENSITIVE_FILESYSTEM false
@@ -1,13 +0,0 @@
#
# All or portions of this file Copyright (c) Amazon.com, Inc. or its affiliates or
# its licensors.
#
# For complete copyright and license terms please see the LICENSE at the root of this
# distribution (the "License"). All use of this software is governed by the License,
# or, if provided, by the license below or the license accompanying this file. Do not
# remove or modify any license notices. This file is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
#
set (PAL_TRAIT_BUILD_ASSETPROCESSOR_APPLICATION_TYPE APPLICATION)
@@ -1 +1 @@
IDI_ICON1 ICON DISCARDABLE "../../native/ui/style/lyassetprocessor.ico"
IDI_ICON1 ICON DISCARDABLE "../../native/ui/style/o3de_assetprocessor.ico"
@@ -0,0 +1,3 @@
version https://git-lfs.github.com/spec/v1
oid sha256:c05ee28bfca3a0411afd13fb2b5fcecd5ca9b5e74627e5f6837d3c9cea25b715
size 107497
@@ -0,0 +1 @@
IDI_ICON1 ICON DISCARDABLE "AssetProcessorBatch.ico"
@@ -11,5 +11,5 @@
*/
#pragma once
#define ASSETPROCESSOR_TRAIT_LEGACY_RC_RELATIVE_PATH "/rc.exe"
#define ASSETPROCESSOR_TRAIT_CASE_SENSITIVE_FILESYSTEM false
#define ASSETPROCESSOR_TRAIT_LEGACY_RC_RELATIVE_PATH "rc.exe"
#define ASSETPROCESSOR_TRAIT_CASE_SENSITIVE_FILESYSTEM false
@@ -1,13 +0,0 @@
#
# All or portions of this file Copyright (c) Amazon.com, Inc. or its affiliates or
# its licensors.
#
# For complete copyright and license terms please see the LICENSE at the root of this
# distribution (the "License"). All use of this software is governed by the License,
# or, if provided, by the license below or the license accompanying this file. Do not
# remove or modify any license notices. This file is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
#
set (PAL_TRAIT_BUILD_ASSETPROCESSOR_APPLICATION_TYPE APPLICATION)
@@ -12,4 +12,5 @@
set(FILES
native/main_batch.cpp
native/AssetProcessorBatchBuildTarget.cpp
Platform/Windows/AssetProcessorBatch.rc
)
@@ -2261,8 +2261,6 @@ namespace AssetProcessor
{
// note, intentional scope created for the statement finalizer
const char* statementToUse = wasAlreadyInDatabase ? UPDATE_PRODUCT : INSERT_PRODUCT;
StatementAutoFinalizer autoFinalizer;
if (wasAlreadyInDatabase)
{
@@ -2285,7 +2283,7 @@ namespace AssetProcessor
if(statement->Step() == Statement::SqlError)
{
AZ_Error(LOG_NAME, false, "Failed to execute the %s statement", statementToUse);
AZ_Error(LOG_NAME, false, "Failed to execute the %s statement", wasAlreadyInDatabase ? UPDATE_PRODUCT : INSERT_PRODUCT);
return false;
}
@@ -36,7 +36,7 @@ namespace AssetProcessor
m_platforms.push_back(QString::fromUtf8(info.m_identifier.c_str()));
}
bool computedCacheRoot = AssetUtilities::ComputeProjectCacheRoot(m_cacheRoot);
[[maybe_unused]] bool computedCacheRoot = AssetUtilities::ComputeProjectCacheRoot(m_cacheRoot);
AZ_Assert(computedCacheRoot, "Could not compute cache root for AssetCatalog");
// save 30mb for this. Really large projects do get this big (and bigger)
@@ -341,10 +341,10 @@ namespace AssetProcessor
{
auto settingsRegistry = AZ::SettingsRegistry::Get();
AZ::SettingsRegistryInterface::FixedValueString cacheRootFolder;
settingsRegistry->Get(cacheRootFolder, AZ::SettingsRegistryMergeUtils::FilePathKey_CacheRootFolder);
settingsRegistry->Get(cacheRootFolder, AZ::SettingsRegistryMergeUtils::FilePathKey_CacheProjectRootFolder);
QString tempRegistryFile = QString("%1/%2").arg(workSpace).arg("assetcatalog.xml.tmp");
QString platformCacheDir = QString::fromUtf8(cacheRootFolder.c_str(), aznumeric_cast<int>(cacheRootFolder.size()));
QString platformCacheDir = QString("%1/%2").arg(cacheRootFolder.c_str()).arg(platform);
QString actualRegistryFile = QString("%1/%2").arg(platformCacheDir).arg("assetcatalog.xml");
AZ_TracePrintf(AssetProcessor::DebugChannel, "Creating asset catalog: %s --> %s\n", tempRegistryFile.toUtf8().constData(), actualRegistryFile.toUtf8().constData());
@@ -359,7 +359,7 @@ namespace AssetProcessor
if (!registryDir.exists())
{
QString absPath = registryDir.absolutePath();
bool makeDirResult = AZ::IO::SystemFile::CreateDir(absPath.toUtf8().constData());
[[maybe_unused]] bool makeDirResult = AZ::IO::SystemFile::CreateDir(absPath.toUtf8().constData());
AZ_Warning(AssetProcessor::ConsoleChannel, makeDirResult, "Failed create folder %s", platformCacheDir.toUtf8().constData());
}
@@ -739,8 +739,6 @@ namespace AssetProcessor
QMutexLocker locker(&m_registriesMutex);
const auto& productDependencies = m_registries[platform].GetAssetDependencies(id);
auto itr = m_registries[platform].m_assetDependencies.find(id);
if (itr == m_registries[platform].m_assetDependencies.end())
@@ -149,7 +149,10 @@ namespace AssetProcessor
{
return normalized.toLower();
}
return normalized;
else
{
return normalized;
}
}
void FileStateCache::AddOrUpdateFileInternal(QFileInfo fileInfo)
@@ -476,7 +476,9 @@ namespace AssetProcessor
// CheckDeletedSourceFile actually expects the database name as the second value
// iter.key is the full path normalized. iter.value is the database path.
// we need the relative path too, which involves removing the scan folder outputprefix it present:
CheckDeletedSourceFile(iter.key(), iter.value().m_sourceRelativeToWatchFolder, iter.value().m_sourceDatabaseName);
CheckDeletedSourceFile(
iter.key(), iter.value().m_sourceRelativeToWatchFolder, iter.value().m_sourceDatabaseName,
AZStd::chrono::system_clock::now());
}
// we want to remove any left over scan folders from the database only after
@@ -1524,7 +1526,7 @@ namespace AssetProcessor
// even if the entry already exists,
// overwrite the entry here, so if you modify, then delete it, its the latest action thats always on the list.
m_filesToExamine[normalizedFilePath] = FileEntry(normalizedFilePath, source.m_isDelete, source.m_isFromScanner);
m_filesToExamine[normalizedFilePath] = FileEntry(normalizedFilePath, source.m_isDelete, source.m_isFromScanner, source.m_initialProcessTime);
// this block of code adds anything which DEPENDS ON the file that was changed, back into the queue so that files
// that depend on it also re-analyze in case they need rebuilding. However, files that are deleted will be added
@@ -1773,40 +1775,31 @@ namespace AssetProcessor
return successfullyRemoved;
}
void AssetProcessorManager::CheckDeletedSourceFile(QString normalizedPath, QString relativePath, QString databaseSourceFile)
void AssetProcessorManager::CheckDeletedSourceFile(QString normalizedPath, QString relativePath, QString databaseSourceFile,
AZStd::chrono::system_clock::time_point initialProcessTime)
{
// getting here means an input asset has been deleted
// and no overrides exist for it.
// we must delete its products.
using namespace AzToolsFramework::AssetDatabase;
// If we fail to delete a product, the deletion event gets requeued
// To avoid retrying forever, we keep track of the time of the first deletion failure and only retry
// if less than this amount of time has passed.
constexpr int MaxRetryPeriodMS = 500;
AZStd::chrono::duration<double, AZStd::milli> duration = AZStd::chrono::system_clock::now() - initialProcessTime;
// Check if this file causes any file types to be re-evaluated
CheckMetaDataRealFiles(normalizedPath);
// when a source is deleted, we also have to queue anything that depended on it, for re-processing:
SourceFileDependencyEntryContainer results;
m_stateData->GetSourceFileDependenciesByDependsOnSource(databaseSourceFile, SourceFileDependencyEntry::DEP_Any, results);
// the jobIdentifiers that have identified it as a job dependency
for (SourceFileDependencyEntry& existingEntry : results)
if (initialProcessTime > AZStd::chrono::system_clock::time_point{}
&& duration >= AZStd::chrono::milliseconds(MaxRetryPeriodMS))
{
// this row is [Source] --> [Depends on Source].
QString absolutePath = m_platformConfig->FindFirstMatchingFile(QString::fromUtf8(existingEntry.m_source.c_str()));
if (!absolutePath.isEmpty())
{
AssessFileInternal(absolutePath, false);
}
// also, update it in the database to be missing, ie, add the "missing file" prefix:
existingEntry.m_dependsOnSource = QString(PlaceHolderFileName + relativePath).toUtf8().constData();
m_stateData->RemoveSourceFileDependency(existingEntry.m_sourceDependencyID);
m_stateData->SetSourceFileDependency(existingEntry);
AZ_Warning(AssetProcessor::ConsoleChannel, false, "Failed to delete product(s) from source file `%s` after retrying for %fms. Giving up.",
normalizedPath.toUtf8().constData(), duration.count());
return;
}
// now that the right hand column (in terms of [thing] -> [depends on thing]) has been updated, eliminate anywhere its on the left hand side:
results.clear();
m_stateData->GetDependsOnSourceBySource(databaseSourceFile.toUtf8().constData(), SourceFileDependencyEntry::DEP_Any, results);
m_stateData->RemoveSourceFileDependencies(results);
bool deleteFailure = false;
AzToolsFramework::AssetDatabase::SourceDatabaseEntryContainer sources;
if (m_stateData->GetSourcesBySourceName(databaseSourceFile, sources))
{
for (const auto& source : sources)
@@ -1827,7 +1820,13 @@ namespace AssetProcessor
{
// DeleteProducts will make an attempt to retry deleting each product
// We can't just re-queue the whole file with CheckSource because we're deleting bits from the database as we go
AZ_TracePrintf(AssetProcessor::ConsoleChannel, "Delete failed on %s.\n", normalizedPath.toUtf8().constData());
deleteFailure = true;
CheckSource(FileEntry(
normalizedPath, true, false,
initialProcessTime > AZStd::chrono::system_clock::time_point{} ? initialProcessTime
: AZStd::chrono::system_clock::now()));
AZ_TracePrintf(AssetProcessor::ConsoleChannel, "Delete failed on %s. Will retry!\n", normalizedPath.toUtf8().constData());
continue;
}
}
else
@@ -1843,13 +1842,48 @@ namespace AssetProcessor
Q_EMIT JobRemoved(jobInfo);
}
}
// delete the source from the database too since otherwise it believes we have no products.
m_stateData->RemoveSource(source.m_sourceID);
if (!deleteFailure)
{
// delete the source from the database too since otherwise it believes we have no products.
m_stateData->RemoveSource(source.m_sourceID);
}
}
}
if(deleteFailure)
{
return;
}
// Check if this file causes any file types to be re-evaluated
CheckMetaDataRealFiles(normalizedPath);
Q_EMIT SourceDeleted(databaseSourceFile); // note that this removes it from the RC Queue Model, also
// when a source is deleted, we also have to queue anything that depended on it, for re-processing:
SourceFileDependencyEntryContainer results;
m_stateData->GetSourceFileDependenciesByDependsOnSource(databaseSourceFile, SourceFileDependencyEntry::DEP_Any, results);
// the jobIdentifiers that have identified it as a job dependency
for (SourceFileDependencyEntry& existingEntry : results)
{
// this row is [Source] --> [Depends on Source].
QString absolutePath = m_platformConfig->FindFirstMatchingFile(QString::fromUtf8(existingEntry.m_source.c_str()));
if (!absolutePath.isEmpty())
{
AssessFileInternal(absolutePath, false);
}
// also, update it in the database to be missing, ie, add the "missing file" prefix:
existingEntry.m_dependsOnSource = QString(PlaceHolderFileName + relativePath).toUtf8().constData();
m_stateData->RemoveSourceFileDependency(existingEntry.m_sourceDependencyID);
m_stateData->SetSourceFileDependency(existingEntry);
}
// now that the right hand column (in terms of [thing] -> [depends on thing]) has been updated, eliminate anywhere its on the left
// hand side:
results.clear();
m_stateData->GetDependsOnSourceBySource(databaseSourceFile.toUtf8().constData(), SourceFileDependencyEntry::DEP_Any, results);
m_stateData->RemoveSourceFileDependencies(results);
Q_EMIT SourceDeleted(databaseSourceFile); // note that this removes it from the RC Queue Model, also
}
void AssetProcessorManager::AddKnownFoldersRecursivelyForFile(QString fullFile, QString root)
@@ -2568,7 +2602,7 @@ namespace AssetProcessor
jobdetail.m_jobParam[AZ_CRC(AutoFailReasonKey)] = AZStd::string::format(
"Source file ( %s ) contains non ASCII characters.\n"
"Lumberyard currently only supports file paths having ASCII characters and therefore asset processor will not be able to process this file.\n"
"Open 3D Engine currently only supports file paths having ASCII characters and therefore asset processor will not be able to process this file.\n"
"Please rename the source file to fix this error.\n",
normalizedPath.toUtf8().data());
@@ -2641,7 +2675,7 @@ namespace AssetProcessor
AZ::Uuid sourceUUID = AssetUtilities::CreateSafeSourceUUIDFromName(databasePathToFile.toUtf8().data());
AzToolsFramework::AssetSystem::SourceFileNotificationMessage message(AZ::OSString(sourceFile.toUtf8().constData()), AZ::OSString(scanFolderInfo->ScanPath().toUtf8().constData()), AzToolsFramework::AssetSystem::SourceFileNotificationMessage::FileRemoved, sourceUUID);
EBUS_EVENT(AssetProcessor::ConnectionBus, Send, 0, message);
CheckDeletedSourceFile(normalizedPath, relativePathToFile, databasePathToFile);
CheckDeletedSourceFile(normalizedPath, relativePathToFile, databasePathToFile, examineFile.m_initialProcessTime);
}
else
{
@@ -116,13 +116,15 @@ namespace AssetProcessor
QString m_fileName;
bool m_isDelete = false;
bool m_isFromScanner = false;
AZStd::chrono::system_clock::time_point m_initialProcessTime{};
FileEntry() = default;
FileEntry(const QString& fileName, bool isDelete, bool isFromScanner=false)
FileEntry(const QString& fileName, bool isDelete, bool isFromScanner = false, AZStd::chrono::system_clock::time_point initialProcessTime = {})
: m_fileName(fileName)
, m_isDelete(isDelete)
, m_isFromScanner(isFromScanner)
, m_initialProcessTime(initialProcessTime)
{
}
@@ -305,7 +307,9 @@ namespace AssetProcessor
void CheckSource(const FileEntry& source);
void CheckMissingJobs(QString relativeSourceFile, const ScanFolderInfo* scanFolder, const AZStd::vector<JobDetails>& jobsThisTime);
void CheckDeletedProductFile(QString normalizedPath);
void CheckDeletedSourceFile(QString normalizedPath, QString relativePath, QString databaseSourceFile);
void CheckDeletedSourceFile(
QString normalizedPath, QString relativePath, QString databaseSourceFile,
AZStd::chrono::system_clock::time_point initialProcessTime);
void CheckModifiedSourceFile(QString normalizedPath, QString databaseSourceFile, const ScanFolderInfo* scanFolderInfo);
bool AnalyzeJob(JobDetails& details);
void CheckDeletedCacheFolder(QString normalizedPath);
@@ -284,7 +284,7 @@ namespace AssetProcessor
for (const auto& settingsKey : settingsToCopy)
{
AZ::SettingsRegistryInterface::FixedValueString settingsValue;
bool settingsCopied = settingsRegistry->Get(settingsValue, settingsKey)
[[maybe_unused]] bool settingsCopied = settingsRegistry->Get(settingsValue, settingsKey)
&& registry.Set(settingsKey, settingsValue);
AZ_Warning("Settings Registry Builder", settingsCopied, "Unable to copy setting %s from AssetProcessor settings registry"
" to local settings registry", settingsKey.c_str());
@@ -150,7 +150,7 @@ namespace AssetProcessor
return true;
}
bool NativeLegacyRCCompiler::Execute(const QString& inputFile, const QString& watchFolder, const QString& platformIdentifier,
bool NativeLegacyRCCompiler::Execute(const QString& inputFile, const QString& watchFolder, const QString& platformIdentifier,
const QString& params, const QString& dest, const AssetBuilderSDK::JobCancelListener* jobCancelListener, Result& result) const
{
if (!this->m_resourceCompilerInitialized)
@@ -177,7 +177,7 @@ namespace AssetProcessor
AZ_TracePrintf("RC Builder", "Executing RC.EXE: '%s' ...\n", processLaunchInfo.m_commandlineParameters.c_str());
AZ_TracePrintf("Rc Builder", "Executing RC.EXE with working directory: '%s' ...\n", processLaunchInfo.m_workingDirectory.c_str());
AzFramework::ProcessWatcher* watcher = AzFramework::ProcessWatcher::LaunchProcess(processLaunchInfo, AzFramework::ProcessCommunicationType::COMMUNICATOR_TYPE_STDINOUT);
if (!watcher)
@@ -256,7 +256,6 @@ namespace AssetProcessor
QString cmdLine;
if (!dest.isEmpty())
{
QString projectName = AssetUtilities::ComputeProjectName();
QString projectPath = AssetUtilities::ComputeProjectPath();
int portNumber = 0;
@@ -264,12 +263,12 @@ namespace AssetProcessor
AZStd::string appBranchToken;
AzFramework::ApplicationRequests::Bus::Broadcast(&AzFramework::ApplicationRequests::CalculateBranchTokenForEngineRoot, appBranchToken);
cmdLine = QString("\"%1\" /p=%2 %3 /unattended=true /gameroot=\"%4\" /watchfolder=\"%6\" /targetroot=\"%5\" /logprefix=\"%5/\" /port=%7 /gamesubdirectory=\"%8\" /branchtoken=\"%9\"");
cmdLine = cmdLine.arg(inputFile, platformIdentifier, params, projectPath, dest, watchFolder).arg(portNumber).arg(projectName).arg(appBranchToken.c_str());
cmdLine = QString("\"%1\" --platform=%2 %3 --unattended=true --project-path=\"%4\" --watchfolder=\"%6\" --targetroot=\"%5\" --logprefix=\"%5/\" --port=%7 --branchtoken=\"%8\"");
cmdLine = cmdLine.arg(inputFile, platformIdentifier, params, projectPath, dest, watchFolder).arg(portNumber).arg(appBranchToken.c_str());
}
else
{
cmdLine = QString("\"%1\" /p=%2 %3").arg(inputFile, platformIdentifier, params);
cmdLine = QString("\"%1\" --platform=%2 %3").arg(inputFile, platformIdentifier, params);
}
return cmdLine;
}
@@ -365,7 +364,7 @@ namespace AssetProcessor
return static_cast<AZ::u32>(crc);
}
//! Constructor to initialize the internal builders and a general internal builder uuid that is used for bus
//! Constructor to initialize the internal builders and a general internal builder uuid that is used for bus
//! registration. This constructor is helpful for deriving other classes from this builder for purposes like
//! unit testing.
InternalRecognizerBasedBuilder::InternalRecognizerBasedBuilder(QHash<QString, BuilderIdAndName> inputBuilderByIdMap, AZ::Uuid internalBuilderUuid)
@@ -425,12 +424,12 @@ namespace AssetProcessor
bool InternalRecognizerBasedBuilder::FindRC(QString& rcAbsolutePathOut)
{
char executableDirectory[AZ_MAX_PATH_LEN];
if (AZ::Utils::GetExecutableDirectory(executableDirectory, AZStd::size(executableDirectory)) == AZ::Utils::ExecutablePathResult::Success)
AZ::IO::FixedMaxPath executableDirectory = AZ::Utils::GetExecutableDirectory();
executableDirectory /= ASSETPROCESSOR_TRAIT_LEGACY_RC_RELATIVE_PATH;
if (AZ::IO::SystemFile::Exists(executableDirectory.c_str()))
{
rcAbsolutePathOut = QString("%1/%2").arg(executableDirectory).arg(QString(ASSETPROCESSOR_TRAIT_LEGACY_RC_RELATIVE_PATH));
return AZ::IO::SystemFile::Exists(rcAbsolutePathOut.toUtf8().data());
rcAbsolutePathOut = QString::fromUtf8(executableDirectory.c_str(), executableDirectory.Native().size());
return true;
}
return false;
@@ -471,32 +470,32 @@ namespace AssetProcessor
// inside of each such recognizer is a map of [platform] --> options for that platform.
// so visualizing this whole struct in summary might look something like
// "Internal RC Builder" :
// "Internal RC Builder" :
// {
// { <----- list of recognizers for that RC builder starts here
// regex: "*.tif",
// builderUUID : "12345-12354-123145",
// platformSpecsByPlatform :
// {
// builderUUID : "12345-12354-123145",
// platformSpecsByPlatform :
// {
// "pc" : "streaming = 1",
// "ios" : "streaming = 0"
// }
// },
// {
// {
// regex: "*.png",
// builderUUID : "12345-12354-123145",
// platformSpecsByPlatform :
// {
// builderUUID : "12345-12354-123145",
// platformSpecsByPlatform :
// {
// "pc" : "split=1"
// }
// },
// },
// "Internal Copy Builder",
// {
// {
// regex: "*.cfg",
// builderUUID : "12345-12354-123145",
// platformSpecsByPlatform :
// {
// builderUUID : "12345-12354-123145",
// platformSpecsByPlatform :
// {
// "pc" : "copy",
// "ios" : "copy"
// }
@@ -517,7 +516,7 @@ namespace AssetProcessor
for (auto internalAssetRecognizer : *internalRecognizerList)
{
// so referring to the structure explanation above, internalAssetRecognizer is
// so referring to the structure explanation above, internalAssetRecognizer is
// one of those objects that has the regex in it, (along with list of commands to apply per platform)
if (internalAssetRecognizer->m_platformSpecsByPlatform.size() == 0)
{
@@ -560,7 +559,7 @@ namespace AssetProcessor
if (builderInfo.GetType() == BuilderIdAndName::Type::REGISTERED_BUILDER)
{
AssetBuilderSDK::AssetBuilderDesc builderDesc = CreateBuilderDesc(builderId, builderPatterns);
// RC Builder also needs to include its platforms and its RC command lines so that if you change this, the jobs
// are re-evaluated.
size_t currentHash = 0;
@@ -626,7 +625,7 @@ namespace AssetProcessor
}
}
}
}
return foundAny;
@@ -646,11 +645,11 @@ namespace AssetProcessor
QString requestedBuilderID = QString(azBuilderId.c_str());
response.m_result = AssetBuilderSDK::CreateJobsResultCode::Failed;
QDir watchFolder(request.m_watchFolder.c_str());
QString normalizedPath = watchFolder.absoluteFilePath(request.m_sourceFile.c_str());
normalizedPath = AssetUtilities::NormalizeFilePath(normalizedPath);
// Locate recognizers that match the file
InternalRecognizerPointerContainer recognizers;
if (!GetMatchingRecognizers(request.m_enabledPlatforms, normalizedPath, recognizers))
@@ -803,15 +802,15 @@ namespace AssetProcessor
// If the job fails due to a networking issue, we will attempt to retry RetriesForJobNetworkError times
int retryCount = 0;
do
do
{
++retryCount;
ProcessLegacyRCJob(request, rcParam, assetRecognizer->m_productAssetType, jobCancelListener, response);
AZ_Warning("RC Builder", response.m_resultCode != AssetBuilderSDK::ProcessJobResult_NetworkIssue, "RC.exe reported a network connection issue. %s",
AZ_Warning("RC Builder", response.m_resultCode != AssetBuilderSDK::ProcessJobResult_NetworkIssue, "RC.exe reported a network connection issue. %s",
retryCount <= AssetProcessor::RetriesForJobNetworkError ? "Attempting to retry job." : "Maximum retry attempts exceeded, giving up.");
} while (response.m_resultCode == AssetBuilderSDK::ProcessJobResult_NetworkIssue && retryCount <= AssetProcessor::RetriesForJobNetworkError);
}
if (jobCancelListener.IsCancelled())
@@ -873,7 +872,7 @@ namespace AssetProcessor
workDir.removeRecursively();
}
}
bool InternalRecognizerBasedBuilder::SaveProcessJobRequestFile(const char* requestFileDir, const char* requestFileName, const AssetBuilderSDK::ProcessJobRequest& request)
{
AZStd::string finalFullPath;
@@ -886,7 +885,7 @@ namespace AssetProcessor
return true;
}
bool InternalRecognizerBasedBuilder::LoadProcessJobResponseFile(const char* responseFileDir, const char* responseFileName, AssetBuilderSDK::ProcessJobResponse& response, bool& responseLoaded)
{
responseLoaded = false;
@@ -914,7 +913,7 @@ namespace AssetProcessor
return true;
}
void InternalRecognizerBasedBuilder::ProcessLegacyRCJob(const AssetBuilderSDK::ProcessJobRequest& request, QString rcParam,
void InternalRecognizerBasedBuilder::ProcessLegacyRCJob(const AssetBuilderSDK::ProcessJobRequest& request, QString rcParam,
AZ::Uuid productAssetType, const AssetBuilderSDK::JobCancelListener& jobCancelListener, AssetBuilderSDK::ProcessJobResponse& response)
{
// Process this job
@@ -954,7 +953,7 @@ namespace AssetProcessor
response.m_resultCode = AssetBuilderSDK::ProcessJobResult_Failed;
return;
}
if (!responseFromRCCompiler)
{
if(rcResult.m_exitCode != 0)
@@ -1094,7 +1093,7 @@ namespace AssetProcessor
AzFramework::StringFunc::Path::Join(dest.toUtf8().constData(), productName.c_str(), joinedPath);
productName.swap(joinedPath);
}
// update it in the structure to be absolute normalized path.
product.m_productFileName = productName;
@@ -1139,10 +1138,10 @@ namespace AssetProcessor
}
void InternalRecognizerBasedBuilder::ProcessCopyJob(
const AssetBuilderSDK::ProcessJobRequest& request,
AZ::Uuid productAssetType,
bool outputProductDependencies,
const AssetBuilderSDK::JobCancelListener& jobCancelListener,
const AssetBuilderSDK::ProcessJobRequest& request,
AZ::Uuid productAssetType,
bool outputProductDependencies,
const AssetBuilderSDK::JobCancelListener& jobCancelListener,
AssetBuilderSDK::ProcessJobResponse& response)
{
AssetBuilderSDK::JobProduct jobProduct(request.m_fullPath, productAssetType);
@@ -1151,7 +1150,7 @@ namespace AssetProcessor
{
jobProduct.m_dependenciesHandled = true; // Copy jobs are meant to be used for assets that have no dependencies and just need to be copied.
}
response.m_outputProducts.push_back(jobProduct);
response.m_resultCode = AssetBuilderSDK::ProcessJobResult_Success;
@@ -364,10 +364,12 @@ namespace AssetProcessor
if (!results.isEmpty())
{
#if defined(AZ_ENABLE_TRACING)
for (const AssetProcessor::QueueElementID& result : results)
{
AZ_TracePrintf(AssetProcessor::DebugChannel, "OnEscalateJobsBySourceUUID: %s --> %s\n", sourceUuid.ToString<AZStd::string>().c_str(), result.GetInputAssetName().toUtf8().constData());
}
#endif
m_RCQueueSortModel.OnEscalateJobs(escalationList);
}
// do not print a warning out when this fails, its fine for things to escalate jobs as a matter of course just to "make sure" they are escalated
@@ -410,6 +410,7 @@ namespace AssetProcessor
AssetProcessor::SetThreadLocalJobId(builderParams.m_rcJob->GetJobEntry().m_jobRunKey);
AssetUtilities::JobLogTraceListener jobLogTraceListener(builderParams.m_rcJob->m_jobDetails.m_jobEntry);
#if defined(AZ_ENABLE_TRACING)
QString sourceFullPath(builderParams.m_processJobRequest.m_fullPath.c_str());
auto failReason = builderParams.m_processJobRequest.m_jobDescription.m_jobParameters.find(AZ_CRC(AssetProcessor::AutoFailReasonKey));
if (failReason != builderParams.m_processJobRequest.m_jobDescription.m_jobParameters.end())
@@ -456,6 +457,7 @@ namespace AssetProcessor
}
});
}
#endif
// note that this line below is printed out to be consistent with the output from a job that normally failed, so
// applications reading log file will find it.
@@ -481,6 +483,7 @@ namespace AssetProcessor
AssetBuilderSDK::JobCancelListener JobCancelListener(builderParams.m_rcJob->m_jobDetails.m_jobEntry.m_jobRunKey);
result.m_resultCode = AssetBuilderSDK::ProcessJobResult_Failed; // failed by default
#if defined(AZ_ENABLE_TRACING)
auto warningMessage = builderParams.m_processJobRequest.m_jobDescription.m_jobParameters.find(AZ_CRC(AssetProcessor::JobWarningKey));
if (warningMessage != builderParams.m_processJobRequest.m_jobDescription.m_jobParameters.end())
{
@@ -492,6 +495,7 @@ namespace AssetProcessor
AZ_Warning(AssetBuilderSDK::WarningWindow, false, "%s", token.c_str());
}
}
#endif
// create a temporary directory for Builder to work in.
// lets make it as a subdir of a known temp dir
@@ -81,7 +81,7 @@ namespace UnitTests
// Product: "someproduct4.dds" subid: 4
void CreateCoverageTestData()
{
m_data->m_scanFolder = { "c:/lumberyard/dev", "dev", "rootportkey", "" };
m_data->m_scanFolder = { "c:/O3DE/dev", "dev", "rootportkey", "" };
ASSERT_TRUE(m_data->m_connection.SetScanFolder(m_data->m_scanFolder));
m_data->m_sourceFile1 = { m_data->m_scanFolder.m_scanFolderID, "somefile.tif", AZ::Uuid::CreateRandom(), "AnalysisFingerprint1"};
@@ -239,7 +239,7 @@ namespace UnitTests
// we'll create all of those first (except product) before starting the product test.
//add a scanfolder. None of this has to exist in real disk, this is a db test only.
ScanFolderDatabaseEntry scanFolder {"c:/lumberyard/dev", "dev", "rootportkey", ""};
ScanFolderDatabaseEntry scanFolder {"c:/O3DE/dev", "dev", "rootportkey", ""};
EXPECT_TRUE(m_data->m_connection.SetScanFolder(scanFolder));
ASSERT_NE(scanFolder.m_scanFolderID, AzToolsFramework::AssetDatabase::InvalidEntryId);
@@ -278,7 +278,7 @@ namespace UnitTests
// to add a product legitimately you have to have a full chain of primary keys, chain is:
// ScanFolder --> Source --> job --> product.
// we'll create all of those first (except product) before starting the product test.
ScanFolderDatabaseEntry scanFolder{ "c:/lumberyard/dev", "dev", "rootportkey", "" };
ScanFolderDatabaseEntry scanFolder{ "c:/O3DE/dev", "dev", "rootportkey", "" };
ASSERT_TRUE(m_data->m_connection.SetScanFolder(scanFolder));
SourceDatabaseEntry sourceEntry{ scanFolder.m_scanFolderID, "somefile.tif", AZ::Uuid::CreateRandom(), "fingerprint1" };
@@ -323,7 +323,7 @@ namespace UnitTests
// this is actually a very common case (same job id, same subID)
TEST_F(AssetDatabaseTest, SetProduct_SpecificPK_Succeeds_SameSubID_SameJobID)
{
ScanFolderDatabaseEntry scanFolder{ "c:/lumberyard/dev", "dev", "rootportkey", "" };
ScanFolderDatabaseEntry scanFolder{ "c:/O3DE/dev", "dev", "rootportkey", "" };
ASSERT_TRUE(m_data->m_connection.SetScanFolder(scanFolder));
SourceDatabaseEntry sourceEntry{ scanFolder.m_scanFolderID, "somefile.tif", AZ::Uuid::CreateRandom(), "fingerprint1" };
ASSERT_TRUE(m_data->m_connection.SetSource(sourceEntry));
@@ -4540,9 +4540,17 @@ void ModtimeScanningTest::TearDown()
void ModtimeScanningTest::ProcessAssetJobs()
{
m_data->m_productPaths.clear();
for (const auto& processResult : m_data->m_processResults)
{
auto file = QDir(processResult.m_destinationPath).absoluteFilePath(processResult.m_jobEntry.m_databaseSourceName + ".arc1");
m_data->m_productPaths.emplace(
QDir(processResult.m_jobEntry.m_watchFolderPath)
.absoluteFilePath(processResult.m_jobEntry.m_databaseSourceName)
.toUtf8()
.constData(),
file);
// Create the file on disk
ASSERT_TRUE(UnitTestUtils::CreateDummyFile(file, "products."));
@@ -4793,6 +4801,123 @@ TEST_F(ModtimeScanningTest, ModtimeSkipping_ModifyFile_AndThenRevert_ProcessesAg
ExpectWork(2, 2);
}
struct LockedFileTest
: ModtimeScanningTest
, AssetProcessor::ConnectionBus::Handler
{
MOCK_METHOD3(SendRaw, size_t (unsigned, unsigned, const QByteArray&));
MOCK_METHOD3(SendPerPlatform, size_t (unsigned, const AzFramework::AssetSystem::BaseAssetProcessorMessage&, const QString&));
MOCK_METHOD4(SendRawPerPlatform, size_t (unsigned, unsigned, const QByteArray&, const QString&));
MOCK_METHOD2(SendRequest, unsigned (const AzFramework::AssetSystem::BaseAssetProcessorMessage&, const ResponseCallback&));
MOCK_METHOD2(SendResponse, size_t (unsigned, const AzFramework::AssetSystem::BaseAssetProcessorMessage&));
MOCK_METHOD1(RemoveResponseHandler, void (unsigned));
size_t Send(unsigned, const AzFramework::AssetSystem::BaseAssetProcessorMessage&) override
{
if(m_callback)
{
m_callback();
}
return 0;
}
void SetUp() override
{
ModtimeScanningTest::SetUp();
ConnectionBus::Handler::BusConnect(0);
}
void TearDown() override
{
ConnectionBus::Handler::BusDisconnect();
ModtimeScanningTest::TearDown();
}
AZStd::function<void()> m_callback;
};
TEST_F(LockedFileTest, DeleteFile_LockedProduct_DeleteFails)
{
auto theFile = m_data->m_absolutePath[1].toUtf8();
const char* theFileString = theFile.constData();
auto [sourcePath, productPath] = *m_data->m_productPaths.find(theFileString);
{
QFile file(theFileString);
file.remove();
}
ASSERT_GT(m_data->m_productPaths.size(), 0);
QFile product(productPath);
ASSERT_TRUE(product.open(QIODevice::ReadOnly));
// Check if we can delete the file now, if we can't, proceed with the test
// If we can, it means the OS running this test doesn't lock open files so there's nothing to test
if (!AZ::IO::SystemFile::Delete(productPath.toUtf8().constData()))
{
QMetaObject::invokeMethod(
m_assetProcessorManager.get(), "AssessDeletedFile", Qt::QueuedConnection, Q_ARG(QString, QString(theFileString)));
EXPECT_TRUE(BlockUntilIdle(5000));
EXPECT_TRUE(QFile::exists(productPath));
EXPECT_EQ(m_data->m_deletedSources.size(), 0);
}
else
{
SUCCEED() << "Skipping test. OS does not lock open files.";
}
}
TEST_F(LockedFileTest, DeleteFile_LockedProduct_DeletesWhenReleased)
{
auto theFile = m_data->m_absolutePath[1].toUtf8();
const char* theFileString = theFile.constData();
auto [sourcePath, productPath] = *m_data->m_productPaths.find(theFileString);
{
QFile file(theFileString);
file.remove();
}
ASSERT_GT(m_data->m_productPaths.size(), 0);
QFile product(productPath);
ASSERT_TRUE(product.open(QIODevice::ReadOnly));
// Check if we can delete the file now, if we can't, proceed with the test
// If we can, it means the OS running this test doesn't lock open files so there's nothing to test
if (!AZ::IO::SystemFile::Delete(productPath.toUtf8().constData()))
{
AZStd::thread workerThread;
m_callback = [&product, &workerThread]() {
workerThread = AZStd::thread([&product]() {
AZStd::this_thread::sleep_for(AZStd::chrono::milliseconds(60));
product.close();
});
};
QMetaObject::invokeMethod(
m_assetProcessorManager.get(), "AssessDeletedFile", Qt::QueuedConnection, Q_ARG(QString, QString(theFileString)));
EXPECT_TRUE(BlockUntilIdle(5000));
EXPECT_FALSE(QFile::exists(productPath));
EXPECT_EQ(m_data->m_deletedSources.size(), 1);
workerThread.join();
}
else
{
SUCCEED() << "Skipping test. OS does not lock open files.";
}
}
TEST_F(ModtimeScanningTest, ModtimeSkipping_ModifyFilesSameHash_BothProcess)
{
using namespace AzToolsFramework::AssetSystem;
@@ -174,6 +174,7 @@ struct ModtimeScanningTest
QString m_relativePathFromWatchFolder[3];
AZStd::vector<QString> m_absolutePath;
AZStd::vector<AssetProcessor::JobDetails> m_processResults;
AZStd::unordered_multimap<AZStd::string, QString> m_productPaths;
AZStd::vector<QString> m_deletedSources;
AZStd::shared_ptr<AssetProcessor::InternalMockBuilder> m_builderTxtBuilder;
MockBuilderInfoHandler m_mockBuilderInfoHandler;
@@ -112,7 +112,7 @@ namespace AssetProcessor
[&](AzToolsFramework::AssetDatabase::SourceDatabaseEntry& sourceEntry)
{
assetId = AZ::Data::AssetId(sourceEntry.m_sourceGuid, productItemData->m_databaseInfo.m_subID);
// Use a decimal value to display the sub ID and not hex. Lumberyard is not consistent about
// Use a decimal value to display the sub ID and not hex. Open 3D Engine is not consistent about
// how sub IDs are displayed, so it's important to double check what format a sub ID is in before using it elsewhere.
m_ui->productAssetIdValueLabel->setText(assetId.ToString<AZStd::string>(AZ::Data::AssetId::SubIdDisplayType::Decimal).c_str());
@@ -76,7 +76,7 @@ namespace AssetProcessor
AzFramework::StringFunc::AssetDatabasePath::Join(scanFolder.m_scanFolder.c_str(), fullPath.c_str(), fullPath, true, false);
// It's common for Lumberyard game projects and scan folders to be in a subfolder
// It's common for Open 3D Engine game projects and scan folders to be in a subfolder
// of the engine install. To improve readability of the source files, strip out
// that portion of the path if it overlaps.
if (!m_assetRootSet)
@@ -16,6 +16,6 @@
<file>AssetProcessor_arrow_down.svg</file>
<file>AssetProcessor_arrow_up.svg</file>
<file>AssetProcessor_refresh.png</file>
<file>lyassetprocessor.png</file>
<file>o3de_assetprocessor.png</file>
</qresource>
</RCC>
@@ -1,3 +0,0 @@
version https://git-lfs.github.com/spec/v1
oid sha256:9965c02521822e92baad09dde15064ac0f326d861533725de283bac6e0ce3618
size 108108
@@ -1,3 +0,0 @@
version https://git-lfs.github.com/spec/v1
oid sha256:68f33fe204a433f8c765d524c9b9e42963f5d16c3442f36df87185bcb4555111
size 8686
@@ -0,0 +1,3 @@
version https://git-lfs.github.com/spec/v1
oid sha256:424c7e9a5bc4819e838e142ae87c987ce96ef40ff853d4a2610d7f068d635851
size 107989
@@ -0,0 +1,3 @@
version https://git-lfs.github.com/spec/v1
oid sha256:9f16c891aaa1686a3735fe84c7a69c3cef24e68af7baa86bf2f54ab4d51b71e8
size 8489
@@ -148,7 +148,7 @@ void AssetProcessingStateDataUnitTest::DataTest(AssetProcessor::AssetDatabaseCon
scanFolders.clear();
//add a scanfolder
scanFolder = ScanFolderDatabaseEntry("c:/lumberyard/dev", "dev", "rootportkey", "");
scanFolder = ScanFolderDatabaseEntry("c:/O3DE/dev", "dev", "rootportkey", "");
UNIT_TEST_EXPECT_TRUE(stateData->SetScanFolder(scanFolder));
if (scanFolder.m_scanFolderID == AzToolsFramework::AssetDatabase::InvalidEntryId)
{
@@ -158,7 +158,7 @@ void AssetProcessingStateDataUnitTest::DataTest(AssetProcessor::AssetDatabaseCon
//add the same folder again, should not add another because it already exists, so we should get the same id
// not only that, but the path should update.
ScanFolderDatabaseEntry dupeScanFolder = ScanFolderDatabaseEntry("c:/lumberyard/dev2", "dev", "rootportkey", "");
ScanFolderDatabaseEntry dupeScanFolder = ScanFolderDatabaseEntry("c:/O3DE/dev2", "dev", "rootportkey", "");
dupeScanFolder.m_scanFolderID = AzToolsFramework::AssetDatabase::InvalidEntryId;
UNIT_TEST_EXPECT_TRUE(stateData->SetScanFolder(dupeScanFolder));
if (!(dupeScanFolder == scanFolder))
@@ -174,7 +174,7 @@ void AssetProcessingStateDataUnitTest::DataTest(AssetProcessor::AssetDatabaseCon
scanFolders.clear();
UNIT_TEST_EXPECT_TRUE(stateData->GetScanFolders(scanFolders));
UNIT_TEST_EXPECT_TRUE(scanFolders.size() == 1);
UNIT_TEST_EXPECT_TRUE(ScanFoldersContainScanPath(scanFolders, "c:/lumberyard/dev2"));
UNIT_TEST_EXPECT_TRUE(ScanFoldersContainScanPath(scanFolders, "c:/O3DE/dev2"));
UNIT_TEST_EXPECT_TRUE(ScanFoldersContainScanFolderID(scanFolders, scanFolder.m_scanFolderID));
UNIT_TEST_EXPECT_TRUE(ScanFoldersContainPortableKey(scanFolders, scanFolder.m_portableKey.c_str()));
UNIT_TEST_EXPECT_TRUE(ScanFoldersContainPortableKey(scanFolders, "rootportkey"));
@@ -200,7 +200,7 @@ void AssetProcessingStateDataUnitTest::DataTest(AssetProcessor::AssetDatabaseCon
}
//add another folder
ScanFolderDatabaseEntry gameScanFolderEntry("c:/lumberyard/game", "game", "gameportkey", "");
ScanFolderDatabaseEntry gameScanFolderEntry("c:/O3DE/game", "game", "gameportkey", "");
UNIT_TEST_EXPECT_TRUE(stateData->SetScanFolder(gameScanFolderEntry));
if (gameScanFolderEntry.m_scanFolderID == AzToolsFramework::AssetDatabase::InvalidEntryId ||
gameScanFolderEntry.m_scanFolderID == scanFolder.m_scanFolderID)
@@ -213,8 +213,8 @@ void AssetProcessingStateDataUnitTest::DataTest(AssetProcessor::AssetDatabaseCon
scanFolders.clear();
UNIT_TEST_EXPECT_TRUE(stateData->GetScanFolders(scanFolders));
UNIT_TEST_EXPECT_TRUE(scanFolders.size() == 2);
UNIT_TEST_EXPECT_TRUE(ScanFoldersContainScanPath(scanFolders, "c:/lumberyard/dev2"));
UNIT_TEST_EXPECT_TRUE(ScanFoldersContainScanPath(scanFolders, "c:/lumberyard/game"));
UNIT_TEST_EXPECT_TRUE(ScanFoldersContainScanPath(scanFolders, "c:/O3DE/dev2"));
UNIT_TEST_EXPECT_TRUE(ScanFoldersContainScanPath(scanFolders, "c:/O3DE/game"));
UNIT_TEST_EXPECT_TRUE(ScanFoldersContainScanFolderID(scanFolders, scanFolder.m_scanFolderID));
UNIT_TEST_EXPECT_TRUE(ScanFoldersContainScanFolderID(scanFolders, gameScanFolderEntry.m_scanFolderID));
@@ -226,11 +226,11 @@ void AssetProcessingStateDataUnitTest::DataTest(AssetProcessor::AssetDatabaseCon
scanFolders.clear();
UNIT_TEST_EXPECT_TRUE(stateData->GetScanFolders(scanFolders));
UNIT_TEST_EXPECT_TRUE(scanFolders.size() == 1);
UNIT_TEST_EXPECT_TRUE(ScanFoldersContainScanPath(scanFolders, "c:/lumberyard/dev2"));
UNIT_TEST_EXPECT_TRUE(ScanFoldersContainScanPath(scanFolders, "c:/O3DE/dev2"));
UNIT_TEST_EXPECT_TRUE(ScanFoldersContainScanFolderID(scanFolders, scanFolder.m_scanFolderID));
//add another folder again
gameScanFolderEntry = ScanFolderDatabaseEntry("c:/lumberyard/game", "game", "gameportkey2", "");
gameScanFolderEntry = ScanFolderDatabaseEntry("c:/O3DE/game", "game", "gameportkey2", "");
UNIT_TEST_EXPECT_TRUE(stateData->SetScanFolder(gameScanFolderEntry));
if (gameScanFolderEntry.m_scanFolderID == AzToolsFramework::AssetDatabase::InvalidEntryId ||
gameScanFolderEntry.m_scanFolderID == scanFolder.m_scanFolderID)
@@ -243,8 +243,8 @@ void AssetProcessingStateDataUnitTest::DataTest(AssetProcessor::AssetDatabaseCon
scanFolders.clear();
UNIT_TEST_EXPECT_TRUE(stateData->GetScanFolders(scanFolders));
UNIT_TEST_EXPECT_TRUE(scanFolders.size() == 2);
UNIT_TEST_EXPECT_TRUE(ScanFoldersContainScanPath(scanFolders, "c:/lumberyard/dev2"));
UNIT_TEST_EXPECT_TRUE(ScanFoldersContainScanPath(scanFolders, "c:/lumberyard/game"));
UNIT_TEST_EXPECT_TRUE(ScanFoldersContainScanPath(scanFolders, "c:/O3DE/dev2"));
UNIT_TEST_EXPECT_TRUE(ScanFoldersContainScanPath(scanFolders, "c:/O3DE/game"));
UNIT_TEST_EXPECT_TRUE(ScanFoldersContainScanFolderID(scanFolders, scanFolder.m_scanFolderID));
UNIT_TEST_EXPECT_TRUE(ScanFoldersContainScanFolderID(scanFolders, gameScanFolderEntry.m_scanFolderID));
@@ -258,7 +258,7 @@ void AssetProcessingStateDataUnitTest::DataTest(AssetProcessor::AssetDatabaseCon
///////////////////////////////////////////////////////////
//setup for sources tests
//for the rest of the test lets add the original scan folder
scanFolder = ScanFolderDatabaseEntry("c:/lumberyard/dev", "dev", "devkey2", "");
scanFolder = ScanFolderDatabaseEntry("c:/O3DE/dev", "dev", "devkey2", "");
UNIT_TEST_EXPECT_TRUE(stateData->SetScanFolder(scanFolder));
///////////////////////////////////////////////////////////
@@ -370,7 +370,7 @@ void AssetProcessingStateDataUnitTest::DataTest(AssetProcessor::AssetDatabaseCon
UNIT_TEST_EXPECT_TRUE(SourcesContainSourceGuid(sources, source.m_sourceGuid));
//add the same source again, but change the scan folder. This should NOT add a new source - even if we don't know what the sourceID is:
ScanFolderDatabaseEntry scanfolder2 = ScanFolderDatabaseEntry("c:/lumberyard/dev2", "dev2", "devkey3", "");
ScanFolderDatabaseEntry scanfolder2 = ScanFolderDatabaseEntry("c:/O3DE/dev2", "dev2", "devkey3", "");
UNIT_TEST_EXPECT_TRUE(stateData->SetScanFolder(scanfolder2));
SourceDatabaseEntry dupeSource2(source);
@@ -554,7 +554,7 @@ void AssetProcessingStateDataUnitTest::DataTest(AssetProcessor::AssetDatabaseCon
////////////////////////////////////////////////////////////////
//Setup for jobs tests by having a scan folder and some sources
//Add a scan folder
scanFolder = ScanFolderDatabaseEntry("c:/lumberyard/dev", "dev", "devkey3", "");
scanFolder = ScanFolderDatabaseEntry("c:/O3DE/dev", "dev", "devkey3", "");
UNIT_TEST_EXPECT_TRUE(stateData->SetScanFolder(scanFolder));
//Add some sources
@@ -108,11 +108,13 @@ void FileWatcherUnitTestRunner::StartTest()
if (outstandingFiles.count() > 0)
{
#if defined(AZ_ENABLE_TRACING)
AZ_TracePrintf(AssetProcessor::DebugChannel, "Timed out waiting for file changes: %d / %d missed\n", outstandingFiles.count(), maxFiles);
for (const QString& pending : outstandingFiles)
{
AZ_TracePrintf(AssetProcessor::DebugChannel, "Missed file: %s", pending.toUtf8().data());
}
#endif
Q_EMIT UnitTestFailed("Missed files waiting for file changes");
return;
}
@@ -308,4 +310,4 @@ void FileWatcherUnitTestRunner::StartTest()
#if !AZ_TRAIT_DISABLE_FAILED_ASSET_PROCESSOR_TESTS
REGISTER_UNIT_TEST(FileWatcherUnitTestRunner)
#endif // !AZ_TRAIT_DISABLE_FAILED_ASSET_PROCESSOR_TESTS
#endif // !AZ_TRAIT_DISABLE_FAILED_ASSET_PROCESSOR_TESTS
@@ -33,6 +33,7 @@
#include <AzToolsFramework/ToolsComponents/ToolsAssetCatalogComponent.h>
#include <AzToolsFramework/AssetBrowser/AssetBrowserComponent.h>
#include <AzToolsFramework/SourceControl/PerforceComponent.h>
#include <AzToolsFramework/Prefab/PrefabSystemComponent.h>
namespace AssetProcessor
{
@@ -153,6 +154,7 @@ AZ::ComponentTypeList AssetProcessorAZApplication::GetRequiredSystemComponents()
}
components.push_back(azrtti_typeid<AzToolsFramework::PerforceComponent>());
components.push_back(azrtti_typeid<AzToolsFramework::Prefab::PrefabSystemComponent>());
return components;
}
@@ -508,7 +510,6 @@ bool ApplicationManager::StartAZFramework()
AZ::ComponentApplication::StartupParameters params;
QString projectName = AssetUtilities::ComputeProjectName();
AZ::SettingsRegistryInterface& registry = *AZ::SettingsRegistry::Get();
// Prevent loading of gems in the Create method of the ComponentApplication
params.m_loadDynamicModules = false;
@@ -11,6 +11,7 @@
*/
#include "ApplicationManagerBase.h"
#include <AzCore/Settings/SettingsRegistryMergeUtils.h>
#include <AzCore/std/smart_ptr/make_shared.h>
#include <AzCore/std/sort.h>
@@ -1592,10 +1593,9 @@ void ApplicationManagerBase::BuilderLogV(const AZ::Uuid& builderId, const char*
if (m_builderDescMap.find(builderId) != m_builderDescMap.end())
{
const AssetBuilderSDK::AssetBuilderDesc& builderDesc = m_builderDescMap[builderId];
char messageBuffer[1024];
azvsnprintf(messageBuffer, 1024, message, list);
AZ_TracePrintf(AssetProcessor::ConsoleChannel, "Builder name : %s Message : %s.\n", builderDesc.m_name.c_str(), messageBuffer);
AZ_TracePrintf(AssetProcessor::ConsoleChannel, "Builder name : %s Message : %s.\n", m_builderDescMap[builderId].m_name.c_str(), messageBuffer);
}
else
{
@@ -1691,7 +1691,7 @@ bool ApplicationManagerBase::CheckSufficientDiskSpace(const QString& savePath, q
}
qint64 bytesFree = 0;
bool result = AzToolsFramework::ToolsFileUtils::GetFreeDiskSpace(savePath, bytesFree);
[[maybe_unused]] bool result = AzToolsFramework::ToolsFileUtils::GetFreeDiskSpace(savePath, bytesFree);
AZ_Assert(result, "Unable to determine the amount of free space on drive containing path (%s).", savePath.toUtf8().constData());
@@ -1717,7 +1717,15 @@ void ApplicationManagerBase::RemoveOldTempFolders()
return;
}
QString startFolder = rootDir.absolutePath();
QString startFolder;
if (auto settingsRegistry = AZ::SettingsRegistry::Get(); settingsRegistry != nullptr)
{
if (AZ::IO::Path userPath; settingsRegistry->Get(userPath.Native(), AZ::SettingsRegistryMergeUtils::FilePathKey_ProjectUserPath))
{
startFolder = QString::fromUtf8(userPath.c_str(), aznumeric_cast<int>(userPath.Native().size()));
}
}
QDir root;
if (!AssetUtilities::CreateTempRootFolder(startFolder, root))
{
@@ -115,9 +115,6 @@ ApplicationManager::BeforeRunStatus GUIApplicationManager::BeforeRun()
#endif
AssetProcessor::MessageInfoBus::Handler::BusConnect();
QString bootstrapPath = devRoot.filePath("bootstrap.cfg");
m_qtFileWatcher.addPath(bootstrapPath);
// we have to monitor both the cache folder and the database file and restart AP if either of them gets deleted
// It is important to note that we are monitoring the parent folder and not the actual cache folder itself since
// we want to handle the use case on Mac OS if the user moves the cache folder to the trash.
@@ -135,33 +132,6 @@ ApplicationManager::BeforeRunStatus GUIApplicationManager::BeforeRun()
QObject::connect(&m_qtFileWatcher, &QFileSystemWatcher::fileChanged, this, &GUIApplicationManager::FileChanged);
QObject::connect(&m_qtFileWatcher, &QFileSystemWatcher::directoryChanged, this, &GUIApplicationManager::DirectoryChanged);
// Register a notifier for when the project_path property changes within the SettingsRegistry
if (auto settingsRegistry = AZ::SettingsRegistry::Get(); settingsRegistry != nullptr)
{
// Needs to be updated to project_path.
auto OnProjectPathChanged = [this, cachedProjectPath = projectPath](AZStd::string_view path, AZ::SettingsRegistryInterface::Type)
{
constexpr auto projectPathKey =
AZ::SettingsRegistryInterface::FixedValueString(AZ::SettingsRegistryMergeUtils::BootstrapSettingsRootKey)
+ "/project_path";
if (projectPathKey == path)
{
AZ::SettingsRegistryInterface::FixedValueString newProjectPath;
if (auto registry = AZ::SettingsRegistry::Get(); registry && registry->Get(newProjectPath, path))
{
// we only have to quit if the project path has changed, not if just the bootstrap has changed.
if (cachedProjectPath.compare(newProjectPath.c_str()) != 0)
{
AZ_TracePrintf(AssetProcessor::ConsoleChannel, "bootstrap.cfg Project Path changed from %s to %s. Quitting\n",
cachedProjectPath.toUtf8().constData(), newProjectPath.c_str());
QMetaObject::invokeMethod(this, "QuitRequested", Qt::QueuedConnection);
}
}
}
};
m_bootstrapGameFolderChangedHandler = settingsRegistry->RegisterNotifier(AZStd::move(OnProjectPathChanged));
}
return ApplicationManager::BeforeRunStatus::Status_Success;
}
@@ -306,7 +276,7 @@ bool GUIApplicationManager::Run()
m_trayIcon = new QSystemTrayIcon(m_mainWindow);
m_trayIcon->setContextMenu(trayIconMenu);
m_trayIcon->setToolTip(QObject::tr("Asset Processor"));
m_trayIcon->setIcon(QIcon(":/lyassetprocessor.png"));
m_trayIcon->setIcon(QIcon(":/o3de_assetprocessor.png"));
m_trayIcon->show();
QObject::connect(m_trayIcon, &QSystemTrayIcon::activated, m_mainWindow, [&, wrapper](QSystemTrayIcon::ActivationReason reason)
{
@@ -328,8 +298,8 @@ bool GUIApplicationManager::Run()
if (startHidden)
{
m_trayIcon->showMessage(
QCoreApplication::translate("Tray Icon", "Lumberyard Asset Processor has started"),
QCoreApplication::translate("Tray Icon", "The Lumberyard Asset Processor monitors raw project assets and converts those assets into runtime-ready data."),
QCoreApplication::translate("Tray Icon", "Open 3D Engine Asset Processor has started"),
QCoreApplication::translate("Tray Icon", "The Open 3D Engine Asset Processor monitors raw project assets and converts those assets into runtime-ready data."),
QSystemTrayIcon::Information, 3000);
}
}
@@ -651,29 +621,10 @@ void GUIApplicationManager::DirectoryChanged([[maybe_unused]] QString path)
void GUIApplicationManager::FileChanged(QString path)
{
QDir devRoot = ApplicationManager::GetSystemRoot();
QString bootstrapPath = devRoot.filePath("bootstrap.cfg");
QDir projectCacheRoot;
AssetUtilities::ComputeProjectCacheRoot(projectCacheRoot);
QString assetDbPath = projectCacheRoot.filePath("assetdb.sqlite");
if (QString::compare(AssetUtilities::NormalizeFilePath(path), bootstrapPath, Qt::CaseInsensitive) == 0)
{
AssetUtilities::UpdateBranchToken();
if (m_connectionManager)
{
m_connectionManager->UpdateAllowedListFromBootStrap();
}
// Re-merge the Bootstrap.cfg into the SettingsRegistry
AZStd::vector<char> scratchBuffer;
AZ::SettingsRegistryInterface* settingsRegistry = AZ::SettingsRegistry::Get();
AZ_Assert(settingsRegistry, "Unable to retrieve global SettingsRegistry, it should be available now");
AZ::SettingsRegistryMergeUtils::MergeSettingsToRegistry_Bootstrap(*settingsRegistry);
AZ::SettingsRegistryMergeUtils::MergeSettingsToRegistry_O3deUserRegistry(*settingsRegistry, AZ_TRAIT_OS_PLATFORM_CODENAME, {});
AZ::SettingsRegistryMergeUtils::MergeSettingsToRegistry_CommandLine(*settingsRegistry, *m_frameworkApp.GetAzCommandLine(), false);
AZ::SettingsRegistryMergeUtils::MergeSettingsToRegistry_AddRuntimeFilePaths(*settingsRegistry);
}
else if (QString::compare(AssetUtilities::NormalizeFilePath(path), assetDbPath, Qt::CaseInsensitive) == 0)
if (QString::compare(AssetUtilities::NormalizeFilePath(path), assetDbPath, Qt::CaseInsensitive) == 0)
{
if (!QFile::exists(assetDbPath))
{
@@ -868,7 +819,7 @@ void GUIApplicationManager::ShowTrayIconErrorMessage(QString msg)
{
m_timeWhenLastWarningWasShown = currentTime;
m_trayIcon->showMessage(
QCoreApplication::translate("Tray Icon", "Lumberyard Asset Processor"),
QCoreApplication::translate("Tray Icon", "Open 3D Engine Asset Processor"),
QCoreApplication::translate("Tray Icon", msg.toUtf8().data()),
QSystemTrayIcon::Critical, 3000);
}
@@ -880,7 +831,7 @@ void GUIApplicationManager::ShowTrayIconMessage(QString msg)
if (m_trayIcon && m_mainWindow && !m_mainWindow->isVisible())
{
m_trayIcon->showMessage(
QCoreApplication::translate("Tray Icon", "Lumberyard Asset Processor"),
QCoreApplication::translate("Tray Icon", "Open 3D Engine Asset Processor"),
QCoreApplication::translate("Tray Icon", msg.toUtf8().data()),
QSystemTrayIcon::Information, 3000);
}
@@ -112,7 +112,6 @@ private:
QPointer<QSystemTrayIcon> m_trayIcon;
QPointer<MainWindow> m_mainWindow;
AZ::SettingsRegistryInterface::NotifyEventHandler m_bootstrapGameFolderChangedHandler;
AZStd::chrono::system_clock::time_point m_timeWhenLastWarningWasShown;
};
@@ -1521,10 +1521,10 @@ namespace AssetProcessor
//! Given a scan folder path, get its complete info
const AssetProcessor::ScanFolderInfo* PlatformConfiguration::GetScanFolderByPath(const QString& scanFolderPath) const
{
QString normalized = AssetUtilities::NormalizeFilePath(scanFolderPath);
AZ::IO::Path scanFolderPathView(scanFolderPath.toUtf8().constData());
for (int pathIdx = 0; pathIdx < m_scanFolders.size(); ++pathIdx)
{
if (QString::compare(m_scanFolders[pathIdx].ScanPath(), normalized, Qt::CaseSensitive) == 0)
if (AZ::IO::PathView(m_scanFolders[pathIdx].ScanPath().toUtf8().constData()) == scanFolderPathView)
{
return &m_scanFolders[pathIdx];
}
@@ -181,13 +181,29 @@ namespace AssetUtilsInternal
if (AZ::SettingsRegistryMergeUtils::DumpSettingsRegistryToStream(settingsRegistry, AssetProcessorUserSettingsRootKey,
apSettingsStream, apDumperSettings))
{
constexpr const char* AssetProcessorTmpSetreg = "asset_processor.setreg.tmp";
// Write to a temporary file first before renaming it to the final file location
// This is needed to reduce the potential of a race condition which occurs when other applications attempt to load settings registry
// files from the project's user Registry folder while the AssetProcessor is writing the file out the asset_processor.setreg
// at the same time
QString tempDirValue;
AssetUtilities::CreateTempWorkspace(tempDirValue);
QDir tempDir(tempDirValue);
AZ::IO::FixedMaxPath tmpSetregPath = tempDir.absoluteFilePath(QString(AssetProcessorTmpSetreg)).toUtf8().data();
constexpr auto modeFlags = AZ::IO::SystemFile::SF_OPEN_WRITE_ONLY | AZ::IO::SystemFile::SF_OPEN_CREATE
| AZ::IO::SystemFile::SF_OPEN_CREATE_PATH;
if (AZ::IO::SystemFile apSetregFile; apSetregFile.Open(setregPath.c_str(), modeFlags))
if (AZ::IO::SystemFile apSetregFile; apSetregFile.Open(tmpSetregPath.c_str(), modeFlags))
{
size_t bytesWritten = apSetregFile.Write(apSettingsJson.data(), apSettingsJson.size());
return bytesWritten == apSettingsJson.size();
// Close the file so that it can be renamed.
apSetregFile.Close();
if (bytesWritten == apSettingsJson.size())
{
// Create the directory to contain the moved setreg file
AZ::IO::SystemFile::CreateDir(AZ::IO::FixedMaxPath(setregPath.ParentPath()).c_str());
return AZ::IO::SystemFile::Rename(tmpSetregPath.c_str(), setregPath.c_str(), true);
}
}
else
{
@@ -1478,7 +1494,14 @@ to ensure that the address is correct. Asset Processor won't be running in serve
return false;
}
#endif
return AzToolsFramework::AssetUtils::UpdateFilePathToCorrectCase(rootPath, relativePathFromRoot);
AZStd::string relPathFromRoot = relativePathFromRoot.toUtf8().constData();
if(AzToolsFramework::AssetUtils::UpdateFilePathToCorrectCase(rootPath.toUtf8().constData(), relPathFromRoot))
{
relativePathFromRoot = QString::fromUtf8(relPathFromRoot.c_str(), aznumeric_cast<int>(relPathFromRoot.size()));
return true;
}
return false;
}
BuilderFilePatternMatcher::BuilderFilePatternMatcher(const AssetBuilderSDK::AssetBuilderPattern& pattern, const AZ::Uuid& builderDescID)
@@ -192,7 +192,6 @@ namespace CrashHandler
return;
}
AZ::IO::FileIOBase* fileIO = AZ::IO::FileIOBase::GetInstance();
#if AZ_TRAIT_CRASHHANDLER_CONVERT_MULTIBYTE_CHARS
wchar_t pathStr[AZ_MAX_PATH_LEN];
@@ -21,7 +21,7 @@
namespace CrashHandler
{
static const char* defaultCrashFolder = "CrashDB/";
static const char* lumberyardProductName = "lumberyard";
static const char* O3DEProductName = "Open 3D Engine";
using CrashHandlerAnnotations = std::map<std::string, std::string>;
using CrashHandlerArguments = std::vector<std::string>;
@@ -48,7 +48,7 @@ namespace CrashHandler
virtual std::string DetermineAppPath() const;
virtual const char* GetProductName() const { return lumberyardProductName; }
virtual const char* GetProductName() const { return O3DEProductName; }
virtual bool CreateCrashHandlerDB(const std::string& reportPath) const;
@@ -80,7 +80,7 @@
</rect>
</property>
<property name="text">
<string>Lumberyard has encountered a fatal error. We're sorry for the inconvenience.</string>
<string>Open 3D Engine has encountered a fatal error. We're sorry for the inconvenience.</string>
</property>
<property name="wordWrap">
<bool>true</bool>
@@ -96,7 +96,7 @@
</rect>
</property>
<property name="text">
<string>A Lumberyard Editor crash debugging file has been created at:</string>
<string>An Open 3D Engine Editor crash debugging file has been created at:</string>
</property>
<property name="wordWrap">
<bool>true</bool>
@@ -134,7 +134,7 @@
</rect>
</property>
<property name="text">
<string>If you are willing to submit this file to Amazon it will help us improve the Lumberyard experience. We will treat this report as confidential.</string>
<string>If you are willing to submit this file to Amazon it will help us improve the Open 3D Engine experience. We will treat this report as confidential.</string>
</property>
<property name="wordWrap">
<bool>true</bool>
@@ -32,11 +32,11 @@
#include "UI/ui_submit_report.h"
namespace Lumberyard
namespace O3de
{
void InstallCrashUploader(int& argc, char* argv[])
{
Lumberyard::CrashUploader::SetCrashUploader(std::make_shared<Lumberyard::ToolsCrashUploader>(argc, argv));
O3de::CrashUploader::SetCrashUploader(std::make_shared<O3de::ToolsCrashUploader>(argc, argv));
}
QString GetReportString(const std::wstring& reportPath)
{
@@ -16,7 +16,7 @@
#include <Uploader/CrashUploader.h>
namespace Lumberyard
namespace O3de
{
class ToolsCrashUploader : public CrashUploader
{
@@ -22,10 +22,10 @@ namespace
{
int HandlerMain(int argc, char* argv[])
{
Lumberyard::InstallCrashUploader(argc, argv);
O3de::InstallCrashUploader(argc, argv);
LOG(ERROR) << "Initializing windows crash uploader";
int resultCode = crashpad::HandlerMain(argc, argv, Lumberyard::CrashUploader::GetCrashUploader()->GetUserStreamSources());
int resultCode = crashpad::HandlerMain(argc, argv, O3de::CrashUploader::GetCrashUploader()->GetUserStreamSources());
return resultCode;
}
@@ -16,7 +16,7 @@
#include <minidump/minidump_user_extension_stream_data_source.h>
#include <vector>
namespace Lumberyard
namespace O3de
{
class BufferedDataStream : public crashpad::MinidumpUserExtensionStreamDataSource
{
@@ -28,7 +28,7 @@
#include <base/logging.h>
#include <string>
namespace Lumberyard
namespace O3de
{
bool CheckConfirmation(const crashpad::CrashReportDatabase::Report& report);
void InstallCrashUploader(int& argc, char* argv[]);
@@ -16,7 +16,7 @@
#include <handler/user_stream_data_source.h>
#include "base/files/file_path.h"
namespace Lumberyard
namespace O3de
{
class FileStreamDataSource : public crashpad::UserStreamDataSource
{
@@ -12,7 +12,7 @@
#include <BufferedDataStream.h>
namespace Lumberyard
namespace O3de
{
BufferedDataStream::BufferedDataStream(uint32_t stream_type, const void* data, size_t data_size)
: crashpad::MinidumpUserExtensionStreamDataSource(stream_type)
@@ -25,7 +25,7 @@
#include <memory>
namespace Lumberyard
namespace O3de
{
using namespace crashpad;
@@ -15,7 +15,7 @@
#include <memory>
namespace Lumberyard
namespace O3de
{
FileStreamDataSource::FileStreamDataSource(const base::FilePath& filePath) :
m_filePath{ filePath }
+1
View File
@@ -103,6 +103,7 @@ namespace FileUtil
if (ftimeModify == nullptr && ftimeCreate == nullptr && ftimeAccess == nullptr)
{
FindClose(hFind);
return true;
}
+1 -1
View File
@@ -114,7 +114,7 @@ PakSystemFile* PakSystem::Open(const char* a_path, const char* a_mode)
dirToSearch = PathHelpers::GetDirectory(dirToSearch);
AZ::IO::LocalFileIO localFileIO;
bool foundOK = localFileIO.FindFiles(dirToSearch.c_str(), "*.pak", [&](const char* filePath) -> bool
localFileIO.FindFiles(dirToSearch.c_str(), "*.pak", [&](const char* filePath) -> bool
{
const string foundFilename(filePath);
if (StringHelpers::EqualsIgnoreCase(PathHelpers::FindExtension(foundFilename), "pak"))
@@ -288,7 +288,10 @@ ZipDir::CachePtr ZipDir::CacheFactory::MakeCache (const char* szFile)
m_f = NULL; // we don't own the file anymore - it's in possession of the cache instance
// try to serialize into the memory
size_t nSizeSerialized = m_treeFileEntries.Serialize (cache->GetRoot());
#if !defined(NDEBUG)
size_t nSizeSerialized =
#endif
m_treeFileEntries.Serialize (cache->GetRoot());
assert (nSizeSerialized == nSizeRequired);
@@ -523,10 +526,9 @@ bool ZipDir::CacheFactory::BuildFileEntryMap()
{
case ZipFile::EXTRA_NTFS:
{
ZipFile::ExtraNTFSHeader& ntfsHdr = *(ZipFile::ExtraNTFSHeader*)pAttrData;
extra.nLastModifyTime = *(uint64*)(pAttrData + sizeof(ZipFile::ExtraNTFSHeader));
uint64 accTime = *(uint64*)(pAttrData + sizeof(ZipFile::ExtraNTFSHeader) + 8);
uint64 crtTime = *(uint64*)(pAttrData + sizeof(ZipFile::ExtraNTFSHeader) + 16);
//uint64 accTime = *(uint64*)(pAttrData + sizeof(ZipFile::ExtraNTFSHeader) + 8);
//uint64 crtTime = *(uint64*)(pAttrData + sizeof(ZipFile::ExtraNTFSHeader) + 16);
}
break;
}
@@ -1699,7 +1699,10 @@ bool ZipDir::CacheRW::WriteCDR(FILE* fTarget, bool encryptCDR)
//arrFiles.SortByFileOffset();
size_t nSizeCDR = arrFiles.GetStats().nSizeCDR;
void* pCDR = malloc(nSizeCDR);
size_t nSizeCDRSerialized = arrFiles.MakeZipCDR(m_lCDROffset, pCDR, encryptCDR);
#if !defined(NDEBUG)
size_t nSizeCDRSerialized =
#endif
arrFiles.MakeZipCDR(m_lCDROffset, pCDR, encryptCDR);
assert (nSizeCDRSerialized == nSizeCDR);
if (encryptCDR)
@@ -1931,9 +1934,6 @@ bool ZipDir::CacheRW::EncryptArchive(EncryptionChange change, IEncryptPredicate*
FileRecordList arrFiles(GetRoot());
arrFiles.SortByFileOffset();
// the total size of data in the queue
unsigned nQueueSize = 0;
size_t unusedSpace = 0;
size_t lastDataEnd = 0;
@@ -1974,8 +1974,6 @@ bool ZipDir::CacheRW::EncryptArchive(EncryptionChange change, IEncryptPredicate*
return false;
}
bool methodChanged = false;
ZipFile::ushort oldMethod = entry->nMethod;
ZipFile::ushort newMethod = oldMethod;
if (change == ENCRYPT)
@@ -65,7 +65,7 @@ void logmessage(const char* text, ...)
error += ret;
bufferlen -= ret;
int count = vsnprintf(error, bufferlen, text, arg);
vsnprintf(error, bufferlen, text, arg);
AZ_TracePrintf(0, szBuffer);
@@ -169,7 +169,6 @@ bool CSTLHelper::FromFile(const std::string& rFileName, std::vector<uint8_t>&
return false;
}
size_t nNumRead = 0;
rIn.resize(fileSize);
AZ::IO::SystemFile::SizeType actualReadAmount = inputFile.Read(fileSize, &rIn[0]);
@@ -276,7 +275,7 @@ bool CSTLHelper::AppendToFile(const std::string& rFileName, const std::vector<ui
return false;
}
AZ::IO::SystemFile::SizeType bytesWritten = outputFile.Write(rOut.data(), rOut.size());
[[maybe_unused]] AZ::IO::SystemFile::SizeType bytesWritten = outputFile.Write(rOut.data(), rOut.size());
AZ_Warning("ShaderCompiler", bytesWritten == rOut.size(), "Did not write out all the data to the file: %s", rFileName.c_str());
return true;
}
@@ -56,14 +56,15 @@ void CCrySimpleCache::Init()
std::string CCrySimpleCache::CreateFileName(const tdHash& rHash) const
{
std::string Name;
Name = CSTLHelper::Hash2String(rHash);
AZStd::string Name;
Name = CSTLHelper::Hash2String(rHash).c_str();
char Tmp[4] = "012";
Tmp[0] = Name.c_str()[0];
Tmp[1] = Name.c_str()[1];
Tmp[2] = Name.c_str()[2];
return SEnviropment::Instance().m_CachePath + Tmp + "/" + Name;
AZ::IO::Path resultFileName = SEnviropment::Instance().m_CachePath / Tmp / Name;
return std::string{ resultFileName.c_str(), resultFileName.Native().size() };
}
@@ -159,7 +160,6 @@ bool CCrySimpleCache::LoadCacheFile(const std::string& filename)
uint32_t num = 0;
uint64_t nFilePos = 0;
uint64_t nFilePos2 = 0;
//////////////////////////////////////////////////////////////////////////
AZ::IO::SystemFile cacheFile;
@@ -303,7 +303,8 @@ void CCrySimpleCache::ThreadFunc_SavePendingCacheEntries()
if (pPendingCacheEntry)
{
CSTLHelper::AppendToFile(SEnviropment::Instance().m_CachePath + "Cache.dat", *pPendingCacheEntry);
AZ::IO::Path cacheDatPath = SEnviropment::Instance().m_CachePath / "Cache.dat";
CSTLHelper::AppendToFile(std::string{ cacheDatPath.c_str(), cacheDatPath.Native().size() }, *pPendingCacheEntry);
delete pPendingCacheEntry;
}
} while (!bListEmpty);
@@ -180,7 +180,7 @@ void CCrySimpleErrorLog::SendMail()
char DispFilename[1024];
azsprintf(DispFilename, "%d-%s", a + 1, err->GetFilename().c_str());
std::string sErrorFile = SEnviropment::Instance().m_ErrorPath + Filename;
std::string sErrorFile = (SEnviropment::Instance().m_ErrorPath / Filename).c_str();
std::vector<uint8_t> bytes;
std::string text = err->GetFileContents();
@@ -211,7 +211,7 @@ void CCrySimpleErrorLog::SendMail()
CSMTPMailer::tstrcol bcc;
CSMTPMailer mail("", "", SEnviropment::Instance().m_MailServer);
bool res = mail.Send(SEnviropment::Instance().m_FailEMail, Rcpt, cc, bcc, err->GetErrorName(), body, Attachment);
mail.Send(SEnviropment::Instance().m_FailEMail, Rcpt, cc, bcc, err->GetErrorName(), body, Attachment);
a = 0;
body = mailBody;
@@ -133,12 +133,12 @@ protected:
Ret += CreateInfoText("<b>Setup</b>:", "");
Ret += CreateInfoText("Root", SEnviropment::Instance().m_Root);
Ret += CreateInfoText("CompilerPath", SEnviropment::Instance().m_CompilerPath);
Ret += CreateInfoText("CachePath", SEnviropment::Instance().m_CachePath);
Ret += CreateInfoText("TempPath", SEnviropment::Instance().m_TempPath);
Ret += CreateInfoText("ErrorPath", SEnviropment::Instance().m_ErrorPath);
Ret += CreateInfoText("ShaderPath", SEnviropment::Instance().m_ShaderPath);
Ret += CreateInfoText("Root", SEnviropment::Instance().m_Root.c_str());
Ret += CreateInfoText("CompilerPath", SEnviropment::Instance().m_CompilerPath.c_str());
Ret += CreateInfoText("CachePath", SEnviropment::Instance().m_CachePath.c_str());
Ret += CreateInfoText("TempPath", SEnviropment::Instance().m_TempPath.c_str());
Ret += CreateInfoText("ErrorPath", SEnviropment::Instance().m_ErrorPath.c_str());
Ret += CreateInfoText("ShaderPath", SEnviropment::Instance().m_ShaderPath.c_str());
Ret += CreateInfoText("FailEMail", SEnviropment::Instance().m_FailEMail);
Ret += CreateInfoText("MailServer", SEnviropment::Instance().m_MailServer);
Ret += CreateInfoText("port", SEnviropment::Instance().m_port);
@@ -170,7 +170,9 @@ bool CCrySimpleJob::ExecuteCommand(const std::string& rCmd, std::string& outErro
threadIdStream << threadId;
// Multiple threads could execute a command, therefore the temporary file has to be unique per thread.
std::string stdErrorTempFilename = SEnviropment::Instance().m_TempPath + "stderr_" + threadIdStream.str() + ".log";
AZ::IO::Path errorTempFilePath = SEnviropment::Instance().m_TempPath / AZStd::string::format("stderr_%s.log", threadIdStream.str().c_str());
std::string stdErrorTempFilename{ errorTempFilePath.c_str(), errorTempFilePath.Native().size() };
CCrySimpleFileGuard FGTmpOutput(stdErrorTempFilename); // Delete file at the end of this function
std::string systemCmd = rCmd;
@@ -282,7 +282,7 @@ bool CCrySimpleJobCompile::Compile(const TiXmlElement* pElement, std::vector<uin
shaderPath = AZStd::string::format("%s%s/", SEnviropment::Instance().m_ShaderPath.c_str(), language.c_str());
}
NormalizePath(shaderPath);
shaderPath = AZ::IO::PathView(shaderPath).LexicallyNormal().Native();
if (!IsPathValid(shaderPath))
{
State(ECSJS_ERROR);
@@ -347,17 +347,14 @@ bool CCrySimpleJobCompile::Compile(const TiXmlElement* pElement, std::vector<uin
static AZStd::atomic_long nTmpCounter = { 0 };
++nTmpCounter;
char tmpstr[64];
azsprintf(tmpstr, "%ld", static_cast<long>(nTmpCounter));
const auto tmpIndex = AZStd::string::format("%ld", static_cast<long>(nTmpCounter));
const AZ::IO::Path TmpIn = SEnviropment::Instance().m_TempPath / (tmpIndex + ".In");
const AZ::IO::Path TmpOut = SEnviropment::Instance().m_TempPath / (tmpIndex + ".Out");
CCrySimpleFileGuard FGTmpIn(TmpIn.c_str());
CCrySimpleFileGuard FGTmpOut(TmpOut.c_str());
CSTLHelper::ToFile(TmpIn.c_str(), std::vector<uint8_t>(pProgram, &pProgram[strlen(pProgram)]));
const std::string TmpIn = SEnviropment::Instance().m_TempPath + tmpstr + ".In";
const std::string TmpOut = SEnviropment::Instance().m_TempPath + tmpstr + ".Out";
CCrySimpleFileGuard FGTmpIn(TmpIn);
CCrySimpleFileGuard FGTmpOut(TmpOut);
CSTLHelper::ToFile(TmpIn, std::vector<uint8_t>(pProgram, &pProgram[strlen(pProgram)]));
const AZStd::string compilerPath = SEnviropment::Instance().m_CompilerPath.c_str();
AZ::IO::Path compilerPath = SEnviropment::Instance().m_CompilerPath;
AZStd::string command;
if (m_Version >= EPV_V0022)
{
@@ -370,7 +367,7 @@ bool CCrySimpleJobCompile::Compile(const TiXmlElement* pElement, std::vector<uin
return false;
}
AZStd::string commandStringToFormat = compilerPath + compilerExecutable;
AZStd::string commandStringToFormat = (compilerPath / compilerExecutable).Native();
#if defined(AZ_PLATFORM_LINUX) || defined(AZ_PLATFORM_MAC)
// Surrounding compiler path+executable with quotes to support spaces in the path.
@@ -392,7 +389,7 @@ bool CCrySimpleJobCompile::Compile(const TiXmlElement* pElement, std::vector<uin
return false;
}
AZStd::string fxcLocation = compilerPath + fxcCompilerExecutable;
AZ::IO::Path fxcLocation = compilerPath / fxcCompilerExecutable;
// Handle an extra string parameter to specify the base directory where the fxc compiler is located
command = AZStd::move(AZStd::string::format(commandStringToFormat.c_str(), fxcLocation.c_str(), pEntry, pProfile, TmpOut.c_str(), TmpIn.c_str()));
@@ -434,7 +431,7 @@ bool CCrySimpleJobCompile::Compile(const TiXmlElement* pElement, std::vector<uin
AZStd::string insertPattern = "\\\"";
// Search for the next space until that path exists. Then we assume that's the path to the executable.
size_t startPos = command.find(compilerPath);
size_t startPos = command.find(compilerPath.Native());
for (size_t pos = command.find(" ", startPos); pos != AZStd::string::npos; pos = command.find(" ", pos + 1))
{
if (AZ::IO::SystemFile::Exists(command.substr(startPos, pos - startPos).c_str()))
@@ -449,7 +446,7 @@ bool CCrySimpleJobCompile::Compile(const TiXmlElement* pElement, std::vector<uin
command = AZStd::move(AZStd::string::format(pCompileFlags, pEntry, pProfile, TmpOut.c_str(), TmpIn.c_str()));
}
command = compilerPath + command;
command = compilerPath.Native() + command;
}
AZStd::string hardwareTarget;
@@ -540,11 +537,13 @@ bool CCrySimpleJobCompile::Compile(const TiXmlElement* pElement, std::vector<uin
std::string tags = pTags ? pTags : "";
std::string filteredError;
CSTLHelper::Replace(filteredError, outError, TmpIn + ".patched", "%filename%"); // DXPS does its own patching
CSTLHelper::Replace(filteredError, filteredError, TmpIn, "%filename%");
AZ::IO::Path patchFilePath = TmpIn;
patchFilePath.ReplaceFilename(AZ::IO::PathView{ AZStd::string{ TmpIn.Filename().Native() } + ".patched" });
CSTLHelper::Replace(filteredError, outError, patchFilePath.c_str(), "%filename%"); // DXPS does its own patching
CSTLHelper::Replace(filteredError, filteredError, TmpIn.c_str(), "%filename%");
// replace any that don't have the full path
CSTLHelper::Replace(filteredError, filteredError, std::string(tmpstr) + ".In.patched", "%filename%"); // DXPS does its own patching
CSTLHelper::Replace(filteredError, filteredError, std::string(tmpstr) + ".In", "%filename%");
CSTLHelper::Replace(filteredError, filteredError, (tmpIndex + ".In.patched").c_str(), "%filename%"); // DXPS does its own patching
CSTLHelper::Replace(filteredError, filteredError, (tmpIndex + ".In").c_str(), "%filename%");
CSTLHelper::Replace(filteredError, filteredError, "\r\n", "\n");
@@ -552,11 +551,11 @@ bool CCrySimpleJobCompile::Compile(const TiXmlElement* pElement, std::vector<uin
throw new CCompilerError(pEntry, filteredError, ccs, sIP, pShaderRequestLine, pProgram, project, platform.c_str(), compiler.c_str(), language.c_str(), tags, pProfile);
}
if (!CSTLHelper::FromFile(TmpOut, rVec))
if (!CSTLHelper::FromFile(TmpOut.c_str(), rVec))
{
State(ECSJS_ERROR_FILEIO);
std::string errorString("Could not read: ");
errorString += TmpOut;
errorString += std::string(TmpOut.c_str(), TmpOut.Native().size());
CrySimple_ERROR(errorString.c_str());
return false;
}
@@ -19,6 +19,7 @@
#include <Core/Common.h>
#include <tinyxml/tinyxml.h>
#include <AzCore/Debug/Trace.h>
#include <AzCore/IO/Path/Path.h>
#include <AzCore/std/string/string.h>
@@ -30,7 +31,7 @@ CCrySimpleJobGetShaderList::CCrySimpleJobGetShaderList(uint32_t requestIP, std::
bool CCrySimpleJobGetShaderList::Execute(const TiXmlElement* pElement)
{
AZStd::string shaderListFilename;
AZ::IO::Path shaderListFilename;
const char* project = pElement->Attribute("Project");
const char* shaderList = pElement->Attribute("ShaderList");
@@ -38,7 +39,10 @@ bool CCrySimpleJobGetShaderList::Execute(const TiXmlElement* pElement)
const char* compiler = pElement->Attribute("Compiler");
const char* language = pElement->Attribute("Language");
shaderListFilename = AZStd::string::format("./Cache/%s%s-%s-%s/%s", project, platform, compiler, language, shaderList);
shaderListFilename = project;
shaderListFilename /= "Cache";
shaderListFilename /= AZStd::string::format("%s-%s-%s", platform, compiler, language);
shaderListFilename /= shaderList;
//open the file and read into the rVec
@@ -23,6 +23,7 @@
#include <Core/WindowsAPIImplementation.h>
#include <AzCore/Debug/Trace.h>
#include <AzCore/IO/Path/Path.h>
#include <AzCore/IO/SystemFile.h>
CCrySimpleJobRequest::CCrySimpleJobRequest(EProtocolVersion Version, uint32_t requestIP)
@@ -41,7 +42,7 @@ bool CCrySimpleJobRequest::Execute(const TiXmlElement* pElement)
return false;
}
AZStd::string shaderListFilename;
AZ::IO::Path shaderListFilename;
if (m_Version >= EPV_V0023)
{
const char* project = pElement->Attribute("Project");
@@ -64,20 +65,16 @@ bool CCrySimpleJobRequest::Execute(const TiXmlElement* pElement)
AZStd::string compiler = pElement->Attribute("Compiler");
AZStd::string language = pElement->Attribute("Language");
shaderListFilename = AZStd::string::format("%s%s-%s-%s/%s", project, platform.c_str(), compiler.c_str(), language.c_str(), shaderList);
shaderListFilename = project;
shaderListFilename /= "Cache";
shaderListFilename /= AZStd::string::format("%s-%s-%s", platform.c_str(), compiler.c_str(), language.c_str());
shaderListFilename /= shaderList;
}
else
{
// In previous versions Platform attribute is the shader list filename directly
shaderListFilename = pElement->Attribute("Platform");
}
if (shaderListFilename.length() >= AZ_MAX_PATH_LEN)
{
State(ECSJS_ERROR);
CrySimple_ERROR("Shader list filename is too long");
return false;
}
std::string shaderRequestLine(shaderRequest);
tdEntryVec toks;
@@ -71,15 +71,6 @@
AZStd::atomic_long CCrySimpleServer::ms_ExceptionCount = {0};
const static std::string SHADER_PROFILER = "NVShaderPerf" EXTENSION;
const static std::string SHADER_PATH_SOURCE = "Source";
const static std::string SHADER_PATH_BINARY = "Binary";
const static std::string SHADER_PATH_HALFSTRIPPED = "HalfStripped";
const static std::string SHADER_PATH_DISASSEMBLED = "DisAsm";
const static std::string SHADER_PATH_STRIPPPED = "Stripped";
const static std::string SHADER_PATH_CACHE = "Cache";
static const bool autoDeleteJobWhenDone = true;
static const int sleepTimeWhenWaiting = 10;
@@ -335,7 +326,6 @@ void CompileJob::Process()
}
const char* pVersion = pElement->Attribute("Version");
const char* pPlatform = pElement->Attribute("Platform");
const char* pHardwareTarget = nullptr;
//new request type?
@@ -575,31 +565,45 @@ void TickThread()
//////////////////////////////////////////////////////////////////////////
void LoadCache()
{
const std::string& cachePath = SEnviropment::Instance().m_CachePath;
if (CCrySimpleCache::Instance().LoadCacheFile(cachePath + "Cache.dat"))
AZ::IO::Path cacheDatFile{ SEnviropment::Instance().m_CachePath };
AZ::IO::Path cacheBakFile = cacheDatFile;
cacheDatFile /= "Cache.dat";
cacheBakFile /= "Cache.bak";
if (CCrySimpleCache::Instance().LoadCacheFile(cacheDatFile.c_str()))
{
AZ::IO::Path cacheBakFile2 = cacheBakFile;
cacheBakFile2.ReplaceFilename("Cache.bak2");
printf("Creating cache backup...\n");
AZ::IO::SystemFile::Delete((cachePath + "Cache.bak2").c_str());
printf("Move %s to %s\n", (cachePath + "Cache.bak").c_str(), (cachePath + "Cache.bak2").c_str());
AZ::IO::SystemFile::Rename((cachePath + "Cache.bak").c_str(), (cachePath + "Cache.bak2").c_str());
printf("Copy %s to %s\n", (cachePath + "Cache.dat").c_str(), (cachePath + "Cache.bak").c_str());
CopyFileOnPlatform((cachePath + "Cache.dat").c_str(), (cachePath + "Cache.bak").c_str(), FALSE);
AZ::IO::SystemFile::Delete(cacheBakFile2.c_str());
printf("Move %s to %s\n", cacheBakFile.c_str(), cacheBakFile2.c_str());
AZ::IO::SystemFile::Rename(cacheBakFile.c_str(), cacheBakFile2.c_str());
printf("Copy %s to %s\n", cacheDatFile.c_str(), cacheBakFile.c_str());
CopyFileOnPlatform(cacheDatFile.c_str(), cacheBakFile.c_str(), false);
printf("Cache backup done.\n");
}
else
{
// Restoring backup cache!
printf("Cache file corrupted!!!\n");
if (AZ::IO::SystemFile::Exists(cacheDatFile.c_str()))
{
printf("Cache file corrupted!!!\n");
AZ::IO::SystemFile::Delete(cacheDatFile.c_str());
}
printf("Restoring backup cache...\n");
AZ::IO::SystemFile::Delete((cachePath + "Cache.dat").c_str());
printf("Copy %s to %s\n", (cachePath + "Cache.bak").c_str(), (cachePath + "Cache.dat").c_str());
CopyFileOnPlatform((cachePath + "Cache.bak").c_str(), (cachePath + "Cache.dat").c_str(), FALSE);
if (!CCrySimpleCache::Instance().LoadCacheFile(cachePath + "Cache.dat"))
printf("Copy %s to %s\n", cacheBakFile.c_str(), cacheDatFile.c_str());
CopyFileOnPlatform(cacheBakFile.c_str(), cacheDatFile.c_str(), false);
if (!CCrySimpleCache::Instance().LoadCacheFile(cacheDatFile.c_str()))
{
// Backup file corrupted too!
printf("Backup file corrupted too!!!\n");
if (AZ::IO::SystemFile::Exists(cacheDatFile.c_str()))
{
printf("Backup file corrupted too!!!\n");
AZ::IO::SystemFile::Delete(cacheDatFile.c_str());
}
printf("Deleting cache completely\n");
AZ::IO::SystemFile::Delete((cachePath + "Cache.dat").c_str());
AZ::IO::SystemFile::Delete(cacheDatFile.c_str());
}
}
@@ -664,56 +668,9 @@ CCrySimpleServer::CCrySimpleServer()
CrySimple_SECURE_END
}
bool GetBaseDirectory(AZStd::string& baseDir)
{
char executableDir[AZ_MAX_PATH_LEN];
if (AZ::Utils::GetExecutableDirectory(executableDir, AZ_MAX_PATH_LEN) == AZ::Utils::ExecutablePathResult::Success)
{
AZStd::string_view executableDirView(executableDir);
if (executableDirView.size() > 1 && !executableDirView.ends_with(AZ_CORRECT_FILESYSTEM_SEPARATOR) && executableDirView.size() < AZStd::size(executableDir) - 1)
{
executableDir[executableDirView.size()] = AZ_CORRECT_FILESYSTEM_SEPARATOR;
executableDir[executableDirView.size() + 1] = '\0';
executableDirView = { executableDir, executableDirView.size() + 1 };
}
baseDir = AZStd::string(executableDir);
return true;
}
else
{
return false;
}
}
void NormalizePath(AZStd::string& pathToNormalize)
{
AzFramework::StringFunc::Root::Normalize(pathToNormalize);
}
void NormalizePath(std::string& pathToNormalize)
{
AZStd::string tempString = pathToNormalize.c_str();
NormalizePath(tempString);
pathToNormalize = tempString.c_str();
}
bool IsPathValid(const AZStd::string& path)
{
// Calculating base directory every time.
// It's slower than using a cached value, but safer.
AZStd::string baseDir;
if (GetBaseDirectory(baseDir))
{
AZStd::string basePath(AZ::IO::Path(baseDir).LexicallyNormal().Native());
AZStd::string subPath(AZ::IO::Path(path).LexicallyNormal().Native());
return strncmp(basePath.c_str(), subPath.c_str(), basePath.size()) == 0;
}
else
{
return false;
}
return AZ::IO::PathView(path).IsRelativeTo(AZ::IO::PathView(SEnviropment::Instance().m_Root));
}
bool IsPathValid(const std::string& path)
@@ -724,48 +681,29 @@ bool IsPathValid(const std::string& path)
void CCrySimpleServer::Init()
{
char executableDir[AZ_MAX_PATH_LEN];
AZ::Utils::GetExecutableDirectory(executableDir, AZ_MAX_PATH_LEN);
AZStd::string_view executableDirView(executableDir);
if (executableDirView.size() > 1 && !executableDirView.ends_with(AZ_CORRECT_FILESYSTEM_SEPARATOR) && executableDirView.size() < AZStd::size(executableDir) - 1)
{
executableDir[executableDirView.size()] = AZ_CORRECT_FILESYSTEM_SEPARATOR;
executableDir[executableDirView.size() + 1] = '\0';
executableDirView = { executableDir, executableDirView.size() + 1 };
}
SEnviropment::Instance().m_Root = std::string(executableDir);
AZStd::string baseDir;
GetBaseDirectory(baseDir);
SEnviropment::Instance().m_CompilerPath = baseDir.c_str();
SEnviropment::Instance().m_CompilerPath += "/Compiler/";
SEnviropment::Instance().m_CachePath = SEnviropment::Instance().m_Root + "Cache/";
SEnviropment::Instance().m_Root = AZ::Utils::GetExecutableDirectory();
SEnviropment::Instance().m_CompilerPath = SEnviropment::Instance().m_Root / "Compiler";
SEnviropment::Instance().m_CachePath = SEnviropment::Instance().m_Root / "Cache";
if (SEnviropment::Instance().m_TempPath.empty())
{
SEnviropment::Instance().m_TempPath = SEnviropment::Instance().m_Root + "Temp/";
SEnviropment::Instance().m_TempPath = SEnviropment::Instance().m_Root / "Temp";
}
if (SEnviropment::Instance().m_ErrorPath.empty())
{
SEnviropment::Instance().m_ErrorPath = SEnviropment::Instance().m_Root + "Error/";
SEnviropment::Instance().m_ErrorPath = SEnviropment::Instance().m_Root / "Error";
}
if (SEnviropment::Instance().m_ShaderPath.empty())
{
SEnviropment::Instance().m_ShaderPath = SEnviropment::Instance().m_Root + "Shaders/";
SEnviropment::Instance().m_ShaderPath = SEnviropment::Instance().m_Root / "Shaders";
}
NormalizePath(SEnviropment::Instance().m_Root);
NormalizePath(SEnviropment::Instance().m_CompilerPath);
NormalizePath(SEnviropment::Instance().m_CachePath);
NormalizePath(SEnviropment::Instance().m_ErrorPath);
NormalizePath(SEnviropment::Instance().m_TempPath);
NormalizePath(SEnviropment::Instance().m_ShaderPath);
AZ::IO::SystemFile::CreateDir(SEnviropment::Instance().m_ErrorPath.c_str());
AZ::IO::SystemFile::CreateDir(SEnviropment::Instance().m_TempPath.c_str());
AZ::IO::SystemFile::CreateDir(SEnviropment::Instance().m_CachePath.c_str());
AZ::IO::SystemFile::CreateDir(SEnviropment::Instance().m_ShaderPath.c_str());
SEnviropment::Instance().m_Root = SEnviropment::Instance().m_Root.LexicallyNormal();
SEnviropment::Instance().m_CompilerPath = SEnviropment::Instance().m_CompilerPath.LexicallyNormal();
SEnviropment::Instance().m_CachePath = SEnviropment::Instance().m_CachePath.LexicallyNormal();
SEnviropment::Instance().m_ErrorPath = SEnviropment::Instance().m_ErrorPath.LexicallyNormal();
SEnviropment::Instance().m_TempPath = SEnviropment::Instance().m_TempPath.LexicallyNormal();
SEnviropment::Instance().m_ShaderPath = SEnviropment::Instance().m_ShaderPath.LexicallyNormal();
if (SEnviropment::Instance().m_Caching)
{
@@ -15,6 +15,7 @@
#define __CRYSIMPLESERVER__
#include <Core/Common.h>
#include <AzCore/IO//Path/Path.h>
#include <AzCore/std/string/string.h>
#include <AzCore/std/containers/unordered_set.h>
#include <AzCore/std/containers/unordered_map.h>
@@ -24,11 +25,6 @@
extern bool g_Success;
bool GetExecutableDirectory(AZStd::string& executableDir);
bool GetBaseDirectory(AZStd::string& baseDir);
void NormalizePath(AZStd::string& pathToNormalize);
void NormalizePath(std::string& pathToNormalize);
bool IsPathValid(const AZStd::string& path);
bool IsPathValid(const std::string& path);
@@ -42,12 +38,12 @@ class CCrySimpleSock;
class SEnviropment
{
public:
std::string m_Root;
std::string m_CompilerPath;
std::string m_CachePath;
std::string m_TempPath;
std::string m_ErrorPath;
std::string m_ShaderPath;
AZ::IO::Path m_Root;
AZ::IO::Path m_CompilerPath;
AZ::IO::Path m_CachePath;
AZ::IO::Path m_TempPath;
AZ::IO::Path m_ErrorPath;
AZ::IO::Path m_ShaderPath;
std::string m_FailEMail;
std::string m_MailServer;
@@ -44,7 +44,7 @@ namespace
};
static AZStd::atomic_long numberOfOpenSockets = {0};
const int MAX_DATA_SIZE = 1024 * 1024; // Only allow 1 MB of data to come through. Lumberyard Game Engine has the same size constraint
const int MAX_DATA_SIZE = 1024 * 1024; // Only allow 1 MB of data to come through. Open 3D Engine has the same size constraint
const size_t BLOCKSIZE = 4 * 1024;
const size_t MAX_ERROR_MESSAGE_SIZE = 1024;
const size_t MAX_HOSTNAME_BUFFER_SIZE = 1024;
@@ -554,7 +554,6 @@ bool CCrySimpleSock::Recv(std::vector<uint8_t>& rVec)
if (size.m_Data64 > MAX_DATA_SIZE)
{
int WSAError = WSAGetLastError();
char acTmp[MAX_ERROR_MESSAGE_SIZE];
azsprintf(acTmp, "Error while receiving size of data - Size is greater than max support data size.");
CrySimple_ERROR(acTmp);
@@ -74,7 +74,7 @@ void CShaderList::Add(const std::string& rShaderListName, const char* pLine)
CCrySimpleMutexAutoLock Lock2(m_Mutex2); //load/save mutex
m_ShaderLists[rShaderListName] = new CShaderListFile(rShaderListName);
it = m_ShaderLists.find(rShaderListName);
it->second->Load((SEnviropment::Instance().m_CachePath + rShaderListName).c_str());
it->second->Load((SEnviropment::Instance().m_CachePath / AZStd::string_view{ rShaderListName.c_str(), rShaderListName.size() }).c_str());
}
}
it->second->InsertLine(pLine);
@@ -65,7 +65,7 @@ public:
}
if (azstricmp(strKey.c_str(), "TempDir") == 0)
{
SEnviropment::Instance().m_TempPath = AddSlash(strValue);
SEnviropment::Instance().m_TempPath = AZStd::string_view{ strValue.c_str(), strValue.size() };
}
if (azstricmp(strKey.c_str(), "MailServer") == 0)
{
+3 -2
View File
@@ -1387,13 +1387,14 @@ XmlNodeRef XmlParser::parseSource(const IXmlBufferSource* source)
bufferSize = sizeof(buffer) / sizeof(buffer[0])
};
m_pImpl->beginParse();
int bytesRead;
while (bytesRead = source->Read(buffer, bufferSize))
int bytesRead = source->Read(buffer, bufferSize);
while (bytesRead)
{
if (!m_pImpl->parse(buffer, bytesRead))
{
break;
}
bytesRead = source->Read(buffer, bufferSize);
}
return m_pImpl->endParse(m_errorString);
}
+15
View File
@@ -13,6 +13,20 @@ if(NOT PAL_TRAIT_BUILD_HOST_TOOLS)
return()
endif()
if(PAL_HOST_PLATFORM_NAME_LOWERCASE STREQUAL "windows")
ly_add_target(
NAME DeltaCataloger EXECUTABLE
NAMESPACE AZ
FILES_CMAKE
deltacataloger_files.cmake
deltacataloger_win_files.cmake
BUILD_DEPENDENCIES
PRIVATE
AZ::AzCore
AZ::AzFramework
AZ::AzToolsFramework
)
else()
ly_add_target(
NAME DeltaCataloger EXECUTABLE
NAMESPACE AZ
@@ -24,6 +38,7 @@ ly_add_target(
AZ::AzFramework
AZ::AzToolsFramework
)
endif()
if(PAL_TRAIT_BUILD_TESTS_SUPPORTED)
@@ -9,5 +9,6 @@
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
#
set (PAL_TRAIT_BUILD_ASSETPROCESSOR_APPLICATION_TYPE APPLICATION)
set(FILES
source/DeltaCataloger.rc
)
@@ -0,0 +1,3 @@
version https://git-lfs.github.com/spec/v1
oid sha256:02a102aad5a014f2dc8cfcfb3f502ccf45207e2f8aca946b84e910488e4027c5
size 103189
@@ -0,0 +1 @@
IDI_ICON1 ICON DISCARDABLE "DeltaCataloger.ico"
@@ -503,7 +503,7 @@ typedef enum _FRAMEBUFFER_FETCH_TYPE
// NOTE: HLSLCC flags are specified by command line when executing this cross compiler.
// If these flags change, the command line switch '-flags=XXX' must change as well.
// Lumberyard composes the command line in file 'dev\Code\CryEngine\RenderDll\Common\Shaders\RemoteCompiler.cpp'
// Open 3D Engine composes the command line in file 'dev\Code\CryEngine\RenderDll\Common\Shaders\RemoteCompiler.cpp'
/*HLSL constant buffers are treated as default-block unform arrays by default. This is done
to support versions of GLSL which lack ARB_uniform_buffer_object functionality.
@@ -52,13 +52,12 @@ AZ_POP_DISABLE_WARNING
static char* cJSON_strdup(const char* str)
{
size_t len;
char* copy;
size_t len = strlen(str) + 1;
char* copy = (char*)cJSON_malloc(len);
len = strlen(str) + 1;
if (!(copy = (char*)cJSON_malloc(len))) return 0;
memcpy(copy,str,len);
return copy;
if (!copy) return 0;
memcpy(copy,str,len);
return copy;
}
void cJSON_InitHooks(cJSON_Hooks* hooks)
@@ -209,7 +208,15 @@ static char *print_string_ptr(const char *str)
const char *ptr;char *ptr2,*out;int len=0;unsigned char token;
if (!str) return cJSON_strdup("");
ptr=str;while ((token=*ptr) && ++len) {if (strchr("\"\\\b\f\n\r\t",token)) len++; else if (token<32) len+=5;ptr++;}
ptr=str;
token = *ptr;
while (token && ++len)
{
if (strchr("\"\\\b\f\n\r\t",token)) len++;
else if (token<32) len+=5;
ptr++;
token = *ptr;
}
out=(char*)cJSON_malloc(len+3);
if (!out) return 0;
@@ -325,8 +332,8 @@ static const char *parse_array(cJSON *item,const char *value)
while (*value==',')
{
cJSON *new_item;
if (!(new_item=cJSON_New_Item())) return 0; /* memory fail */
cJSON *new_item = cJSON_New_Item();
if (!new_item) return 0; /* memory fail */
child->next=new_item;new_item->prev=child;child=new_item;
value=skip(parse_value(child,skip(value+1)));
if (!value) return 0; /* memory fail */
@@ -415,8 +422,8 @@ static const char *parse_object(cJSON *item,const char *value)
while (*value==',')
{
cJSON *new_item;
if (!(new_item=cJSON_New_Item())) return 0; /* memory fail */
cJSON* new_item = cJSON_New_Item();
if (!new_item) return 0; /* memory fail */
child->next=new_item;new_item->prev=child;child=new_item;
value=skip(parse_string(child,skip(value+1)));
if (!value) return 0;
@@ -349,7 +349,6 @@ int GetOptions(int argc, char** argv, Options* psOptions)
{
int i;
int fullShaderChain = -1;
int hashOut = 0;
InitOptions(psOptions);
@@ -412,8 +411,6 @@ int GetOptions(int argc, char** argv, Options* psOptions)
uint64_t hash = hash64((const uint8_t*)psOptions->outputShaderFile, (uint32_t)strlen(psOptions->outputShaderFile), 0);
uint32_t high = (uint32_t)(hash >> 32);
uint32_t low = (uint32_t)(hash & 0x00000000FFFFFFFF);
dir = strrchr(psOptions->outputShaderFile, '\\');
@@ -484,7 +481,6 @@ int Run(const char* srcPath, const char* destPath, GLLang language, int flags, c
Timer_t timer;
int compiledOK = 0;
double crossCompileTime = 0;
double glslCompileTime = 0;
HLSLcc_SetMemoryFunctions(malloc_hook, calloc_hook, free_hook, realloc_hook);
-11
View File
@@ -628,11 +628,6 @@ const uint32_t* DecodeDeclaration(Shader* psShader, const uint32_t* pui32Token,
{
ui32TokenLength = pui32Token[1];
{
int iTupleSrc = 0, iTupleDest = 0;
// const uint32_t ui32ConstCount = pui32Token[1] - 2;
// const uint32_t ui32TupleCount = (ui32ConstCount / 4);
CUSTOMDATA_CLASS eClass = DecodeCustomDataClass(pui32Token[0]);
const uint32_t ui32NumVec4 = (ui32TokenLength - 2) / 4;
uint32_t uIdx = 0;
@@ -671,8 +666,6 @@ const uint32_t* DecodeDeclaration(Shader* psShader, const uint32_t* pui32Token,
}
case OPCODE_DCL_UNORDERED_ACCESS_VIEW_RAW:
{
ResourceBinding* psBinding = NULL;
ConstantBuffer* psBuffer = NULL;
psDecl->ui32NumOperands = 1;
psDecl->sUAV.ui32GloballyCoherentAccess = DecodeAccessCoherencyFlags(*pui32Token);
@@ -725,8 +718,6 @@ const uint32_t* DecodeDeclaration(Shader* psShader, const uint32_t* pui32Token,
}
case OPCODE_DCL_THREAD_GROUP_SHARED_MEMORY_STRUCTURED:
{
ResourceBinding* psBinding = NULL;
ConstantBuffer* psBuffer = NULL;
psDecl->ui32NumOperands = 1;
psDecl->sUAV.ui32GloballyCoherentAccess = 0;
@@ -739,8 +730,6 @@ const uint32_t* DecodeDeclaration(Shader* psShader, const uint32_t* pui32Token,
}
case OPCODE_DCL_THREAD_GROUP_SHARED_MEMORY_RAW:
{
ResourceBinding* psBinding = NULL;
ConstantBuffer* psBuffer = NULL;
psDecl->ui32NumOperands = 1;
psDecl->sUAV.ui32GloballyCoherentAccess = 0;
@@ -407,8 +407,6 @@ static void SetupRegisterUsage(const Shader* psShader, const uint32_t ui32Token0
// In dx9 there is only one constant buffer per shader.
static void DeclareConstantBuffer(const Shader* psShader, Declaration* psDecl)
{
DECLUSAGE_DX9 eUsage = (DECLUSAGE_DX9)0;
uint32_t ui32UsageIndex = 0;
// Pick any constant register in the table. Might not start at c0 (e.g. when register(cX) is used).
uint32_t ui32RegNum = psShader->sInfo.psConstantBuffers->asVars[0].ui32StartOffset / 16;
OPERAND_TYPE_DX9 ui32RegType = OPERAND_TYPE_DX9_CONST;
@@ -446,9 +444,6 @@ static void DeclareConstantBuffer(const Shader* psShader, Declaration* psDecl)
static void DecodeDeclarationDX9(const Shader* psShader, const uint32_t ui32Token0, const uint32_t ui32Token1, Declaration* psDecl)
{
DECLUSAGE_DX9 eUsage = DecodeUsageDX9(ui32Token0);
uint32_t ui32UsageIndex = DecodeUsageIndexDX9(ui32Token0);
uint32_t ui32RegNum = DecodeOperandRegisterNumberDX9(ui32Token1);
uint32_t ui32RegType = DecodeOperandTypeDX9(ui32Token1);
if (psShader->eShaderType == VERTEX_SHADER)
+9 -10
View File
@@ -73,7 +73,7 @@ static void ReadInputSignatures(const uint32_t* pui32Tokens, ShaderInfo* psShade
InOutSignature* psSignatures;
const uint32_t* pui32FirstSignatureToken = pui32Tokens;
const uint32_t ui32ElementCount = *pui32Tokens++;
const uint32_t ui32Key = *pui32Tokens++;
/* const uint32_t ui32Key = */ *pui32Tokens++;
psSignatures = hlslcc_malloc(sizeof(InOutSignature) * ui32ElementCount);
psShaderInfo->psInputSignatures = psSignatures;
@@ -116,7 +116,7 @@ static void ReadOutputSignatures(const uint32_t* pui32Tokens, ShaderInfo* psShad
InOutSignature* psSignatures;
const uint32_t* pui32FirstSignatureToken = pui32Tokens;
const uint32_t ui32ElementCount = *pui32Tokens++;
const uint32_t ui32Key = *pui32Tokens++;
/* const uint32_t ui32Key = */ *pui32Tokens++;
psSignatures = hlslcc_malloc(sizeof(InOutSignature) * ui32ElementCount);
psShaderInfo->psOutputSignatures = psSignatures;
@@ -258,10 +258,10 @@ static const uint32_t* ReadConstantBuffer(ShaderInfo* psShaderInfo, const uint32
if (psShaderInfo->ui32MajorVersion >= 5)
{
uint32_t StartTexture = *pui32VarToken++;
uint32_t TextureSize = *pui32VarToken++;
uint32_t StartSampler = *pui32VarToken++;
uint32_t SamplerSize = *pui32VarToken++;
/* uint32_t StartTexture = */ *pui32VarToken++;
/* uint32_t TextureSize = */ *pui32VarToken++;
/* uint32_t StartSampler = */ *pui32VarToken++;
/* uint32_t SamplerSize = */ *pui32VarToken++;
}
psVar->haveDefaultValue = 0;
@@ -314,8 +314,8 @@ static void ReadResources(const uint32_t* pui32Tokens, ShaderInfo* psShaderInfo)
uint32_t ui32NumResourceBindings = *pui32Tokens++;
uint32_t ui32ResourceBindingOffset = *pui32Tokens++;
uint32_t ui32ShaderModel = *pui32Tokens++;
uint32_t ui32CompileFlags = *pui32Tokens++;//D3DCompile flags? http://msdn.microsoft.com/en-us/library/gg615083(v=vs.85).aspx
/* uint32_t ui32ShaderModel = */ *pui32Tokens++;
/* uint32_t ui32CompileFlags = */ *pui32Tokens++;//D3DCompile flags? http://msdn.microsoft.com/en-us/library/gg615083(v=vs.85).aspx
//Resources
pui32ResourceBindings = (const uint32_t*)((const char*)pui32FirstToken + ui32ResourceBindingOffset);
@@ -408,7 +408,7 @@ static void ReadInterfaces(const uint32_t* pui32Tokens, ShaderInfo* psShaderInfo
const uint32_t ui32ClassInstanceCount = *pui32Tokens++;
const uint32_t ui32ClassTypeCount = *pui32Tokens++;
const uint32_t ui32InterfaceSlotRecordCount = *pui32Tokens++;
const uint32_t ui32InterfaceSlotCount = *pui32Tokens++;
/* const uint32_t ui32InterfaceSlotCount = */ *pui32Tokens++;
const uint32_t ui32ClassInstanceOffset = *pui32Tokens++;
const uint32_t ui32ClassTypeOffset = *pui32Tokens++;
const uint32_t ui32InterfaceSlotOffset = *pui32Tokens++;
@@ -663,7 +663,6 @@ static int IsOffsetInType(ShaderVarType* psType, uint32_t parentOffset, uint32_t
int GetShaderVarFromOffset(const uint32_t ui32Vec4Offset, const uint32_t* pui32Swizzle, ConstantBuffer* psCBuf, ShaderVarType** ppsShaderVar, int32_t* pi32Index, int32_t* pi32Rebase)
{
uint32_t i;
const uint32_t ui32BaseByteOffset = ui32Vec4Offset * 16;
uint32_t ui32ByteOffset = ui32Vec4Offset * 16;
+4 -4
View File
@@ -18,7 +18,7 @@
#if defined(_WIN32) && !defined(PORTABLE)
#include <AzCore/PlatformDef.h>
AZ_PUSH_DISABLE_WARNING(4115, "-Wunknown-warning-option")
AZ_PUSH_DISABLE_WARNING(4115, "-Wunknown-warning-option") // 4115: named type definition in parentheses
#include <d3dcompiler.h>
AZ_POP_DISABLE_WARNING
#pragma comment(lib,"d3dcompiler.lib")
@@ -181,7 +181,6 @@ void AddVersionDependentCode(HLSLCrossCompilerContext* psContext)
// #extension directive must occur before any non-preprocessor token
if (EmulateDepthClamp(psContext->psShader->eTargetLanguage) && (psContext->psShader->eShaderType == VERTEX_SHADER || psContext->psShader->eShaderType == PIXEL_SHADER))
{
char* szInOut = psContext->psShader->eShaderType == VERTEX_SHADER ? "out" : "in";
ui32DepthClampImp = AddImport(psContext, SYMBOL_EMULATE_DEPTH_CLAMP, 0, 0);
bformata(glsl, "#if IMPORT_%d > 0\n", ui32DepthClampImp);
@@ -920,7 +919,6 @@ void WritePostStepTrace(HLSLCrossCompilerContext* psContext, uint32_t uStep)
uint16_t uOpcodeWriteMask = GetOpcodeWriteMask(psInstruction->eOpcode);
uint8_t uOperand = 0;
OPERAND_TYPE eOperandType = OPERAND_TYPE_NULL;
SHADER_VARIABLE_TYPE eVarToFlags = TO_FLAG_NONE;
Operand* psOperand = NULL;
uint32_t uiIgnoreSwizzle = 0;
@@ -1725,10 +1723,12 @@ void RemoveDoubleUnderscores(char* szName)
size_t length;
length = strlen(szName);
position = szName;
while (position = strstr(position, "__"))
position = strstr(position, "__");
while (position)
{
position[1] = '0';
position += 2;
position = strstr(position, "__");
}
}
@@ -299,8 +299,9 @@ void PreDeclareStructType(HLSLCrossCompilerContext* psContext, const char* Name,
if (psType->Class == SVC_STRUCT)
{
#if !defined(NDEBUG)
uint32_t unnamed_struct = strcmp(Name, "$Element") == 0 ? 1 : 0;
#endif
//Not supported at the moment
ASSERT(!unnamed_struct);
@@ -483,6 +484,7 @@ char* GetDeclaredOutputName(const HLSLCrossCompilerContext* psContext,
&psContext->psShader->sInfo,
&psOut);
(void)(foundOutput);
ASSERT(foundOutput);
if (eShaderType == GEOMETRY_SHADER)
@@ -590,7 +592,6 @@ static void DeclareInput(
{
if (iNumComponents == 1)
{
const uint32_t regNum = psDecl->asOperands[0].ui32RegisterNumber;
const uint32_t arraySize = psDecl->asOperands[0].aui32ArraySizes[0];
psContext->psShader->abScalarInput[psDecl->asOperands[0].ui32RegisterNumber] = -1;
@@ -1134,7 +1135,6 @@ void AddUserOutput(HLSLCrossCompilerContext* psContext, const Declaration* psDec
case VERTEX_SHADER:
{
int iNumComponents = 4; //GetMaxComponentFromComponentMask(&psDecl->asOperands[0]);
const char* Interpolation = "";
int stream = 0;
char* OutputName = GetDeclaredOutputName(psContext, VERTEX_SHADER, psOperand, &stream);
@@ -1413,7 +1413,9 @@ void DeclareBufferVariable(HLSLCrossCompilerContext* psContext, const uint32_t u
{
const char* Name = psCBuf->Name;
bstring StructName;
#if !defined(NDEBUG)
uint32_t unnamed_struct = strcmp(psCBuf->asVars[0].Name, "$Element") == 0 ? 1 : 0;
#endif
bstring glsl = *psContext->currentGLSLString;
ASSERT(psCBuf->ui32NumVars == 1);
@@ -1489,7 +1491,9 @@ void DeclarePLSVariable(HLSLCrossCompilerContext* psContext, const uint32_t ui32
(void)eResourceType;
const char* Name = plsVar->Name;
#if !defined(NDEBUG)
uint32_t unnamed_struct = strcmp(plsVar->asVars[0].Name, "$Element") == 0 ? 1 : 0;
#endif
bstring glsl = *psContext->currentGLSLString;
ASSERT(plsVar->ui32NumVars == 1);
@@ -1970,7 +1974,6 @@ void TranslateDeclaration(HLSLCrossCompilerContext* psContext, const Declaration
}
case OPCODE_DCL_TEMPS:
{
uint32_t i = 0;
const uint32_t ui32NumTemps = psDecl->value.ui32NumTemps;
if (psContext->flags & HLSLCC_FLAG_AVOID_TEMP_REGISTER_ALIASING && psContext->psShader->eShaderType != HULL_SHADER)
@@ -2106,7 +2109,6 @@ void TranslateDeclaration(HLSLCrossCompilerContext* psContext, const Declaration
const char* Precision = "highp";
const char* outputName = "PixOutput";
const char* qualifier = psContext->rendertargetUse[regNum] & OUTPUT_RENDERTARGET ? "inout" : "in";
bformata(glsl, "layout(location = %d) ", regNum);
bformata(glsl, "inout %s vec%d %s%d;\n", Precision, numElements, outputName, regNum);
@@ -943,8 +943,7 @@ static void TranslateTexCoord(HLSLCrossCompilerContext* psContext,
static int GetNumTextureDimensions(HLSLCrossCompilerContext* psContext,
const RESOURCE_DIMENSION eResDim)
{
int constructor = 0;
bstring glsl = *psContext->currentGLSLString;
(void)(psContext);
switch (eResDim)
{
@@ -1441,7 +1440,6 @@ static void TranslateShaderStorageStore(HLSLCrossCompilerContext* psContext, Ins
{
bstring glsl = *psContext->currentGLSLString;
ShaderVarType* psVarType = NULL;
uint32_t ui32DataTypeFlag = TO_FLAG_INTEGER;
int component;
int srcComponent = 0;
@@ -1450,7 +1448,6 @@ static void TranslateShaderStorageStore(HLSLCrossCompilerContext* psContext, Ins
Operand* psDestByteOff = 0;
Operand* psSrc = 0;
int structured = 0;
int groupshared = 0;
switch (psInst->eOpcode)
{
@@ -1474,7 +1471,6 @@ static void TranslateShaderStorageStore(HLSLCrossCompilerContext* psContext, Ins
ASSERT(psInst->asOperands[0].eSelMode == OPERAND_4_COMPONENT_MASK_MODE);
if (psInst->asOperands[0].ui32CompMask & (1 << component))
{
SHADER_VARIABLE_TYPE eSrcDataType = GetOperandDataType(psContext, psSrc);
uint32_t swizzle = 0;
if (structured && psDest->eType != OPERAND_TYPE_THREAD_GROUP_SHARED_MEMORY)
{
@@ -1575,7 +1571,6 @@ static void TranslateShaderPLSStore(HLSLCrossCompilerContext* psContext, Instruc
{
bstring glsl = *psContext->currentGLSLString;
ShaderVarType* psVarType = NULL;
uint32_t ui32DataTypeFlag = TO_FLAG_INTEGER;
int component;
int srcComponent = 0;
@@ -1607,7 +1602,6 @@ static void TranslateShaderPLSStore(HLSLCrossCompilerContext* psContext, Instruc
ASSERT(psInst->asOperands[0].eSelMode == OPERAND_4_COMPONENT_MASK_MODE);
if (psInst->asOperands[0].ui32CompMask & (1 << component))
{
SHADER_VARIABLE_TYPE eSrcDataType = GetOperandDataType(psContext, psSrc);
ASSERT(psDest->eType != OPERAND_TYPE_THREAD_GROUP_SHARED_MEMORY);
@@ -1999,7 +1993,6 @@ static void TranslateShaderPLSLoad(HLSLCrossCompilerContext* psContext, Instruct
bstring glsl = *psContext->currentGLSLString;
ShaderVarType* psVarType = NULL;
uint32_t aui32Swizzle[4] = { OPERAND_4_COMPONENT_X };
uint32_t ui32DataTypeFlag = TO_FLAG_INTEGER;
int component;
int destComponent = 0;
@@ -2833,7 +2826,6 @@ void SetDataTypes(HLSLCrossCompilerContext* psContext, Instruction* psInst, cons
case OPCODE_IMM_ATOMIC_CMP_EXCH:
{
Operand* dest = &psInst->asOperands[1];
Operand* destAddr = &psInst->asOperands[2];
ShaderVarType* type = LookupStructuredVar(psContext, dest, NULL, 0);
eNewType = type->Type;
break;
@@ -4485,7 +4477,6 @@ void TranslateInstruction(HLSLCrossCompilerContext* psContext, Instruction* psIn
case OPCODE_LD_MS:
{
ResourceBinding* psBinding = 0;
uint32_t dstSwizCount = GetNumSwizzleElements(&psInst->asOperands[0]);
uint32_t ui32FetchTypeToFlags;
#ifdef _DEBUG
AddIndentation(psContext);
@@ -5120,7 +5111,6 @@ void TranslateInstruction(HLSLCrossCompilerContext* psContext, Instruction* psIn
// Here's the description of what bitfieldExtract actually does
// https://www.opengl.org/registry/specs/ARB/gpu_shader5.txt
int numComponents = psInst->asOperands[0].iNumComponents;
AddIndentation(psContext);
bcatcstr(glsl, "{\n");
@@ -5456,7 +5446,6 @@ void TranslateInstruction(HLSLCrossCompilerContext* psContext, Instruction* psIn
}
case OPCODE_RESINFO:
{
const RESOURCE_DIMENSION eResDim = psContext->psShader->aeResourceDims[psInst->asOperands[2].ui32RegisterNumber];
const RESINFO_RETURN_TYPE eResInfoReturnType = psInst->eResInfoReturnType;
uint32_t destElemCount = GetNumSwizzleElements(&psInst->asOperands[0]);
uint32_t destElem;
@@ -1406,7 +1406,6 @@ void TranslateVariableName(HLSLCrossCompilerContext* psContext, const Operand* p
{
bool hasConstructor = false;
bstring glsl = *psContext->currentGLSLString;
int numComponents = GetNumSwizzleElements(psOperand);
*pui32IgnoreSwizzle = 0;
@@ -1747,7 +1746,6 @@ char ResourceGroupPrefix(ResourceGroup eResGroup)
void ResourceName(bstring output, Shader* psShader, const char* szName, ResourceGroup eGroup, const char* szSecondaryName, ResourceGroup eSecondaryGroup, uint32_t ui32ArrayOffset, const char* szModifier)
{
int i = 0;
const char* pBracket;
@@ -1784,7 +1782,6 @@ void TextureName(bstring output, Shader* psShader, const uint32_t ui32TextureReg
ResourceBinding* psTextureBinding = 0;
ResourceBinding* psSamplerBinding = 0;
int found;
uint32_t ui32ArrayOffset = 0;
const char* szModifier = bCompare ? "c" : "";
found = GetResourceFromBindingPoint(RGROUP_TEXTURE, ui32TextureRegister, &psShader->sInfo, &psTextureBinding);
@@ -445,7 +445,7 @@ typedef struct
// NOTE: HLSLCC flags are specified by command line when executing this cross compiler.
// If these flags change, the command line switch '-flags=XXX' must change as well.
// Lumberyard composes the command line in file 'dev\Code\CryEngine\RenderDll\Common\Shaders\RemoteCompiler.cpp'
// Open 3D Engine composes the command line in file 'dev\Code\CryEngine\RenderDll\Common\Shaders\RemoteCompiler.cpp'
/*HLSL constant buffers are treated as default-block unform arrays by default. This is done
to support versions of GLSL which lack ARB_uniform_buffer_object functionality.
@@ -52,11 +52,10 @@ AZ_POP_DISABLE_WARNING
static char* cJSON_strdup(const char* str)
{
size_t len;
char* copy;
size_t len = strlen(str) + 1;
char* copy = (char*)cJSON_malloc(len);
len = strlen(str) + 1;
if (!(copy = (char*)cJSON_malloc(len))) return 0;
if (!copy) return 0;
memcpy(copy,str,len);
return copy;
}
@@ -209,7 +208,15 @@ static char *print_string_ptr(const char *str)
const char *ptr;char *ptr2,*out;int len=0;unsigned char token;
if (!str) return cJSON_strdup("");
ptr=str;while ((token=*ptr) && ++len) {if (strchr("\"\\\b\f\n\r\t",token)) len++; else if (token<32) len+=5;ptr++;}
ptr=str;
token = *ptr;
while (token && ++len)
{
if (strchr("\"\\\b\f\n\r\t",token)) len++;
else if (token<32) len+=5;
ptr++;
token = *ptr;
}
out=(char*)cJSON_malloc(len+3);
if (!out) return 0;
@@ -325,8 +332,8 @@ static const char *parse_array(cJSON *item,const char *value)
while (*value==',')
{
cJSON *new_item;
if (!(new_item=cJSON_New_Item())) return 0; /* memory fail */
cJSON *new_item = cJSON_New_Item();
if (!new_item) return 0; /* memory fail */
child->next=new_item;new_item->prev=child;child=new_item;
value=skip(parse_value(child,skip(value+1)));
if (!value) return 0; /* memory fail */
@@ -415,8 +422,8 @@ static const char *parse_object(cJSON *item,const char *value)
while (*value==',')
{
cJSON *new_item;
if (!(new_item=cJSON_New_Item())) return 0; /* memory fail */
cJSON *new_item = cJSON_New_Item();
if (!new_item) return 0; /* memory fail */
child->next=new_item;new_item->prev=child;child=new_item;
value=skip(parse_string(child,skip(value+1)));
if (!value) return 0;
@@ -352,7 +352,6 @@ int GetOptions(int argc, char** argv, Options* psOptions)
{
int i;
int fullShaderChain = -1;
int hashOut = 0;
InitOptions(psOptions);
@@ -415,9 +414,6 @@ int GetOptions(int argc, char** argv, Options* psOptions)
uint64_t hash = hash64((const uint8_t*)psOptions->outputShaderFile, (uint32_t)strlen(psOptions->outputShaderFile), 0);
uint32_t high = (uint32_t)(hash >> 32);
uint32_t low = (uint32_t)(hash & 0x00000000FFFFFFFF);
dir = strrchr(psOptions->outputShaderFile, '\\');
if (!dir)
@@ -186,7 +186,6 @@ void MarkTextureAsShadow(ShaderInfo* psShaderInfo, Declaration* psDeclList, cons
{
(void)psShaderInfo;
ResourceBinding* psBinding = 0;
Declaration* psDecl = psDeclList;
uint32_t i;
@@ -733,11 +732,6 @@ const uint32_t* DecodeDeclaration(ShaderData* psShader, const uint32_t* pui32Tok
{
ui32TokenLength = pui32Token[1];
{
int iTupleSrc = 0, iTupleDest = 0;
//const uint32_t ui32ConstCount = pui32Token[1] - 2;
//const uint32_t ui32TupleCount = (ui32ConstCount / 4);
CUSTOMDATA_CLASS eClass = DecodeCustomDataClass(pui32Token[0]);
const uint32_t ui32NumVec4 = (ui32TokenLength - 2) / 4;
uint32_t uIdx = 0;
@@ -776,9 +770,6 @@ const uint32_t* DecodeDeclaration(ShaderData* psShader, const uint32_t* pui32Tok
}
case OPCODE_DCL_UNORDERED_ACCESS_VIEW_RAW:
{
ResourceBinding* psBinding = NULL;
ConstantBuffer* psBuffer = NULL;
psDecl->ui32NumOperands = 1;
psDecl->sUAV.ui32GloballyCoherentAccess = DecodeAccessCoherencyFlags(*pui32Token);
psDecl->sUAV.bCounter = 0;
@@ -830,9 +821,6 @@ const uint32_t* DecodeDeclaration(ShaderData* psShader, const uint32_t* pui32Tok
}
case OPCODE_DCL_THREAD_GROUP_SHARED_MEMORY_STRUCTURED:
{
ResourceBinding* psBinding = NULL;
ConstantBuffer* psBuffer = NULL;
psDecl->ui32NumOperands = 1;
psDecl->sUAV.ui32GloballyCoherentAccess = 0;
@@ -844,9 +832,6 @@ const uint32_t* DecodeDeclaration(ShaderData* psShader, const uint32_t* pui32Tok
}
case OPCODE_DCL_THREAD_GROUP_SHARED_MEMORY_RAW:
{
ResourceBinding* psBinding = NULL;
ConstantBuffer* psBuffer = NULL;
psDecl->ui32NumOperands = 1;
psDecl->sUAV.ui32GloballyCoherentAccess = 0;
@@ -1512,7 +1497,6 @@ void AllocateHullPhaseArrays(const uint32_t* pui32Tokens,
while (1) //Keep going until we reach the first non-declaration token, or the end of the shader.
{
uint32_t ui32TokenLength = DecodeInstructionLength(*pui32CurrentToken);
const uint32_t bExtended = DecodeIsOpcodeExtended(*pui32CurrentToken);
const OPCODE_TYPE eOpcode = DecodeOpcodeType(*pui32CurrentToken);
if (eOpcode == OPCODE_CUSTOMDATA)
@@ -408,8 +408,6 @@ static void SetupRegisterUsage(const ShaderData* psShader, const uint32_t ui32To
// In dx9 there is only one constant buffer per shader.
static void DeclareConstantBuffer(const ShaderData* psShader, Declaration* psDecl)
{
DECLUSAGE_DX9 eUsage = (DECLUSAGE_DX9)0;
uint32_t ui32UsageIndex = 0;
// Pick any constant register in the table. Might not start at c0 (e.g. when register(cX) is used).
uint32_t ui32RegNum = psShader->sInfo.psConstantBuffers->asVars[0].ui32StartOffset / 16;
OPERAND_TYPE_DX9 ui32RegType = OPERAND_TYPE_DX9_CONST;
@@ -447,9 +445,7 @@ static void DeclareConstantBuffer(const ShaderData* psShader, Declaration* psDec
static void DecodeDeclarationDX9(const ShaderData* psShader, const uint32_t ui32Token0, const uint32_t ui32Token1, Declaration* psDecl)
{
DECLUSAGE_DX9 eUsage = DecodeUsageDX9(ui32Token0);
uint32_t ui32UsageIndex = DecodeUsageIndexDX9(ui32Token0);
uint32_t ui32RegNum = DecodeOperandRegisterNumberDX9(ui32Token1);
/*uint32_t ui32UsageIndex = DecodeUsageIndexDX9(ui32Token0);*/
uint32_t ui32RegType = DecodeOperandTypeDX9(ui32Token1);
if (psShader->eShaderType == VERTEX_SHADER)
@@ -9,7 +9,29 @@
#include <malloc.h>
#endif
void* (*hlslcc_malloc)(size_t size) = malloc;
void* (*hlslcc_calloc)(size_t num,size_t size) = calloc;
void (*hlslcc_free)(void *p) = free;
void* (*hlslcc_realloc)(void *p,size_t size) = realloc;
// Wrapping these functions since we are taking the address of them and the std functions are dllimport which produce
// warning C4232
void* std_malloc(size_t size)
{
return malloc(size);
}
void* std_calloc(size_t num, size_t size)
{
return calloc(num, size);
}
void std_free(void* p)
{
free(p);
}
void* std_realloc(void* p, size_t size)
{
return realloc(p, size);
}
void* (*hlslcc_malloc)(size_t size) = std_malloc;
void* (*hlslcc_calloc)(size_t num,size_t size) = std_calloc;
void (*hlslcc_free)(void *p) = std_free;
void* (*hlslcc_realloc)(void *p,size_t size) = std_realloc;
@@ -12,4 +12,4 @@ extern void* (*hlslcc_realloc)(void *p,size_t size);
#define bstr__alloc hlslcc_malloc
#define bstr__free hlslcc_free
#define bstr__realloc hlslcc_realloc
#endif
#endif

Some files were not shown because too many files have changed in this diff Show More