From 9ee9730294bb1f4b4a260a90c5f551cb8c42549d Mon Sep 17 00:00:00 2001 From: AMZN-stankowi <4838196+AMZN-stankowi@users.noreply.github.com> Date: Mon, 2 Aug 2021 10:57:57 -0700 Subject: [PATCH] Automated test for scene files with and without python scripts running python incorrectly (#2373) * Cleared m_scriptFilename between scene files. This fixes a bug where a Python script file would be run on a scene file that didn't have a script file set. Added a general case version to SceneBuilderWorker.cpp, to make it easy to mark all scene files as dirty. Automated tests for this will come in a separate pull request. Signed-off-by: stankowi <4838196+AMZN-stankowi@users.noreply.github.com> * Work in progress automated tests Signed-off-by: stankowi <4838196+AMZN-stankowi@users.noreply.github.com> * Python test done Signed-off-by: stankowi <4838196+AMZN-stankowi@users.noreply.github.com> * Sorted jobs work now. This may sort too aggressively, I'll remove the additional sorting after some testing. Signed-off-by: stankowi <4838196+AMZN-stankowi@users.noreply.github.com> * Cleaned up test Signed-off-by: stankowi <4838196+AMZN-stankowi@users.noreply.github.com> * Fixed stray ' Signed-off-by: stankowi <4838196+AMZN-stankowi@users.noreply.github.com> * Removed temp code from test Signed-off-by: stankowi <4838196+AMZN-stankowi@users.noreply.github.com> * Command line help options for AP Removed job sorting that wasn't actually sorting jobs Signed-off-by: stankowi <4838196+AMZN-stankowi@users.noreply.github.com> * Changed constant variable names to match coding standards Signed-off-by: stankowi <4838196+AMZN-stankowi@users.noreply.github.com> --- .../PythonTests/assetpipeline/CMakeLists.txt | 1 + .../assetpipeline/fbx_tests/CMakeLists.txt | 21 ++++ .../a_simple_box_with_script.fbx | 3 + .../a_simple_box_with_script.fbx.assetinfo | 9 ++ .../b_simple_box_no_script.fbx | 3 + .../b_simple_box_no_script.fbx.assetinfo | 15 +++ .../python_builder.py | 45 +++++++ .../fbx_tests/pythonassetbuildertests.py | 94 ++++++++++++++ .../AssetManager/assetProcessorManager.cpp | 6 +- .../AssetManager/assetProcessorManager.h | 5 + .../AssetProcessor/native/assetprocessor.h | 5 + .../resourcecompiler/RCQueueSortModel.cpp | 10 ++ .../resourcecompiler/RCQueueSortModel.h | 9 ++ .../native/resourcecompiler/rccontroller.cpp | 5 + .../native/resourcecompiler/rccontroller.h | 5 +- .../utilities/ApplicationManagerBase.cpp | 118 ++++++++++++++---- .../native/utilities/ApplicationManagerBase.h | 5 + 17 files changed, 329 insertions(+), 30 deletions(-) create mode 100644 AutomatedTesting/Gem/PythonTests/assetpipeline/fbx_tests/CMakeLists.txt create mode 100644 AutomatedTesting/Gem/PythonTests/assetpipeline/fbx_tests/assets/TwoSceneFiles_OneWithPythonOneWithout_PythonOnlyRunsOnFirstScene/a_simple_box_with_script.fbx create mode 100644 AutomatedTesting/Gem/PythonTests/assetpipeline/fbx_tests/assets/TwoSceneFiles_OneWithPythonOneWithout_PythonOnlyRunsOnFirstScene/a_simple_box_with_script.fbx.assetinfo create mode 100644 AutomatedTesting/Gem/PythonTests/assetpipeline/fbx_tests/assets/TwoSceneFiles_OneWithPythonOneWithout_PythonOnlyRunsOnFirstScene/b_simple_box_no_script.fbx create mode 100644 AutomatedTesting/Gem/PythonTests/assetpipeline/fbx_tests/assets/TwoSceneFiles_OneWithPythonOneWithout_PythonOnlyRunsOnFirstScene/b_simple_box_no_script.fbx.assetinfo create mode 100644 AutomatedTesting/Gem/PythonTests/assetpipeline/fbx_tests/assets/TwoSceneFiles_OneWithPythonOneWithout_PythonOnlyRunsOnFirstScene/python_builder.py create mode 100644 AutomatedTesting/Gem/PythonTests/assetpipeline/fbx_tests/pythonassetbuildertests.py diff --git a/AutomatedTesting/Gem/PythonTests/assetpipeline/CMakeLists.txt b/AutomatedTesting/Gem/PythonTests/assetpipeline/CMakeLists.txt index 1b42d0d871..29b30a12b4 100644 --- a/AutomatedTesting/Gem/PythonTests/assetpipeline/CMakeLists.txt +++ b/AutomatedTesting/Gem/PythonTests/assetpipeline/CMakeLists.txt @@ -7,6 +7,7 @@ # add_subdirectory(asset_processor_tests) +add_subdirectory(fbx_tests) if(PAL_TRAIT_BUILD_TESTS_SUPPORTED AND PAL_TRAIT_BUILD_HOST_TOOLS) ## AP Python Tests ## diff --git a/AutomatedTesting/Gem/PythonTests/assetpipeline/fbx_tests/CMakeLists.txt b/AutomatedTesting/Gem/PythonTests/assetpipeline/fbx_tests/CMakeLists.txt new file mode 100644 index 0000000000..4a26500ee2 --- /dev/null +++ b/AutomatedTesting/Gem/PythonTests/assetpipeline/fbx_tests/CMakeLists.txt @@ -0,0 +1,21 @@ +# +# Copyright (c) Contributors to the Open 3D Engine Project. +# For complete copyright and license terms please see the LICENSE at the root of this distribution. +# +# SPDX-License-Identifier: Apache-2.0 OR MIT +# +# + +if(PAL_TRAIT_BUILD_TESTS_SUPPORTED AND PAL_TRAIT_BUILD_HOST_TOOLS) + ly_add_pytest( + NAME SceneProcessingTests.PythonAssetBuilderTests + TEST_SUITE main + PATH ${CMAKE_CURRENT_LIST_DIR}/pythonassetbuildertests.py + PYTEST_MARKS "not SUITE_sandbox" # don't run sandbox tests in this file + EXCLUDE_TEST_RUN_TARGET_FROM_IDE + RUNTIME_DEPENDENCIES + AZ::AssetProcessorBatch + AZ::AssetProcessor + ) + +endif() diff --git a/AutomatedTesting/Gem/PythonTests/assetpipeline/fbx_tests/assets/TwoSceneFiles_OneWithPythonOneWithout_PythonOnlyRunsOnFirstScene/a_simple_box_with_script.fbx b/AutomatedTesting/Gem/PythonTests/assetpipeline/fbx_tests/assets/TwoSceneFiles_OneWithPythonOneWithout_PythonOnlyRunsOnFirstScene/a_simple_box_with_script.fbx new file mode 100644 index 0000000000..e31b4a96f2 --- /dev/null +++ b/AutomatedTesting/Gem/PythonTests/assetpipeline/fbx_tests/assets/TwoSceneFiles_OneWithPythonOneWithout_PythonOnlyRunsOnFirstScene/a_simple_box_with_script.fbx @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:f82aecb36faf5cf9f2730e5ad264db38a3a469f8f48aff9b74682d1a32b098f0 +size 11644 diff --git a/AutomatedTesting/Gem/PythonTests/assetpipeline/fbx_tests/assets/TwoSceneFiles_OneWithPythonOneWithout_PythonOnlyRunsOnFirstScene/a_simple_box_with_script.fbx.assetinfo b/AutomatedTesting/Gem/PythonTests/assetpipeline/fbx_tests/assets/TwoSceneFiles_OneWithPythonOneWithout_PythonOnlyRunsOnFirstScene/a_simple_box_with_script.fbx.assetinfo new file mode 100644 index 0000000000..07018ab521 --- /dev/null +++ b/AutomatedTesting/Gem/PythonTests/assetpipeline/fbx_tests/assets/TwoSceneFiles_OneWithPythonOneWithout_PythonOnlyRunsOnFirstScene/a_simple_box_with_script.fbx.assetinfo @@ -0,0 +1,9 @@ +{ + "values": + [ + { + "$type": "ScriptProcessorRule", + "scriptFilename": "TwoSceneFiles_OneWithPythonOneWithout_PythonOnlyRunsOnFirstScene/python_builder.py" + } + ] +} diff --git a/AutomatedTesting/Gem/PythonTests/assetpipeline/fbx_tests/assets/TwoSceneFiles_OneWithPythonOneWithout_PythonOnlyRunsOnFirstScene/b_simple_box_no_script.fbx b/AutomatedTesting/Gem/PythonTests/assetpipeline/fbx_tests/assets/TwoSceneFiles_OneWithPythonOneWithout_PythonOnlyRunsOnFirstScene/b_simple_box_no_script.fbx new file mode 100644 index 0000000000..e31b4a96f2 --- /dev/null +++ b/AutomatedTesting/Gem/PythonTests/assetpipeline/fbx_tests/assets/TwoSceneFiles_OneWithPythonOneWithout_PythonOnlyRunsOnFirstScene/b_simple_box_no_script.fbx @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:f82aecb36faf5cf9f2730e5ad264db38a3a469f8f48aff9b74682d1a32b098f0 +size 11644 diff --git a/AutomatedTesting/Gem/PythonTests/assetpipeline/fbx_tests/assets/TwoSceneFiles_OneWithPythonOneWithout_PythonOnlyRunsOnFirstScene/b_simple_box_no_script.fbx.assetinfo b/AutomatedTesting/Gem/PythonTests/assetpipeline/fbx_tests/assets/TwoSceneFiles_OneWithPythonOneWithout_PythonOnlyRunsOnFirstScene/b_simple_box_no_script.fbx.assetinfo new file mode 100644 index 0000000000..72d756d655 --- /dev/null +++ b/AutomatedTesting/Gem/PythonTests/assetpipeline/fbx_tests/assets/TwoSceneFiles_OneWithPythonOneWithout_PythonOnlyRunsOnFirstScene/b_simple_box_no_script.fbx.assetinfo @@ -0,0 +1,15 @@ +{ + "values": [ + { + "$type": "{07B356B7-3635-40B5-878A-FAC4EFD5AD86} MeshGroup", + "name": "b_simple_box_no_script", + "nodeSelectionList": { + "selectedNodes": [ + {}, + "RootNode", + "RootNode.Cube" + ] + } + } + ] +} \ No newline at end of file diff --git a/AutomatedTesting/Gem/PythonTests/assetpipeline/fbx_tests/assets/TwoSceneFiles_OneWithPythonOneWithout_PythonOnlyRunsOnFirstScene/python_builder.py b/AutomatedTesting/Gem/PythonTests/assetpipeline/fbx_tests/assets/TwoSceneFiles_OneWithPythonOneWithout_PythonOnlyRunsOnFirstScene/python_builder.py new file mode 100644 index 0000000000..7ad5894a86 --- /dev/null +++ b/AutomatedTesting/Gem/PythonTests/assetpipeline/fbx_tests/assets/TwoSceneFiles_OneWithPythonOneWithout_PythonOnlyRunsOnFirstScene/python_builder.py @@ -0,0 +1,45 @@ +""" +Copyright (c) Contributors to the Open 3D Engine Project. +For complete copyright and license terms please see the LICENSE at the root of this distribution. + +SPDX-License-Identifier: Apache-2.0 OR MIT +""" +import datetime, uuid, os +import azlmbr.scene as sceneApi +import azlmbr.scene.graph + +def output_test_data(scene): + source_filename = os.path.basename(scene.sourceFilename) + source_filename = source_filename.replace('.','_') + + log_output_file_name = f"{source_filename}.log" + + log_output_folder = os.path.dirname(scene.sourceFilename) + log_output_location = os.path.join(log_output_folder, log_output_file_name) + + # Saving a file to the temp folder is the easiest way to have this test communicate + # with the outer python test. + with open(log_output_location, "w") as f: + # Just write something to the file, but the filename is the main information + # used for the test. + f.write(f"scene.sourceFilename: {scene.sourceFilename}\n") + return True + +mySceneJobHandler = None + +def on_update_manifest(args): + scene = args[0] + result = output_test_data(scene) + global mySceneJobHandler + mySceneJobHandler.disconnect() + mySceneJobHandler = None + return result + +def main(): + global mySceneJobHandler + mySceneJobHandler = sceneApi.ScriptBuildingNotificationBusHandler() + mySceneJobHandler.connect() + mySceneJobHandler.add_callback('OnUpdateManifest', on_update_manifest) + +if __name__ == "__main__": + main() \ No newline at end of file diff --git a/AutomatedTesting/Gem/PythonTests/assetpipeline/fbx_tests/pythonassetbuildertests.py b/AutomatedTesting/Gem/PythonTests/assetpipeline/fbx_tests/pythonassetbuildertests.py new file mode 100644 index 0000000000..6a568349d2 --- /dev/null +++ b/AutomatedTesting/Gem/PythonTests/assetpipeline/fbx_tests/pythonassetbuildertests.py @@ -0,0 +1,94 @@ +""" +Copyright (c) Contributors to the Open 3D Engine Project. +For complete copyright and license terms please see the LICENSE at the root of this distribution. + +SPDX-License-Identifier: Apache-2.0 OR MIT +""" + +# Import builtin libraries +import pytest +import logging +import os +import stat + +# Import LyTestTools +from ly_test_tools.o3de.asset_processor import AssetProcessor +from ly_test_tools.o3de import asset_processor as asset_processor_utils +import ly_test_tools.environment.file_system as fs + +# Import fixtures +from ..ap_fixtures.asset_processor_fixture import asset_processor as asset_processor +from ..ap_fixtures.ap_setup_fixture import ap_setup_fixture as ap_setup_fixture + +# Import LyShared +from ly_test_tools.o3de.ap_log_parser import APLogParser, APOutputParser +import ly_test_tools.o3de.pipeline_utils as utils + +# Use the following logging pattern to hook all test logging together: +logger = logging.getLogger(__name__) +# Configuring the logging is done in ly_test_tools at the following location: +# ~/dev/Tools/LyTestTools/ly_test_tools/log/py_logging_util.py + +# Helper: variables we will use for parameter values in the test: +targetProjects = ["AutomatedTesting"] + +@pytest.fixture +def local_resources(request, workspace, ap_setup_fixture): + ap_setup_fixture["tests_dir"] = os.path.dirname(os.path.realpath(__file__)) + + +@pytest.mark.usefixtures("asset_processor") +@pytest.mark.usefixtures("ap_setup_fixture") +@pytest.mark.usefixtures("local_resources") +@pytest.mark.parametrize("project", targetProjects) +@pytest.mark.assetpipeline +@pytest.mark.SUITE_main +class TestsPythonAssetProcessing_APBatch(object): + + @pytest.mark.BAT + @pytest.mark.assetpipeline + def test_ProcessAssetWithoutScriptAfterAssetWithScript_ScriptOnlyRunsOnExpectedAsset(self, workspace, ap_setup_fixture, asset_processor): + # This is a regression test. The situation it's testing is, the Python script to run + # defined in scene manifest files was persisting in a single builder. So if + # that builder processed file a.fbx, then b.fbx, and a.fbx has a Python script to run, + # it was also running that Python script on b.fbx. + + asset_processor.prepare_test_environment(ap_setup_fixture["tests_dir"], "TwoSceneFiles_OneWithPythonOneWithout_PythonOnlyRunsOnFirstScene") + + asset_processor_extra_params = [ + # Disabling Atom assets disables most products, using the debugOutput flag ensures one product is output. + "--debugOutput", + # By default, if job priorities are equal, jobs run in an arbitrary order. This makes sure + # jobs are run by sorting on the database source name, so they run in the same order each time + # when this test is run. + "--sortJobsByDBSourceName", + # Disabling Atom products means this asset won't need a lot of source dependencies to be processed, + # keeping the scope of this test down. + "--regset=\"/O3DE/SceneAPI/AssetImporter/SkipAtomOutput=true\"", + # The bug this regression test happened when the same builder processed FBX files with and without Python. + # This flag ensures that only one builder is launched, so that situation can be replicated. + "--regset=\"/Amazon/AssetProcessor/Settings/Jobs/maxJobs=1\""] + + result, _ = asset_processor.batch_process(extra_params=asset_processor_extra_params) + assert result, "AP Batch failed" + + expected_product_list = [ + "a_simple_box_with_script.dbgsg", + "b_simple_box_no_script.dbgsg" + ] + + missing_assets, _ = utils.compare_assets_with_cache(expected_product_list, + asset_processor.project_test_cache_folder()) + assert not missing_assets, f'The following assets were expected to be in, but not found in cache: {str(missing_assets)}' + + # The Python script loaded in the scene manifest will write a log file with the source file's name + # to the temp folder. This is the easiest way to have the internal Python there communicate with this test. + expected_path = os.path.join(asset_processor.project_test_source_folder(), "a_simple_box_with_script_fbx.log") + unexpected_path = os.path.join(asset_processor.project_test_source_folder(), "b_simple_box_no_script_fbx.log") + + # Simple check to make sure the Python script in the scene manifest ran on the file it should have ran on. + assert os.path.exists(expected_path), f"Did not find expected output test asset {expected_path}" + # If this test fails here, it means the Python script from the first processed FBX file is being run + # on the second FBX file, when it should not be. + assert not os.path.exists(unexpected_path), f"Found unexpected output test asset {unexpected_path}" + diff --git a/Code/Tools/AssetProcessor/native/AssetManager/assetProcessorManager.cpp b/Code/Tools/AssetProcessor/native/AssetManager/assetProcessorManager.cpp index 21089ef020..3a7bc6ef3e 100644 --- a/Code/Tools/AssetProcessor/native/AssetManager/assetProcessorManager.cpp +++ b/Code/Tools/AssetProcessor/native/AssetManager/assetProcessorManager.cpp @@ -3072,6 +3072,7 @@ namespace AssetProcessor QElapsedTimer elapsedTimer; elapsedTimer.start(); + for (auto jobIter = m_jobsToProcess.begin(); jobIter != m_jobsToProcess.end();) { JobDetails& job = *jobIter; @@ -3082,7 +3083,7 @@ namespace AssetProcessor jobIter = m_jobsToProcess.erase(jobIter); m_numOfJobsToAnalyze--; - // Update the remaining job status occasionally + // Update the remaining job status occasionally if (elapsedTimer.elapsed() >= MILLISECONDS_BETWEEN_PROCESS_JOBS_STATUS_UPDATE) { Q_EMIT NumRemainingJobsChanged(m_activeFiles.size() + m_filesToExamine.size() + m_numOfJobsToAnalyze); @@ -3102,7 +3103,8 @@ namespace AssetProcessor // Process the first job if no jobs were analyzed. auto jobIter = m_jobsToProcess.begin(); JobDetails& job = *jobIter; - AZ_Warning(AssetProcessor::DebugChannel, false, " Cyclic job dependency detected. Processing job (%s, %s, %s, %s) to unblock.", + AZ_Warning( + AssetProcessor::DebugChannel, false, " Cyclic job dependency detected. Processing job (%s, %s, %s, %s) to unblock.", job.m_jobEntry.m_databaseSourceName.toUtf8().data(), job.m_jobEntry.m_jobKey.toUtf8().data(), job.m_jobEntry.m_platformInfo.m_identifier.c_str(), job.m_jobEntry.m_builderGuid.ToString().c_str()); ProcessJob(job); diff --git a/Code/Tools/AssetProcessor/native/AssetManager/assetProcessorManager.h b/Code/Tools/AssetProcessor/native/AssetManager/assetProcessorManager.h index 5fc7a73035..376af5d773 100644 --- a/Code/Tools/AssetProcessor/native/AssetManager/assetProcessorManager.h +++ b/Code/Tools/AssetProcessor/native/AssetManager/assetProcessorManager.h @@ -207,6 +207,11 @@ namespace AssetProcessor //! or a job dependency and we can only resolve these dependencies once all the create jobs are completed. struct JobToProcessEntry { + bool operator<(const JobToProcessEntry& other) + { + return m_sourceFileInfo.m_pathRelativeToScanFolder < other.m_sourceFileInfo.m_pathRelativeToScanFolder; + } + SourceFileInfo m_sourceFileInfo; AZStd::vector m_jobsToAnalyze; // a vector of pairs of diff --git a/Code/Tools/AssetProcessor/native/assetprocessor.h b/Code/Tools/AssetProcessor/native/assetprocessor.h index 2397f3b1ff..1c13eca200 100644 --- a/Code/Tools/AssetProcessor/native/assetprocessor.h +++ b/Code/Tools/AssetProcessor/native/assetprocessor.h @@ -244,6 +244,11 @@ namespace AssetProcessor m_jobEntry.m_builderGuid == rhs.m_jobEntry.m_builderGuid); } + static bool DatabaseSourceLexCompare(const JobDetails& left, const JobDetails& right) + { + return left.m_jobEntry.m_databaseSourceName <= right.m_jobEntry.m_databaseSourceName; + } + JobDetails() = default; }; diff --git a/Code/Tools/AssetProcessor/native/resourcecompiler/RCQueueSortModel.cpp b/Code/Tools/AssetProcessor/native/resourcecompiler/RCQueueSortModel.cpp index c39a0c66e9..774876234d 100644 --- a/Code/Tools/AssetProcessor/native/resourcecompiler/RCQueueSortModel.cpp +++ b/Code/Tools/AssetProcessor/native/resourcecompiler/RCQueueSortModel.cpp @@ -197,10 +197,20 @@ namespace AssetProcessor { return priorityLeft > priorityRight; } + + // Optionally stabilize queue order on the source name. + // This is used in automated tests, to allow tests to have a stable + // order that jobs with otherwise equal priority run, so tests process + // assets in the same order each time they are run. + if (m_sortQueueOnDBSourceName) + { + return leftJob->GetJobEntry().m_databaseSourceName < rightJob->GetJobEntry().m_databaseSourceName; + } // if we get all the way down here it means we're dealing with two assets which are not // in any compile groups, not a priority platform, not a priority type, priority platform, etc. // we can arrange these any way we want, but must pick at least a stable order. + return leftJob->GetJobEntry().m_jobRunKey < rightJob->GetJobEntry().m_jobRunKey; } diff --git a/Code/Tools/AssetProcessor/native/resourcecompiler/RCQueueSortModel.h b/Code/Tools/AssetProcessor/native/resourcecompiler/RCQueueSortModel.h index fbeacd8b8c..7fc60fdd60 100644 --- a/Code/Tools/AssetProcessor/native/resourcecompiler/RCQueueSortModel.h +++ b/Code/Tools/AssetProcessor/native/resourcecompiler/RCQueueSortModel.h @@ -50,6 +50,10 @@ namespace AssetProcessor void AddJobIdEntry(AssetProcessor::RCJob* rcJob); void RemoveJobIdEntry(AssetProcessor::RCJob* rcJob); + void SetQueueSortOnDBSourceName() + { + m_sortQueueOnDBSourceName = true; + } // implement QSortFilteRProxyModel: bool filterAcceptsRow(int source_row, const QModelIndex& source_parent) const override; @@ -68,6 +72,11 @@ namespace AssetProcessor QSet m_currentlyConnectedPlatforms; bool m_dirtyNeedsResort = false; // instead of constantly resorting, we resort only when someone wants to pull an element from us + // By default, jobs with equal priority and escalation sort on the job run key. This flag changes + // jobs to sort on the database source name. This is used for testing, to guarantee jobs run in the same + // order for those tests each time they are run. + bool m_sortQueueOnDBSourceName = false; + // --------------------------------------------------------- // AssetProcessorPlatformBus::Handler void AssetProcessorPlatformConnected(const AZStd::string platform) override; diff --git a/Code/Tools/AssetProcessor/native/resourcecompiler/rccontroller.cpp b/Code/Tools/AssetProcessor/native/resourcecompiler/rccontroller.cpp index 11ef68de29..660144095b 100644 --- a/Code/Tools/AssetProcessor/native/resourcecompiler/rccontroller.cpp +++ b/Code/Tools/AssetProcessor/native/resourcecompiler/rccontroller.cpp @@ -163,6 +163,11 @@ namespace AssetProcessor return ((!m_RCQueueSortModel.GetNextPendingJob()) && (m_RCJobListModel.jobsInFlight() == 0)); } + void RCController::SetQueueSortOnDBSourceName() + { + m_RCQueueSortModel.SetQueueSortOnDBSourceName(); + } + void RCController::JobSubmitted(JobDetails details) { AssetProcessor::QueueElementID checkFile(details.m_jobEntry.m_databaseSourceName, details.m_jobEntry.m_platformInfo.m_identifier.c_str(), details.m_jobEntry.m_jobKey); diff --git a/Code/Tools/AssetProcessor/native/resourcecompiler/rccontroller.h b/Code/Tools/AssetProcessor/native/resourcecompiler/rccontroller.h index c555c7a585..51ae6b4a26 100644 --- a/Code/Tools/AssetProcessor/native/resourcecompiler/rccontroller.h +++ b/Code/Tools/AssetProcessor/native/resourcecompiler/rccontroller.h @@ -54,10 +54,11 @@ namespace AssetProcessor void StartJob(AssetProcessor::RCJob* rcJob); int NumberOfPendingCriticalJobsPerPlatform(QString platform); - void SetSystemRoot(const QDir& systemRoot); int NumberOfPendingJobsPerPlatform(QString platform); bool IsIdle(); - bool IsPriorityCopyJob(AssetProcessor::RCJob* rcJob); + + void SetQueueSortOnDBSourceName(); + Q_SIGNALS: void FileCompiled(JobEntry entry, AssetBuilderSDK::ProcessJobResponse response); void FileFailed(JobEntry entry); diff --git a/Code/Tools/AssetProcessor/native/utilities/ApplicationManagerBase.cpp b/Code/Tools/AssetProcessor/native/utilities/ApplicationManagerBase.cpp index 5e2de8e4cd..b90364e1c5 100644 --- a/Code/Tools/AssetProcessor/native/utilities/ApplicationManagerBase.cpp +++ b/Code/Tools/AssetProcessor/native/utilities/ApplicationManagerBase.cpp @@ -49,8 +49,6 @@ static const qint64 s_ReservedDiskSpaceInBytes = 256 * 1024; //! Maximum number of temp folders allowed static const int s_MaximumTempFolders = 10000; -const char AdditionalScanFolders[] = "additionalScanFolders"; - ApplicationManagerBase::ApplicationManagerBase(int* argc, char*** argv, QObject* parent) : ApplicationManager(argc, argv, parent) { @@ -155,55 +153,90 @@ void ApplicationManagerBase::InitAssetProcessorManager() const AzFramework::CommandLine* commandLine = nullptr; AzFramework::ApplicationRequests::Bus::BroadcastResult(commandLine, &AzFramework::ApplicationRequests::GetCommandLine); - if(commandLine->HasSwitch("zeroAnalysisMode")) + struct APCommandLineSwitch + { + APCommandLineSwitch(const char* switchTitle, const char* helpText) + : m_switch(switchTitle) + , m_helpText(helpText) + { + + } + const char* m_switch; + const char* m_helpText; + }; + + const APCommandLineSwitch Command_waitOnLaunch("waitOnLaunch", "Briefly pauses Asset Processor during initializiation. Useful if you want to attach a debugger."); + const APCommandLineSwitch Command_zeroAnalysisMode("zeroAnalysisMode", "Enables using file modification time when examining source assets for processing."); + const APCommandLineSwitch Command_enableQueryLogging("enableQueryLogging", "Enables logging database queries."); + const APCommandLineSwitch Command_dependencyScanPattern("dependencyScanPattern", "Scans assets that match the given pattern for missing product dependencies."); + const APCommandLineSwitch Command_dsp("dsp", Command_dependencyScanPattern.m_helpText); + const APCommandLineSwitch Command_fileDependencyScanPattern("fileDependencyScanPattern", "Used with dependencyScanPattern to farther filter the scan."); + const APCommandLineSwitch Command_fdsp("fdsp", Command_fileDependencyScanPattern.m_helpText); + const APCommandLineSwitch Command_additionalScanFolders("additionalScanFolders", "Used with dependencyScanPattern to farther filter the scan."); + const APCommandLineSwitch Command_dependencyScanMaxIteration("dependencyScanMaxIteration", "Used to limit the number of recursive searches per line when running dependencyScanPattern."); + const APCommandLineSwitch Command_warningLevel("warningLevel", "Configure the error and warning reporting level for AssetProcessor. Pass in 1 for fatal errors, 2 for fatal errors and warnings."); + const APCommandLineSwitch Command_acceptInput("acceptInput", "Enable external control messaging via the ControlRequestHandler, used with automated tests."); + const APCommandLineSwitch Command_debugOutput("debugOutput", "When enabled, builders that support it will output debug information as product assets. Used primarily with scene files."); + const APCommandLineSwitch Command_sortJobsByDBSourceName("sortJobsByDBSourceName", "When enabled, sorts pending jobs with equal priority and dependencies by database source name instead of job ID. Useful for automated tests to process assets in the same order each time."); + const APCommandLineSwitch Command_truncatefingerprint("truncatefingerprint", "Truncates the fingerprint used for processed assets. Useful if you plan to compress product assets to share on another machine because some compression formats like zip will truncate file mod timestamps."); + const APCommandLineSwitch Command_help("help", "Displays this message."); + const APCommandLineSwitch Command_h("h", Command_help.m_helpText); + + if (commandLine->HasSwitch(Command_waitOnLaunch.m_switch)) + { + // Useful for attaching the debugger, this forces a short pause. + AZStd::this_thread::sleep_for(AZStd::chrono::seconds(20)); + } + + if (commandLine->HasSwitch(Command_zeroAnalysisMode.m_switch)) { m_assetProcessorManager->SetEnableModtimeSkippingFeature(true); } - if(commandLine->HasSwitch("enableQueryLogging")) + if (commandLine->HasSwitch(Command_enableQueryLogging.m_switch)) { m_assetProcessorManager->SetQueryLogging(true); } - if (commandLine->HasSwitch("dependencyScanPattern")) + if (commandLine->HasSwitch(Command_dependencyScanPattern.m_switch)) { - m_dependencyScanPattern = commandLine->GetSwitchValue("dependencyScanPattern", 0).c_str(); + m_dependencyScanPattern = commandLine->GetSwitchValue(Command_dependencyScanPattern.m_switch, 0).c_str(); } - else if (commandLine->HasSwitch("dsp")) + else if (commandLine->HasSwitch(Command_dsp.m_switch)) { - m_dependencyScanPattern = commandLine->GetSwitchValue("dsp", 0).c_str(); + m_dependencyScanPattern = commandLine->GetSwitchValue(Command_dsp.m_switch, 0).c_str(); } m_fileDependencyScanPattern = "*"; - if (commandLine->HasSwitch("fileDependencyScanPattern")) + if (commandLine->HasSwitch(Command_fileDependencyScanPattern.m_switch)) { - m_fileDependencyScanPattern = commandLine->GetSwitchValue("fileDependencyScanPattern", 0).c_str(); + m_fileDependencyScanPattern = commandLine->GetSwitchValue(Command_fileDependencyScanPattern.m_switch, 0).c_str(); } - else if (commandLine->HasSwitch("fdsp")) + else if (commandLine->HasSwitch(Command_fdsp.m_switch)) { - m_fileDependencyScanPattern = commandLine->GetSwitchValue("fdsp", 0).c_str(); + m_fileDependencyScanPattern = commandLine->GetSwitchValue(Command_fdsp.m_switch, 0).c_str(); } - if (commandLine->HasSwitch(AdditionalScanFolders)) + if (commandLine->HasSwitch(Command_additionalScanFolders.m_switch)) { - for (size_t idx = 0; idx < commandLine->GetNumSwitchValues(AdditionalScanFolders); idx++) + for (size_t idx = 0; idx < commandLine->GetNumSwitchValues(Command_additionalScanFolders.m_switch); idx++) { - AZStd::string value = commandLine->GetSwitchValue(AdditionalScanFolders, idx); + AZStd::string value = commandLine->GetSwitchValue(Command_additionalScanFolders.m_switch, idx); m_dependencyAddtionalScanFolders.emplace_back(AZStd::move(value)); } } - if (commandLine->HasSwitch("dependencyScanMaxIteration")) + if (commandLine->HasSwitch(Command_dependencyScanMaxIteration.m_switch)) { - AZStd::string maxIterationAsString = commandLine->GetSwitchValue("dependencyScanMaxIteration", 0); + AZStd::string maxIterationAsString = commandLine->GetSwitchValue(Command_dependencyScanMaxIteration.m_switch, 0); m_dependencyScanMaxIteration = AZStd::stoi(maxIterationAsString); } - if (commandLine->HasSwitch("warningLevel")) + if (commandLine->HasSwitch(Command_warningLevel.m_switch)) { using namespace AssetProcessor; - const AZStd::string& levelString = commandLine->GetSwitchValue("warningLevel", 0); + const AZStd::string& levelString = commandLine->GetSwitchValue(Command_warningLevel.m_switch, 0); WarningLevel warningLevel = WarningLevel::Default; switch(AZStd::stoi(levelString)) @@ -217,26 +250,30 @@ void ApplicationManagerBase::InitAssetProcessorManager() } AssetProcessor::JobDiagnosticRequestBus::Broadcast(&AssetProcessor::JobDiagnosticRequestBus::Events::SetWarningLevel, warningLevel); } - if (commandLine->HasSwitch("acceptInput")) + if (commandLine->HasSwitch(Command_acceptInput.m_switch)) { InitControlRequestHandler(); } - if (commandLine->HasSwitch("debugOutput")) + if (commandLine->HasSwitch(Command_debugOutput.m_switch)) { m_assetProcessorManager->SetBuilderDebugFlag(true); } - constexpr char truncateFingerprintSwitch[] = "truncatefingerprint"; - if(commandLine->HasSwitch(truncateFingerprintSwitch)) + if (commandLine->HasSwitch(Command_sortJobsByDBSourceName.m_switch)) + { + m_sortJobsByDBSourceName = true; + } + + if (commandLine->HasSwitch(Command_truncatefingerprint.m_switch)) { // Zip archive format uses 2 second precision truncated const int ArchivePrecision = 2000; int precision = ArchivePrecision; - if(commandLine->GetNumSwitchValues(truncateFingerprintSwitch) > 0) + if (commandLine->GetNumSwitchValues(Command_truncatefingerprint.m_switch) > 0) { - precision = AZStd::stoi(commandLine->GetSwitchValue(truncateFingerprintSwitch, 0)); + precision = AZStd::stoi(commandLine->GetSwitchValue(Command_truncatefingerprint.m_switch, 0)); if(precision < 1) { @@ -246,6 +283,31 @@ void ApplicationManagerBase::InitAssetProcessorManager() AssetUtilities::SetTruncateFingerprintTimestamp(precision); } + + if (commandLine->HasSwitch(Command_help.m_switch) || commandLine->HasSwitch(Command_h.m_switch)) + { + // Other O3DE tools have a more full featured system for registering command flags + // that includes help output, but right now the AssetProcessor just checks strings + // via HasSwitch. This means this help output has to be updated manually. + AZ_TracePrintf("AssetProcessor", "Asset Processor Command Line Flags:\n"); + AZ_TracePrintf("AssetProcessor", "\t%s : %s\n", Command_waitOnLaunch.m_switch, Command_waitOnLaunch.m_helpText); + AZ_TracePrintf("AssetProcessor", "\t%s : %s\n", Command_zeroAnalysisMode.m_switch, Command_zeroAnalysisMode.m_helpText); + AZ_TracePrintf("AssetProcessor", "\t%s : %s\n", Command_enableQueryLogging.m_switch, Command_enableQueryLogging.m_helpText); + AZ_TracePrintf("AssetProcessor", "\t%s : %s\n", Command_dependencyScanPattern.m_switch, Command_dependencyScanPattern.m_helpText); + AZ_TracePrintf("AssetProcessor", "\t%s : %s\n", Command_dsp.m_switch, Command_dsp.m_helpText); + AZ_TracePrintf("AssetProcessor", "\t%s : %s\n", Command_fileDependencyScanPattern.m_switch, Command_fileDependencyScanPattern.m_helpText); + AZ_TracePrintf("AssetProcessor", "\t%s : %s\n", Command_fdsp.m_switch, Command_fdsp.m_helpText); + AZ_TracePrintf("AssetProcessor", "\t%s : %s\n", Command_additionalScanFolders.m_switch, Command_additionalScanFolders.m_helpText); + AZ_TracePrintf("AssetProcessor", "\t%s : %s\n", Command_dependencyScanMaxIteration.m_switch, Command_dependencyScanMaxIteration.m_helpText); + AZ_TracePrintf("AssetProcessor", "\t%s : %s\n", Command_warningLevel.m_switch, Command_warningLevel.m_helpText); + AZ_TracePrintf("AssetProcessor", "\t%s : %s\n", Command_acceptInput.m_switch, Command_acceptInput.m_helpText); + AZ_TracePrintf("AssetProcessor", "\t%s : %s\n", Command_debugOutput.m_switch, Command_debugOutput.m_helpText); + AZ_TracePrintf("AssetProcessor", "\t%s : %s\n", Command_sortJobsByDBSourceName.m_switch, Command_sortJobsByDBSourceName.m_helpText); + AZ_TracePrintf("AssetProcessor", "\t%s : %s\n", Command_truncatefingerprint.m_switch, Command_truncatefingerprint.m_helpText); + AZ_TracePrintf("AssetProcessor", "\t%s : %s\n", Command_help.m_switch, Command_help.m_helpText); + AZ_TracePrintf("AssetProcessor", "\t%s : %s\n", Command_h.m_switch, Command_h.m_helpText); + AZ_TracePrintf("AssetProcessor", "\tregset : set the given registry key to the given value.\n"); + } } void ApplicationManagerBase::Rescan() @@ -281,6 +343,11 @@ void ApplicationManagerBase::InitRCController() { m_rcController = new AssetProcessor::RCController(m_platformConfiguration->GetMinJobs(), m_platformConfiguration->GetMaxJobs()); + if (m_sortJobsByDBSourceName) + { + m_rcController->SetQueueSortOnDBSourceName(); + } + QObject::connect(m_assetProcessorManager, &AssetProcessor::AssetProcessorManager::AssetToProcess, m_rcController, &AssetProcessor::RCController::JobSubmitted); QObject::connect(m_rcController, &AssetProcessor::RCController::FileCompiled, m_assetProcessorManager, &AssetProcessor::AssetProcessorManager::AssetProcessed, Qt::UniqueConnection); QObject::connect(m_rcController, &AssetProcessor::RCController::FileFailed, m_assetProcessorManager, &AssetProcessor::AssetProcessorManager::AssetFailed); @@ -1807,4 +1874,3 @@ void ApplicationManagerBase::OnActiveJobsCountChanged(unsigned int count) AssetProcessor::AssetProcessorStatusEntry entry(AssetProcessor::AssetProcessorStatus::Processing_Jobs, count); Q_EMIT AssetProcessorStatusChanged(entry); } - diff --git a/Code/Tools/AssetProcessor/native/utilities/ApplicationManagerBase.h b/Code/Tools/AssetProcessor/native/utilities/ApplicationManagerBase.h index 40106c69ec..7e6347b4d1 100644 --- a/Code/Tools/AssetProcessor/native/utilities/ApplicationManagerBase.h +++ b/Code/Tools/AssetProcessor/native/utilities/ApplicationManagerBase.h @@ -236,6 +236,11 @@ protected: int m_remainingAPMJobs = 0; bool m_assetProcessorManagerIsReady = false; + // When job priority and escalation is equal, jobs sort in order by job key. + // This switches that behavior to instead sort by the DB source name, which + // allows automated tests to get deterministic behavior out of Asset Processor. + bool m_sortJobsByDBSourceName = false; + unsigned int m_highestConnId = 0; AzToolsFramework::Ticker* m_ticker = nullptr; // for ticking the tickbus.