Merge branch 'development' into cmake/SPEC-7484

Signed-off-by: Esteban Papp <81431996+amznestebanpapp@users.noreply.github.com>

# Conflicts:
#	Code/Editor/ToolBox.cpp

Code/Tools/AssetProcessor/native/AssetManager/assetProcessorManager.cpp

@@ -3072,6 +3072,7 @@ namespace AssetProcessor
 
         QElapsedTimer elapsedTimer;
         elapsedTimer.start();
+
         for (auto jobIter = m_jobsToProcess.begin(); jobIter != m_jobsToProcess.end();)
         {
             JobDetails& job = *jobIter;
@@ -3082,7 +3083,7 @@ namespace AssetProcessor
                 jobIter = m_jobsToProcess.erase(jobIter);
                 m_numOfJobsToAnalyze--;
 
-                // Update the remaining job status occasionally 
+                // Update the remaining job status occasionally
                 if (elapsedTimer.elapsed() >= MILLISECONDS_BETWEEN_PROCESS_JOBS_STATUS_UPDATE)
                 {
                     Q_EMIT NumRemainingJobsChanged(m_activeFiles.size() + m_filesToExamine.size() + m_numOfJobsToAnalyze);
@@ -3102,7 +3103,8 @@ namespace AssetProcessor
                 // Process the first job if no jobs were analyzed.
                 auto jobIter = m_jobsToProcess.begin();
                 JobDetails& job = *jobIter;
-                AZ_Warning(AssetProcessor::DebugChannel, false, " Cyclic job dependency detected. Processing job (%s, %s, %s, %s) to unblock.", 
+                AZ_Warning(
+                    AssetProcessor::DebugChannel, false, " Cyclic job dependency detected. Processing job (%s, %s, %s, %s) to unblock.",
                     job.m_jobEntry.m_databaseSourceName.toUtf8().data(), job.m_jobEntry.m_jobKey.toUtf8().data(),
                     job.m_jobEntry.m_platformInfo.m_identifier.c_str(), job.m_jobEntry.m_builderGuid.ToString<AZStd::string>().c_str());
                 ProcessJob(job);

Code/Tools/AssetProcessor/native/AssetManager/assetProcessorManager.h

@@ -207,6 +207,11 @@ namespace AssetProcessor
         //! or a job dependency and we can only resolve these dependencies once all the create jobs are completed.
         struct JobToProcessEntry
         {
+            bool operator<(const JobToProcessEntry& other)
+            {
+                return m_sourceFileInfo.m_pathRelativeToScanFolder < other.m_sourceFileInfo.m_pathRelativeToScanFolder;
+            }
+
             SourceFileInfo m_sourceFileInfo;
             AZStd::vector<JobDetails> m_jobsToAnalyze;
             // a vector of pairs of <builder which emitted it, the dependency>

Code/Tools/AssetProcessor/native/assetprocessor.h

@@ -244,6 +244,11 @@ namespace AssetProcessor
                 m_jobEntry.m_builderGuid == rhs.m_jobEntry.m_builderGuid);
         }
 
+        static bool DatabaseSourceLexCompare(const JobDetails& left, const JobDetails& right)
+        {
+            return left.m_jobEntry.m_databaseSourceName <= right.m_jobEntry.m_databaseSourceName;
+        }
+
         JobDetails() = default;
     };
  

Code/Tools/AssetProcessor/native/resourcecompiler/RCQueueSortModel.cpp

@@ -197,10 +197,20 @@ namespace AssetProcessor
         {
             return priorityLeft > priorityRight;
         }
+
+        // Optionally stabilize queue order on the source name.
+        // This is used in automated tests, to allow tests to have a stable
+        // order that jobs with otherwise equal priority run, so tests process
+        // assets in the same order each time they are run.
+        if (m_sortQueueOnDBSourceName)
+        {
+            return leftJob->GetJobEntry().m_databaseSourceName < rightJob->GetJobEntry().m_databaseSourceName;
+        }
         
         // if we get all the way down here it means we're dealing with two assets which are not
         // in any compile groups, not a priority platform, not a priority type, priority platform, etc.
         // we can arrange these any way we want, but must pick at least a stable order.
+
         return leftJob->GetJobEntry().m_jobRunKey < rightJob->GetJobEntry().m_jobRunKey;
     }
 

Code/Tools/AssetProcessor/native/resourcecompiler/RCQueueSortModel.h

@@ -50,6 +50,10 @@ namespace AssetProcessor
         void AddJobIdEntry(AssetProcessor::RCJob* rcJob);
         void RemoveJobIdEntry(AssetProcessor::RCJob* rcJob);
 
+        void SetQueueSortOnDBSourceName()
+        {
+            m_sortQueueOnDBSourceName = true;
+        }
 
         // implement QSortFilteRProxyModel:
         bool filterAcceptsRow(int source_row, const QModelIndex& source_parent) const override;
@@ -68,6 +72,11 @@ namespace AssetProcessor
         QSet<QString> m_currentlyConnectedPlatforms;
         bool m_dirtyNeedsResort = false; // instead of constantly resorting, we resort only when someone wants to pull an element from us
 
+        // By default, jobs with equal priority and escalation sort on the job run key. This flag changes
+        // jobs to sort on the database source name. This is used for testing, to guarantee jobs run in the same
+        // order for those tests each time they are run.
+        bool m_sortQueueOnDBSourceName = false;
+
         // ---------------------------------------------------------
         // AssetProcessorPlatformBus::Handler
         void AssetProcessorPlatformConnected(const AZStd::string platform) override;

Code/Tools/AssetProcessor/native/resourcecompiler/rccontroller.cpp

@@ -163,6 +163,11 @@ namespace AssetProcessor
         return ((!m_RCQueueSortModel.GetNextPendingJob()) && (m_RCJobListModel.jobsInFlight() == 0));
     }
 
+    void RCController::SetQueueSortOnDBSourceName()
+    {
+        m_RCQueueSortModel.SetQueueSortOnDBSourceName();
+    }
+
     void RCController::JobSubmitted(JobDetails details)
     {
         AssetProcessor::QueueElementID checkFile(details.m_jobEntry.m_databaseSourceName, details.m_jobEntry.m_platformInfo.m_identifier.c_str(), details.m_jobEntry.m_jobKey);

Code/Tools/AssetProcessor/native/resourcecompiler/rccontroller.h

@@ -54,10 +54,11 @@ namespace AssetProcessor
         void StartJob(AssetProcessor::RCJob* rcJob);
         int NumberOfPendingCriticalJobsPerPlatform(QString platform);
 
-        void SetSystemRoot(const QDir& systemRoot);
         int NumberOfPendingJobsPerPlatform(QString platform);
         bool IsIdle();
-        bool IsPriorityCopyJob(AssetProcessor::RCJob* rcJob);
+
+        void SetQueueSortOnDBSourceName();
+
     Q_SIGNALS:
         void FileCompiled(JobEntry entry, AssetBuilderSDK::ProcessJobResponse response);
         void FileFailed(JobEntry entry);

Code/Tools/AssetProcessor/native/utilities/ApplicationManagerBase.cpp

@@ -49,8 +49,6 @@ static const qint64 s_ReservedDiskSpaceInBytes = 256 * 1024;
 //! Maximum number of temp folders allowed
 static const int s_MaximumTempFolders = 10000;
 
-const char AdditionalScanFolders[] = "additionalScanFolders";
-
 ApplicationManagerBase::ApplicationManagerBase(int* argc, char*** argv, QObject* parent)
     : ApplicationManager(argc, argv, parent)
 {
@@ -155,55 +153,90 @@ void ApplicationManagerBase::InitAssetProcessorManager()
     const AzFramework::CommandLine* commandLine = nullptr;
     AzFramework::ApplicationRequests::Bus::BroadcastResult(commandLine, &AzFramework::ApplicationRequests::GetCommandLine);
 
-    if(commandLine->HasSwitch("zeroAnalysisMode"))
+    struct APCommandLineSwitch
+    {
+        APCommandLineSwitch(const char* switchTitle, const char* helpText)
+            : m_switch(switchTitle)
+            , m_helpText(helpText)
+        {
+
+        }
+        const char* m_switch;
+        const char* m_helpText;
+    };
+
+    const APCommandLineSwitch Command_waitOnLaunch("waitOnLaunch", "Briefly pauses Asset Processor during initializiation. Useful if you want to attach a debugger.");
+    const APCommandLineSwitch Command_zeroAnalysisMode("zeroAnalysisMode", "Enables using file modification time when examining source assets for processing.");
+    const APCommandLineSwitch Command_enableQueryLogging("enableQueryLogging", "Enables logging database queries.");
+    const APCommandLineSwitch Command_dependencyScanPattern("dependencyScanPattern", "Scans assets that match the given pattern for missing product dependencies.");
+    const APCommandLineSwitch Command_dsp("dsp", Command_dependencyScanPattern.m_helpText);
+    const APCommandLineSwitch Command_fileDependencyScanPattern("fileDependencyScanPattern", "Used with dependencyScanPattern to farther filter the scan.");
+    const APCommandLineSwitch Command_fdsp("fdsp", Command_fileDependencyScanPattern.m_helpText);
+    const APCommandLineSwitch Command_additionalScanFolders("additionalScanFolders", "Used with dependencyScanPattern to farther filter the scan.");
+    const APCommandLineSwitch Command_dependencyScanMaxIteration("dependencyScanMaxIteration", "Used to limit the number of recursive searches per line when running dependencyScanPattern.");
+    const APCommandLineSwitch Command_warningLevel("warningLevel", "Configure the error and warning reporting level for AssetProcessor. Pass in 1 for fatal errors, 2 for fatal errors and warnings.");
+    const APCommandLineSwitch Command_acceptInput("acceptInput", "Enable external control messaging via the ControlRequestHandler, used with automated tests.");
+    const APCommandLineSwitch Command_debugOutput("debugOutput", "When enabled, builders that support it will output debug information as product assets. Used primarily with scene files.");
+    const APCommandLineSwitch Command_sortJobsByDBSourceName("sortJobsByDBSourceName", "When enabled, sorts pending jobs with equal priority and dependencies by database source name instead of job ID. Useful for automated tests to process assets in the same order each time.");
+    const APCommandLineSwitch Command_truncatefingerprint("truncatefingerprint", "Truncates the fingerprint used for processed assets. Useful if you plan to compress product assets to share on another machine because some compression formats like zip will truncate file mod timestamps.");
+    const APCommandLineSwitch Command_help("help", "Displays this message.");
+    const APCommandLineSwitch Command_h("h", Command_help.m_helpText);
+
+    if (commandLine->HasSwitch(Command_waitOnLaunch.m_switch))
+    {
+        // Useful for attaching the debugger, this forces a short pause.
+        AZStd::this_thread::sleep_for(AZStd::chrono::seconds(20));
+    }
+
+    if (commandLine->HasSwitch(Command_zeroAnalysisMode.m_switch))
     {
         m_assetProcessorManager->SetEnableModtimeSkippingFeature(true);
     }
     
-    if(commandLine->HasSwitch("enableQueryLogging"))
+    if (commandLine->HasSwitch(Command_enableQueryLogging.m_switch))
     {
         m_assetProcessorManager->SetQueryLogging(true);
     }
 
-    if (commandLine->HasSwitch("dependencyScanPattern"))
+    if (commandLine->HasSwitch(Command_dependencyScanPattern.m_switch))
     {
-        m_dependencyScanPattern = commandLine->GetSwitchValue("dependencyScanPattern", 0).c_str();
+        m_dependencyScanPattern = commandLine->GetSwitchValue(Command_dependencyScanPattern.m_switch, 0).c_str();
     }
-    else if (commandLine->HasSwitch("dsp"))
+    else if (commandLine->HasSwitch(Command_dsp.m_switch))
     {
-        m_dependencyScanPattern = commandLine->GetSwitchValue("dsp", 0).c_str();
+        m_dependencyScanPattern = commandLine->GetSwitchValue(Command_dsp.m_switch, 0).c_str();
     }
     
     m_fileDependencyScanPattern = "*";
 
-    if (commandLine->HasSwitch("fileDependencyScanPattern"))
+    if (commandLine->HasSwitch(Command_fileDependencyScanPattern.m_switch))
     {
-        m_fileDependencyScanPattern = commandLine->GetSwitchValue("fileDependencyScanPattern", 0).c_str();
+        m_fileDependencyScanPattern = commandLine->GetSwitchValue(Command_fileDependencyScanPattern.m_switch, 0).c_str();
     }
-    else if (commandLine->HasSwitch("fdsp"))
+    else if (commandLine->HasSwitch(Command_fdsp.m_switch))
     {
-        m_fileDependencyScanPattern = commandLine->GetSwitchValue("fdsp", 0).c_str();
+        m_fileDependencyScanPattern = commandLine->GetSwitchValue(Command_fdsp.m_switch, 0).c_str();
     }
 
-    if (commandLine->HasSwitch(AdditionalScanFolders))
+    if (commandLine->HasSwitch(Command_additionalScanFolders.m_switch))
     {
-        for (size_t idx = 0; idx < commandLine->GetNumSwitchValues(AdditionalScanFolders); idx++)
+        for (size_t idx = 0; idx < commandLine->GetNumSwitchValues(Command_additionalScanFolders.m_switch); idx++)
         {
-            AZStd::string value = commandLine->GetSwitchValue(AdditionalScanFolders, idx);
+            AZStd::string value = commandLine->GetSwitchValue(Command_additionalScanFolders.m_switch, idx);
             m_dependencyAddtionalScanFolders.emplace_back(AZStd::move(value));
         }
     }
 
-    if (commandLine->HasSwitch("dependencyScanMaxIteration"))
+    if (commandLine->HasSwitch(Command_dependencyScanMaxIteration.m_switch))
     {
-        AZStd::string maxIterationAsString = commandLine->GetSwitchValue("dependencyScanMaxIteration", 0);
+        AZStd::string maxIterationAsString = commandLine->GetSwitchValue(Command_dependencyScanMaxIteration.m_switch, 0);
         m_dependencyScanMaxIteration = AZStd::stoi(maxIterationAsString);
     }
 
-    if (commandLine->HasSwitch("warningLevel"))
+    if (commandLine->HasSwitch(Command_warningLevel.m_switch))
     {
         using namespace AssetProcessor;
-        const AZStd::string& levelString = commandLine->GetSwitchValue("warningLevel", 0);
+        const AZStd::string& levelString = commandLine->GetSwitchValue(Command_warningLevel.m_switch, 0);
         WarningLevel warningLevel = WarningLevel::Default;
 
         switch(AZStd::stoi(levelString))
@@ -217,26 +250,30 @@ void ApplicationManagerBase::InitAssetProcessorManager()
         }
         AssetProcessor::JobDiagnosticRequestBus::Broadcast(&AssetProcessor::JobDiagnosticRequestBus::Events::SetWarningLevel, warningLevel);
     }
-    if (commandLine->HasSwitch("acceptInput"))
+    if (commandLine->HasSwitch(Command_acceptInput.m_switch))
     {
         InitControlRequestHandler();
     }
 
-    if (commandLine->HasSwitch("debugOutput"))
+    if (commandLine->HasSwitch(Command_debugOutput.m_switch))
     {
         m_assetProcessorManager->SetBuilderDebugFlag(true);
     }
 
-    constexpr char truncateFingerprintSwitch[] = "truncatefingerprint";
-    if(commandLine->HasSwitch(truncateFingerprintSwitch))
+    if (commandLine->HasSwitch(Command_sortJobsByDBSourceName.m_switch))
+    {
+        m_sortJobsByDBSourceName = true;
+    }
+
+    if (commandLine->HasSwitch(Command_truncatefingerprint.m_switch))
     {
         // Zip archive format uses 2 second precision truncated
         const int ArchivePrecision = 2000;
         int precision = ArchivePrecision;
 
-        if(commandLine->GetNumSwitchValues(truncateFingerprintSwitch) > 0)
+        if (commandLine->GetNumSwitchValues(Command_truncatefingerprint.m_switch) > 0)
         {
-            precision = AZStd::stoi(commandLine->GetSwitchValue(truncateFingerprintSwitch, 0));
+            precision = AZStd::stoi(commandLine->GetSwitchValue(Command_truncatefingerprint.m_switch, 0));
 
             if(precision < 1)
             {
@@ -246,6 +283,31 @@ void ApplicationManagerBase::InitAssetProcessorManager()
 
         AssetUtilities::SetTruncateFingerprintTimestamp(precision);
     }
+
+    if (commandLine->HasSwitch(Command_help.m_switch) || commandLine->HasSwitch(Command_h.m_switch))
+    {
+        // Other O3DE tools have a more full featured system for registering command flags
+        // that includes help output, but right now the AssetProcessor just checks strings
+        // via HasSwitch. This means this help output has to be updated manually.
+        AZ_TracePrintf("AssetProcessor", "Asset Processor Command Line Flags:\n");
+        AZ_TracePrintf("AssetProcessor", "\t%s : %s\n", Command_waitOnLaunch.m_switch, Command_waitOnLaunch.m_helpText);
+        AZ_TracePrintf("AssetProcessor", "\t%s : %s\n", Command_zeroAnalysisMode.m_switch, Command_zeroAnalysisMode.m_helpText);
+        AZ_TracePrintf("AssetProcessor", "\t%s : %s\n", Command_enableQueryLogging.m_switch, Command_enableQueryLogging.m_helpText);
+        AZ_TracePrintf("AssetProcessor", "\t%s : %s\n", Command_dependencyScanPattern.m_switch, Command_dependencyScanPattern.m_helpText);
+        AZ_TracePrintf("AssetProcessor", "\t%s : %s\n", Command_dsp.m_switch, Command_dsp.m_helpText);
+        AZ_TracePrintf("AssetProcessor", "\t%s : %s\n", Command_fileDependencyScanPattern.m_switch, Command_fileDependencyScanPattern.m_helpText);
+        AZ_TracePrintf("AssetProcessor", "\t%s : %s\n", Command_fdsp.m_switch, Command_fdsp.m_helpText);
+        AZ_TracePrintf("AssetProcessor", "\t%s : %s\n", Command_additionalScanFolders.m_switch, Command_additionalScanFolders.m_helpText);
+        AZ_TracePrintf("AssetProcessor", "\t%s : %s\n", Command_dependencyScanMaxIteration.m_switch, Command_dependencyScanMaxIteration.m_helpText);
+        AZ_TracePrintf("AssetProcessor", "\t%s : %s\n", Command_warningLevel.m_switch, Command_warningLevel.m_helpText);
+        AZ_TracePrintf("AssetProcessor", "\t%s : %s\n", Command_acceptInput.m_switch, Command_acceptInput.m_helpText);
+        AZ_TracePrintf("AssetProcessor", "\t%s : %s\n", Command_debugOutput.m_switch, Command_debugOutput.m_helpText);
+        AZ_TracePrintf("AssetProcessor", "\t%s : %s\n", Command_sortJobsByDBSourceName.m_switch, Command_sortJobsByDBSourceName.m_helpText);
+        AZ_TracePrintf("AssetProcessor", "\t%s : %s\n", Command_truncatefingerprint.m_switch, Command_truncatefingerprint.m_helpText);
+        AZ_TracePrintf("AssetProcessor", "\t%s : %s\n", Command_help.m_switch, Command_help.m_helpText);
+        AZ_TracePrintf("AssetProcessor", "\t%s : %s\n", Command_h.m_switch, Command_h.m_helpText);
+        AZ_TracePrintf("AssetProcessor", "\tregset : set the given registry key to the given value.\n");
+    }
 }
 
 void ApplicationManagerBase::Rescan()
@@ -281,6 +343,11 @@ void ApplicationManagerBase::InitRCController()
 {
     m_rcController = new AssetProcessor::RCController(m_platformConfiguration->GetMinJobs(), m_platformConfiguration->GetMaxJobs());
 
+    if (m_sortJobsByDBSourceName)
+    {
+        m_rcController->SetQueueSortOnDBSourceName();
+    }
+
     QObject::connect(m_assetProcessorManager, &AssetProcessor::AssetProcessorManager::AssetToProcess, m_rcController, &AssetProcessor::RCController::JobSubmitted);
     QObject::connect(m_rcController, &AssetProcessor::RCController::FileCompiled, m_assetProcessorManager, &AssetProcessor::AssetProcessorManager::AssetProcessed, Qt::UniqueConnection);
     QObject::connect(m_rcController, &AssetProcessor::RCController::FileFailed, m_assetProcessorManager, &AssetProcessor::AssetProcessorManager::AssetFailed);
@@ -1807,4 +1874,3 @@ void ApplicationManagerBase::OnActiveJobsCountChanged(unsigned int count)
     AssetProcessor::AssetProcessorStatusEntry entry(AssetProcessor::AssetProcessorStatus::Processing_Jobs, count);
     Q_EMIT AssetProcessorStatusChanged(entry);
 }
-

Code/Tools/AssetProcessor/native/utilities/ApplicationManagerBase.h

@@ -236,6 +236,11 @@ protected:
     int m_remainingAPMJobs = 0;
     bool m_assetProcessorManagerIsReady = false;
 
+    // When job priority and escalation is equal, jobs sort in order by job key.
+    // This switches that behavior to instead sort by the DB source name, which
+    // allows automated tests to get deterministic behavior out of Asset Processor.
+    bool m_sortJobsByDBSourceName = false;
+
     unsigned int m_highestConnId = 0;
     AzToolsFramework::Ticker* m_ticker = nullptr; // for ticking the tickbus.