Initial commit

Code/Tools/AssetProcessor/native/resourcecompiler/JobsModel.cpp

@@ -0,0 +1,522 @@
+/*
+* All or portions of this file Copyright (c) Amazon.com, Inc. or its affiliates or
+* its licensors.
+*
+* For complete copyright and license terms please see the LICENSE at the root of this
+* distribution (the "License"). All use of this software is governed by the License,
+* or, if provided, by the license below or the license accompanying this file. Do not
+* remove or modify any license notices. This file is distributed on an "AS IS" BASIS,
+* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+*
+*/
+
+#include <native/resourcecompiler/JobsModel.h>
+#include <AzToolsFramework/API/AssetDatabaseBus.h>
+#include <native/utilities/assetUtils.h>
+#include <native/AssetDatabase/AssetDatabase.h>
+#include <native/resourcecompiler/RCJobSortFilterProxyModel.h>
+#include <native/utilities/JobDiagnosticTracker.h>
+
+namespace AssetProcessor
+{
+    JobsModel::JobsModel(QObject* parent)
+        : QAbstractItemModel(parent)
+        , m_pendingIcon(QStringLiteral(":/stylesheet/img/logging/pending.svg"))
+        , m_errorIcon(QStringLiteral(":/stylesheet/img/logging/error.svg"))
+        , m_warningIcon(QStringLiteral(":/stylesheet/img/logging/warning-yellow.svg"))
+        , m_okIcon(QStringLiteral(":/stylesheet/img/logging/valid.svg"))
+        , m_processingIcon(QStringLiteral(":/stylesheet/img/logging/processing.svg"))
+    {
+    }
+
+    JobsModel::~JobsModel()
+    {
+        for (int idx = 0; idx < m_cachedJobs.size(); idx++)
+        {
+            delete m_cachedJobs[idx];
+        }
+
+        m_cachedJobs.clear();
+        m_cachedJobsLookup.clear();
+    }
+
+    QModelIndex JobsModel::parent(const QModelIndex& index) const
+    {
+        AZ_UNUSED(index);
+        return QModelIndex();
+    }
+
+    QModelIndex JobsModel::index(int row, int column, const QModelIndex& parent) const
+    {
+        if (row >= rowCount(parent) || column >= columnCount(parent))
+        {
+            return QModelIndex();
+        }
+        return createIndex(row, column);
+    }
+
+    int JobsModel::rowCount(const QModelIndex& parent) const
+    {
+        return parent.isValid() ? 0 : itemCount();
+    }
+
+    int JobsModel::columnCount(const QModelIndex& parent) const
+    {
+        return parent.isValid() ? 0 : Column::Max;
+    }
+
+    QVariant JobsModel::headerData(int section, Qt::Orientation orientation, int role) const
+    {
+        if (orientation != Qt::Horizontal)
+        {
+            return QAbstractItemModel::headerData(section, orientation, role);
+        }
+
+        switch (role)
+        {
+            case Qt::DisplayRole:
+            {
+                switch (section)
+                {
+                case ColumnStatus:
+                    return tr("Status");
+                case ColumnSource:
+                    return tr("Source");
+                case ColumnPlatform:
+                    return tr("Platform");
+                case ColumnJobKey:
+                    return tr("Job Key");
+                case ColumnCompleted:
+                    return tr("Completed");
+                default:
+                    break;
+                }
+            }
+
+            case Qt::TextAlignmentRole:
+            {
+                return Qt::AlignLeft + Qt::AlignVCenter;
+            }
+
+            default:
+                break;
+        }
+
+        return QAbstractItemModel::headerData(section, orientation, role);
+    }
+
+    int JobsModel::itemCount() const
+    {
+        return aznumeric_caster(m_cachedJobs.size());
+    }
+
+
+    QVariant JobsModel::data(const QModelIndex& index, int role) const
+    {
+        if (!index.isValid())
+        {
+            return QVariant();
+        }
+
+        if (index.row() >= itemCount())
+        {
+            return QVariant();
+        }
+
+        switch (role)
+        {
+        case Qt::DecorationRole:
+        {
+            if (index.column() == ColumnStatus) {
+                using namespace AzToolsFramework::AssetSystem;
+
+                switch (getItem(index.row())->m_jobState) {
+                case JobStatus::Queued:
+                    return m_pendingIcon;
+                case JobStatus::Failed_InvalidSourceNameExceedsMaxLimit:  // fall through intentional
+                case JobStatus::Failed:
+                    return m_errorIcon;
+                case JobStatus::Completed:
+                {
+                    CachedJobInfo* jobInfo = getItem(index.row());
+
+                    if(jobInfo->m_warningCount > 0 || jobInfo->m_errorCount > 0)
+                    {
+                        // Warning icon is used for both warnings and errors.
+                        return m_warningIcon;
+                    }
+
+                    return m_okIcon;
+                }
+                case JobStatus::InProgress:
+                    return m_processingIcon;
+                }
+            }
+
+            break;
+        }
+        case Qt::DisplayRole:
+        case SortRole:
+            switch (index.column())
+            {
+            case ColumnStatus:
+            {
+                CachedJobInfo* jobInfo = getItem(index.row());
+                return GetStatusInString(jobInfo->m_jobState, jobInfo->m_warningCount, jobInfo->m_errorCount);
+            }
+            case ColumnSource:
+                return getItem(index.row())->m_elementId.GetInputAssetName();
+            case ColumnPlatform:
+                return getItem(index.row())->m_elementId.GetPlatform();
+            case ColumnJobKey:
+                return getItem(index.row())->m_elementId.GetJobDescriptor();
+            case ColumnCompleted:
+                if (role == SortRole)
+                {
+                    return getItem(index.row())->m_completedTime;
+                }
+                else
+                {
+                    return getItem(index.row())->m_completedTime.toString("hh:mm:ss.zzz MMM dd, yyyy");
+                }
+            default:
+                break;
+            }
+        case logRole:
+        {
+            using namespace AzToolsFramework::AssetSystem;
+            using namespace AssetUtilities;
+
+            JobInfo jobInfo;
+            AssetJobLogResponse jobLogResponse;
+            auto* cachedJobInfo = getItem(index.row());
+            jobInfo.m_sourceFile = cachedJobInfo->m_elementId.GetInputAssetName().toUtf8().data();
+            jobInfo.m_platform = cachedJobInfo->m_elementId.GetPlatform().toUtf8().data();
+            jobInfo.m_jobKey = cachedJobInfo->m_elementId.GetJobDescriptor().toUtf8().data();
+            jobInfo.m_builderGuid = cachedJobInfo->m_builderGuid;
+            jobInfo.m_jobRunKey = cachedJobInfo->m_jobRunKey;
+            jobInfo.m_warningCount = cachedJobInfo->m_warningCount;
+            jobInfo.m_errorCount = cachedJobInfo->m_errorCount;
+
+            ReadJobLogResult readJobLogResult = ReadJobLog(jobInfo, jobLogResponse);
+
+            const char* jobLogData = jobLogResponse.m_jobLog.c_str();
+
+            // ReadJobLog prepends the result with Error: if it can't find the file, even if the job was
+            // completed successfully or is still pending, so we detect that and give a less panicky response
+            // to the end user.
+            if (readJobLogResult == ReadJobLogResult::MissingLogFile)
+            {
+                switch (cachedJobInfo->m_jobState)
+                {
+                    case JobStatus::Completed:
+                        jobLogData = "The log file from the last (successful) run of this job could not be found.\nLogs are not always generated for successful jobs and this does not indicate an error.";
+                        break;
+
+                    case JobStatus::InProgress:
+                    case JobStatus::Queued:
+                        jobLogData = "The job is still processing and the log file has not yet been created";
+                        break;
+
+                    default:
+                        // leave the job log as it is
+                        break;
+                }
+            }
+
+            return QVariant(jobLogData);
+        }
+        case Qt::TextAlignmentRole:
+        {
+            return Qt::AlignLeft + Qt::AlignVCenter;
+        }
+        case statusRole:
+        {
+            CachedJobInfo* jobInfo = getItem(index.row());
+            return QVariant::fromValue(JobStatusInfo{ jobInfo->m_jobState, jobInfo->m_warningCount, jobInfo->m_errorCount });
+        }
+        case logFileRole:
+        {
+            AzToolsFramework::AssetSystem::JobInfo jobInfo;
+            CachedJobInfo* cachedJobInfo = getItem(index.row());
+            jobInfo.m_sourceFile = cachedJobInfo->m_elementId.GetInputAssetName().toUtf8().data();
+            jobInfo.m_platform = cachedJobInfo->m_elementId.GetPlatform().toUtf8().data();
+            jobInfo.m_jobKey = cachedJobInfo->m_elementId.GetJobDescriptor().toUtf8().data();
+            jobInfo.m_builderGuid = cachedJobInfo->m_builderGuid;
+            jobInfo.m_jobRunKey = cachedJobInfo->m_jobRunKey;
+            jobInfo.m_warningCount = cachedJobInfo->m_warningCount;
+            jobInfo.m_errorCount = cachedJobInfo->m_errorCount;
+
+            AZStd::string logFile = AssetUtilities::ComputeJobLogFolder() + "/" + AssetUtilities::ComputeJobLogFileName(jobInfo);
+            return QVariant(logFile.c_str());
+        }
+
+        default:
+            break;
+        }
+
+        return QVariant();
+    }
+
+    CachedJobInfo* JobsModel::getItem(int index) const
+    {
+        if (index >= 0 && index < m_cachedJobs.size())
+        {
+            return m_cachedJobs[index];
+        }
+
+        return nullptr; //invalid index
+    }
+
+    void Append(QString& base, const QString& input, const QString& seperator = ", ")
+    {
+        if(input.isEmpty())
+        {
+            return;
+        }
+
+        if(!base.isEmpty())
+        {
+            base.append(seperator);
+        }
+
+        base.append(input);
+    }
+
+    QString JobsModel::GetStatusInString(const AzToolsFramework::AssetSystem::JobStatus& state, AZ::u32 warningCount, AZ::u32 errorCount)
+    {
+        using namespace AzToolsFramework::AssetSystem;
+
+        switch (state)
+        {
+            case JobStatus::Queued:
+                return tr("Pending");
+            case JobStatus::Failed_InvalidSourceNameExceedsMaxLimit:  // fall through intentional
+            case JobStatus::Failed:
+                return tr("Failed");
+            case JobStatus::Completed:
+            {
+                QString message = tr("Completed");
+                QString extra;
+
+                if (warningCount > 0)
+                {
+                    extra.append(QString("%1 %2").arg(warningCount).arg( warningCount == 1 ? tr("warning") : tr("warnings") ));
+                }
+
+                if (errorCount > 0)
+                {
+                    Append(extra, QString("%1 %2").arg(errorCount).arg( errorCount == 1 ? tr("error") : tr("errors") ));
+                }
+
+                Append(message, extra, ": ");
+
+                return message;
+            }
+            case JobStatus::InProgress:
+                return tr("InProgress");
+        }
+
+        return QString();
+    }
+
+    void JobsModel::PopulateJobsFromDatabase()
+    {
+        beginResetModel();
+        AZStd::string databaseLocation;
+        AzToolsFramework::AssetDatabase::AssetDatabaseRequestsBus::Broadcast(&AzToolsFramework::AssetDatabase::AssetDatabaseRequests::GetAssetDatabaseLocation, databaseLocation);
+
+        if (!databaseLocation.empty())
+        {
+            AssetProcessor::AssetDatabaseConnection assetDatabaseConnection;
+            assetDatabaseConnection.OpenDatabase();
+
+            auto jobsFunction = [this, &assetDatabaseConnection](AzToolsFramework::AssetDatabase::JobDatabaseEntry& entry)
+            {
+                AzToolsFramework::AssetDatabase::SourceDatabaseEntry source;
+                assetDatabaseConnection.GetSourceBySourceID(entry.m_sourcePK, source);
+                CachedJobInfo* jobInfo = new CachedJobInfo();
+                jobInfo->m_elementId.SetInputAssetName(source.m_sourceName.c_str());
+                jobInfo->m_elementId.SetPlatform(entry.m_platform.c_str());
+                jobInfo->m_elementId.SetJobDescriptor(entry.m_jobKey.c_str());
+                jobInfo->m_jobState = entry.m_status;
+                jobInfo->m_jobRunKey = aznumeric_cast<uint32_t>(entry.m_jobRunKey);
+                jobInfo->m_builderGuid = entry.m_builderGuid;
+                jobInfo->m_completedTime = QDateTime::fromMSecsSinceEpoch(entry.m_lastLogTime);
+                jobInfo->m_warningCount = entry.m_warningCount;
+                jobInfo->m_errorCount = entry.m_errorCount;
+                m_cachedJobs.push_back(jobInfo);
+                m_cachedJobsLookup.insert(jobInfo->m_elementId, aznumeric_caster(m_cachedJobs.size() - 1));
+                return true;
+            };
+            assetDatabaseConnection.QueryJobsTable(jobsFunction);
+        }
+
+        endResetModel();
+    }
+
+    QModelIndex JobsModel::GetJobFromProduct(const AzToolsFramework::AssetDatabase::ProductDatabaseEntry& productEntry, AzToolsFramework::AssetDatabase::AssetDatabaseConnection& assetDatabaseConnection)
+    {
+        AZStd::string sourceForProduct;
+        assetDatabaseConnection.QuerySourceByProductID(
+            productEntry.m_productID,
+            [&](AzToolsFramework::AssetDatabase::SourceDatabaseEntry& sourceEntry)
+        {
+            sourceForProduct = sourceEntry.m_sourceName;
+            return false;
+        });
+
+        if (sourceForProduct.empty())
+        {
+            return QModelIndex();
+        }
+
+        AzToolsFramework::AssetDatabase::JobDatabaseEntry foundJobEntry;
+        assetDatabaseConnection.QueryJobByProductID(
+            productEntry.m_productID,
+            [&](AzToolsFramework::AssetDatabase::JobDatabaseEntry& jobEntry)
+        {
+            foundJobEntry = jobEntry;
+            return false;
+        });
+
+        if (foundJobEntry.m_jobID == AzToolsFramework::AssetDatabase::InvalidEntryId)
+        {
+            return QModelIndex();
+        }
+        return GetJobFromSourceAndJobInfo(sourceForProduct, foundJobEntry.m_platform, foundJobEntry.m_jobKey);
+    }
+
+    QModelIndex JobsModel::GetJobFromSourceAndJobInfo(const AZStd::string& source, const AZStd::string& platform, const AZStd::string& jobKey)
+    {
+        QueueElementID elementId(source.c_str(), platform.c_str(), jobKey.c_str());
+        auto iter = m_cachedJobsLookup.find(elementId);
+        if (iter == m_cachedJobsLookup.end())
+        {
+            return QModelIndex();
+        }
+
+        return index(iter.value(), 0, QModelIndex());
+    }
+
+    void JobsModel::OnJobStatusChanged(JobEntry entry, AzToolsFramework::AssetSystem::JobStatus status)
+    {
+        QueueElementID elementId(entry.m_databaseSourceName, entry.m_platformInfo.m_identifier.c_str(), entry.m_jobKey);
+        CachedJobInfo* jobInfo = nullptr;
+        unsigned int jobIndex = 0;
+
+        JobDiagnosticInfo jobDiagnosticInfo{};
+        JobDiagnosticRequestBus::BroadcastResult(jobDiagnosticInfo, &JobDiagnosticRequestBus::Events::GetDiagnosticInfo, entry.m_jobRunKey);
+
+        auto iter = m_cachedJobsLookup.find(elementId);
+        if (iter == m_cachedJobsLookup.end())
+        {
+            jobInfo = new CachedJobInfo();
+            jobInfo->m_elementId.SetInputAssetName(entry.m_databaseSourceName.toUtf8().data());
+            jobInfo->m_elementId.SetPlatform(entry.m_platformInfo.m_identifier.c_str());
+            jobInfo->m_elementId.SetJobDescriptor(entry.m_jobKey.toUtf8().data());
+            jobInfo->m_jobRunKey = aznumeric_cast<uint32_t>(entry.m_jobRunKey);
+            jobInfo->m_builderGuid = entry.m_builderGuid;
+            jobInfo->m_jobState = status;
+            jobInfo->m_warningCount = jobDiagnosticInfo.m_warningCount;
+            jobInfo->m_errorCount = jobDiagnosticInfo.m_errorCount;
+            jobIndex = aznumeric_caster(m_cachedJobs.size());
+            beginInsertRows(QModelIndex(), jobIndex, jobIndex);
+            m_cachedJobs.push_back(jobInfo);
+            m_cachedJobsLookup.insert(jobInfo->m_elementId, jobIndex);
+            endInsertRows();
+        }
+        else
+        {
+            jobIndex = iter.value();
+            jobInfo = m_cachedJobs[jobIndex];
+            jobInfo->m_jobState = status;
+            jobInfo->m_jobRunKey = aznumeric_cast<uint32_t>(entry.m_jobRunKey);
+            jobInfo->m_builderGuid = entry.m_builderGuid;
+            jobInfo->m_warningCount = jobDiagnosticInfo.m_warningCount;
+            jobInfo->m_errorCount = jobDiagnosticInfo.m_errorCount;
+            if (jobInfo->m_jobState == AzToolsFramework::AssetSystem::JobStatus::Completed || jobInfo->m_jobState == AzToolsFramework::AssetSystem::JobStatus::Failed)
+            {
+                jobInfo->m_completedTime = QDateTime::currentDateTime();
+            }
+            else
+            {
+                jobInfo->m_completedTime = QDateTime();
+            }
+            Q_EMIT dataChanged(index(jobIndex, 0, QModelIndex()), index(jobIndex, columnCount() - 1, QModelIndex()));
+        }
+    }
+
+    void JobsModel::OnSourceRemoved(QString sourceDatabasePath)
+    {
+        // when a source is removed, we need to eliminate all job entries for that source regardless of all other details of it.
+        QList<AssetProcessor::QueueElementID> elementsToRemove;
+        for (int index = 0; index < m_cachedJobs.size(); ++index)
+        {
+            if (QString::compare(m_cachedJobs[index]->m_elementId.GetInputAssetName(), sourceDatabasePath, Qt::CaseSensitive) == 0)
+            {
+                elementsToRemove.push_back(m_cachedJobs[index]->m_elementId);
+            }
+        }
+
+        // now that we've collected all the elements to remove, we can remove them.  
+        // Doing it this way avoids problems with mutating these cache structures while iterating them.
+        for (const AssetProcessor::QueueElementID& removal : elementsToRemove)
+        {
+            RemoveJob(removal);
+        }
+    }
+
+    void JobsModel::OnFolderRemoved(QString folderPath)
+    {
+        QList<AssetProcessor::QueueElementID> elementsToRemove;
+        for (int index = 0; index < m_cachedJobs.size(); ++index)
+        {
+            if (m_cachedJobs[index]->m_elementId.GetInputAssetName().startsWith(folderPath, Qt::CaseSensitive))
+            {
+                elementsToRemove.push_back(m_cachedJobs[index]->m_elementId);
+            }
+        }
+
+        // now that we've collected all the elements to remove, we can remove them.  
+        // Doing it this way avoids problems with mutating these cache structures while iterating them.
+        for (const AssetProcessor::QueueElementID& removal : elementsToRemove)
+        {
+            RemoveJob(removal);
+        }
+
+    }
+
+    void JobsModel::OnJobRemoved(AzToolsFramework::AssetSystem::JobInfo jobInfo)
+    {
+        RemoveJob(QueueElementID(jobInfo.m_sourceFile.c_str(), jobInfo.m_platform.c_str(), jobInfo.m_jobKey.c_str()));
+    }
+
+    void JobsModel::RemoveJob(const AssetProcessor::QueueElementID& elementId)
+    {
+        auto iter = m_cachedJobsLookup.find(elementId);
+        if (iter != m_cachedJobsLookup.end())
+        {
+            unsigned int jobIndex = iter.value();
+            CachedJobInfo* jobInfo = m_cachedJobs[jobIndex];
+            beginRemoveRows(QModelIndex(), jobIndex, jobIndex);
+            m_cachedJobs.erase(m_cachedJobs.begin() + jobIndex);
+            delete jobInfo;
+            m_cachedJobsLookup.erase(iter);
+            // Since we are storing the jobIndex for each job for faster lookup therefore 
+            // we need to update the jobIndex for jobs that were after the removed job. 
+            for (int idx = jobIndex; idx < m_cachedJobs.size(); idx++)
+            {
+                jobInfo = m_cachedJobs[idx];
+                auto iterator = m_cachedJobsLookup.find(jobInfo->m_elementId);
+                if (iterator != m_cachedJobsLookup.end())
+                {
+                    unsigned int index = iterator.value();
+                    m_cachedJobsLookup[jobInfo->m_elementId] = --index;
+                }
+            }
+            endRemoveRows();
+        }
+    }
+} //namespace AssetProcessor
+

Code/Tools/AssetProcessor/native/resourcecompiler/JobsModel.h

@@ -0,0 +1,113 @@
+/*
+* All or portions of this file Copyright (c) Amazon.com, Inc. or its affiliates or
+* its licensors.
+*
+* For complete copyright and license terms please see the LICENSE at the root of this
+* distribution (the "License"). All use of this software is governed by the License,
+* or, if provided, by the license below or the license accompanying this file. Do not
+* remove or modify any license notices. This file is distributed on an "AS IS" BASIS,
+* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+*
+*/
+#pragma once
+
+#if !defined(Q_MOC_RUN)
+#include <AzToolsFramework/API/EditorAssetSystemAPI.h>
+#include <native/resourcecompiler/RCCommon.h>
+#include <QAbstractItemModel>
+#include <QDateTime>
+#include <QHash>
+#include <QIcon>
+#include <QVector>
+
+// Don't reorder above RCCommon
+#include <native/assetprocessor.h>
+#endif
+
+// Do this here, rather than EditorAssetSystemAPI.h so that we don't have to link against Qt5Core to
+// use EditorAssetSystemAPI.h
+Q_DECLARE_METATYPE(AzToolsFramework::AssetSystem::JobStatus);
+
+namespace AzToolsFramework
+{
+    namespace AssetDatabase
+    {
+        class ProductDatabaseEntry;
+        class AssetDatabaseConnection;
+    }
+}
+namespace AssetProcessor
+{
+    //! CachedJobInfo stores all the necessary information needed for showing a particular job including its log
+    struct CachedJobInfo
+    {
+        QueueElementID m_elementId;
+        QDateTime m_completedTime;
+        AzToolsFramework::AssetSystem::JobStatus m_jobState;
+        AZ::u32 m_warningCount;
+        AZ::u32 m_errorCount;
+        unsigned int m_jobRunKey;
+        AZ::Uuid m_builderGuid;
+    };
+    /**
+    * The JobsModel class contains list of jobs from both the Database and the RCController
+    */
+    class JobsModel
+        : public QAbstractItemModel
+    {
+        Q_OBJECT
+    public:
+
+        enum DataRoles
+        {
+            logRole = Qt::UserRole + 1,
+            statusRole,
+            logFileRole,
+            SortRole,
+        };
+
+        enum Column
+        {
+            ColumnStatus,
+            ColumnSource,
+            ColumnCompleted,
+            ColumnPlatform,
+            ColumnJobKey,
+            Max
+        };
+
+        explicit JobsModel(QObject* parent = nullptr);
+        virtual ~JobsModel();
+        QModelIndex parent(const QModelIndex& index) const override;
+        QModelIndex index(int row, int column, const QModelIndex& parent = QModelIndex()) const override;
+        int columnCount(const QModelIndex& parent = QModelIndex()) const override;
+        int rowCount(const QModelIndex& parent = QModelIndex()) const override;
+        QVariant headerData(int section, Qt::Orientation orientation, int role) const override;
+        QVariant data(const QModelIndex& index, int role) const override;
+        int itemCount() const;
+        CachedJobInfo* getItem(int index) const;
+        static QString GetStatusInString(const AzToolsFramework::AssetSystem::JobStatus& state, AZ::u32 warningCount, AZ::u32 errorCount);
+        void PopulateJobsFromDatabase();
+
+        QModelIndex GetJobFromProduct(const AzToolsFramework::AssetDatabase::ProductDatabaseEntry& productEntry, AzToolsFramework::AssetDatabase::AssetDatabaseConnection& assetDatabaseConnection);
+        QModelIndex GetJobFromSourceAndJobInfo(const AZStd::string& source, const AZStd::string& platform, const AZStd::string& jobKey);
+
+public Q_SLOTS:
+        void OnJobStatusChanged(JobEntry entry, AzToolsFramework::AssetSystem::JobStatus status);
+        void OnJobRemoved(AzToolsFramework::AssetSystem::JobInfo jobInfo);
+        void OnSourceRemoved(QString sourceDatabasePath);
+        void OnFolderRemoved(QString folderPath);
+
+    protected:
+        QIcon m_pendingIcon;
+        QIcon m_errorIcon;
+        QIcon m_warningIcon;
+        QIcon m_okIcon;
+        QIcon m_processingIcon;
+        AZStd::vector<CachedJobInfo*> m_cachedJobs;
+        QHash<AssetProcessor::QueueElementID, int> m_cachedJobsLookup; // QVector uses int as type of index.  
+
+        void RemoveJob(const AssetProcessor::QueueElementID& elementId);
+    };
+
+} //namespace AssetProcessor

Code/Tools/AssetProcessor/native/resourcecompiler/RCBuilder.h

@@ -0,0 +1,211 @@
+/*
+* All or portions of this file Copyright(c) Amazon.com, Inc.or its affiliates or
+* its licensors.
+*
+* For complete copyright and license terms please see the LICENSE at the root of this
+* distribution (the "License"). All use of this software is governed by the License,
+*or, if provided, by the license below or the license accompanying this file. Do not
+* remove or modify any license notices. This file is distributed on an "AS IS" BASIS,
+*WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+*
+*/
+
+#pragma once
+
+#include "../utilities/ApplicationManagerAPI.h"
+#include "native/AssetManager/assetProcessorManager.h"
+#include "native/utilities/PlatformConfiguration.h"
+
+namespace AssetProcessor
+{
+    struct RCCompiler
+    {
+        //! RC.exe execution result
+        struct Result
+        {
+            Result(int exitCode, bool crashed, const QString& outputDir);
+            Result() = default;
+            int     m_exitCode = 1;
+            bool    m_crashed = false;
+            QString m_outputDir;
+        };
+
+        virtual ~RCCompiler() = default;
+        virtual bool Initialize(const QString& systemRoot, const QString& rcExecutableFullPath) = 0;
+        virtual bool Execute(const QString& inputFile, const QString& watchFolder, const QString& platformIdentifier, const QString& params,
+            const QString& dest, const AssetBuilderSDK::JobCancelListener* jobCancelListener, Result& result) const = 0;
+        virtual void RequestQuit() = 0;
+    };
+
+    //! Worker class to handle shell execution of the legacy rc.exe compiler
+    class NativeLegacyRCCompiler
+        : public RCCompiler
+    {
+    public:
+        NativeLegacyRCCompiler();
+
+        bool Initialize(const QString& systemRoot, const QString& rcExecutableFullPath) override;
+        bool Execute(const QString& inputFile, const QString& watchFolder, const QString& platformIdentifier, const QString& params, const QString& dest,
+            const AssetBuilderSDK::JobCancelListener* jobCancelListener, Result& result) const override;
+        static QString BuildCommand(const QString& inputFile, const QString& watchFolder, const QString& platformIdentifier, const QString& params, const QString& dest);
+        void RequestQuit()  override;
+    private:
+        static const int            s_maxSleepTime;
+        static const unsigned int   s_jobMaximumWaitTime;
+        bool                        m_resourceCompilerInitialized;
+        QDir                        m_systemRoot;
+        QString                     m_rcExecutableFullPath;
+        volatile bool               m_requestedQuit;
+    };
+
+    //! Internal structure that consolidates a internal builder id, its name, and its custom fixed rc param match
+    class BuilderIdAndName
+    {
+    public:
+        enum class Type
+        {
+            REGISTERED_BUILDER,
+            UNREGISTERED_BUILDER
+        };
+
+        BuilderIdAndName() = default;
+        BuilderIdAndName(QString builderName, QString builderId, Type type, QString rcParam = QString(""));
+        BuilderIdAndName(const BuilderIdAndName& src) = default;
+
+        BuilderIdAndName& operator=(const AssetProcessor::BuilderIdAndName& src);
+
+
+        const QString& GetName() const;
+        bool GetUuid(AZ::Uuid& builderUuid) const;
+        const QString& GetRcParam() const;
+        const QString& GetId() const;
+        const Type GetType() const;
+    private:
+        Type    m_type = Type::UNREGISTERED_BUILDER;
+        QString m_builderName;
+        QString m_builderId;
+        QString m_rcParam;
+    };
+
+    extern const BuilderIdAndName  BUILDER_ID_COPY;
+    extern const BuilderIdAndName  BUILDER_ID_RC;
+    extern const BuilderIdAndName  BUILDER_ID_SKIP;
+    extern const QHash<QString, BuilderIdAndName>  INTERNAL_BUILDER_BY_ID;
+
+    //! Internal Builder version of the asset recognizer structure that is read in from the platform configuration class
+    struct InternalAssetRecognizer
+        : public AssetRecognizer
+    {
+        InternalAssetRecognizer(const AssetRecognizer& src, const QString& builderId, const QHash<QString, AssetPlatformSpec>& assetPlatformSpecByPlatform);
+        InternalAssetRecognizer(const InternalAssetRecognizer& src) = default;
+
+        AZ::u32 CalculateCRC() const;
+
+        //! Map of platform specs based on the identifier of the platform
+        QHash<QString, AssetPlatformSpec>           m_platformSpecsByPlatform;
+
+        //! unique id that is generated for each unique internal asset recognizer
+        //! which can be used as the key for the job parameter map (see AssetBuilderSDK::JobParameterMap)
+        AZ::u32                                 m_paramID;
+
+        //! Keep track which internal builder type this recognizer is for
+        const QString                          m_builderId;
+    };
+    typedef QHash<AZ::u32, InternalAssetRecognizer*> InternalRecognizerContainer;
+    typedef QList<const InternalAssetRecognizer*> InternalRecognizerPointerContainer;
+    typedef AZStd::function<void(const AssetBuilderSDK::AssetBuilderDesc& builderDesc)> RegisterBuilderDescCallback;
+    typedef AZStd::list<InternalAssetRecognizer*> InternalAssetRecognizerList;
+
+    class InternalRecognizerBasedBuilder
+        : public AssetBuilderSDK::AssetBuilderCommandBus::Handler
+    {
+    public:
+        //! Constructor to initialize the internal based builder to a present set of internal builders and fixed bus id
+        InternalRecognizerBasedBuilder();
+
+        virtual ~InternalRecognizerBasedBuilder();
+
+        AssetBuilderSDK::AssetBuilderDesc CreateBuilderDesc(const QString& builderId, const AZStd::vector<AssetBuilderSDK::AssetBuilderPattern>& builderPatterns);
+
+        void ShutDown() override;
+
+        virtual bool Initialize(const RecognizerConfiguration& recognizerConfig);
+        virtual void InitializeAssetRecognizers(const RecognizerContainer& assetRecognizers);
+        virtual void UnInitialize();
+
+        //! Returns false if there were no matches, otherwise returns true
+        virtual bool GetMatchingRecognizers(const AZStd::vector<AssetBuilderSDK::PlatformInfo>& platformInfos, const QString& fileName, InternalRecognizerPointerContainer& output) const;
+
+        void CreateJobs(const AssetBuilderSDK::CreateJobsRequest& request, AssetBuilderSDK::CreateJobsResponse& response);
+        void ProcessJob(const AssetBuilderSDK::ProcessJobRequest& request, AssetBuilderSDK::ProcessJobResponse& response);
+
+        static bool MatchTempFileToSkip(const QString& outputFilename);
+
+        static void RegisterInternalAssetRecognizerToMap(
+            const AssetRecognizer& assetRecognizer,
+            const QString& builderId,
+            QHash<QString, AssetPlatformSpec>& sourceAssetPlatformSpecs,
+            QHash<QString, InternalAssetRecognizerList>& internalRecognizerListByType);
+
+        //! Split all of the asset recognizers from a container into buckets based on their specific builder action type
+        static void BuildInternalAssetRecognizersByType(
+            const RecognizerContainer& assetRecognizers,
+            QHash<QString, InternalAssetRecognizerList>& internalRecognizerListByType);
+
+    protected:
+        //! Constructor to initialize the internal builders and a general internal builder uuid that is used for bus
+        //! registration.  This constructor is helpful for deriving other classes from this builder for purposes like
+        //! unit testing.
+        InternalRecognizerBasedBuilder(QHash<QString, BuilderIdAndName> inputBuilderByIdMap, AZ::Uuid internalBuilderUuid);
+
+        // overridden in unit tests.  Searches for RC.EXE
+        virtual bool FindRC(QString& rcAbsolutePathOut);
+
+        void CreateLegacyRCJob(
+            const AssetBuilderSDK::CreateJobsRequest& request,
+            QString rcParam,
+            AssetBuilderSDK::CreateJobsResponse& response);
+
+        void ProcessLegacyRCJob(
+            const AssetBuilderSDK::ProcessJobRequest& request,
+            QString rcParam,
+            AZ::Uuid productAssetType,
+            const AssetBuilderSDK::JobCancelListener& jobCancelListener,
+            AssetBuilderSDK::ProcessJobResponse& response);
+
+        //! Given a folder (dest) containing the aftermath of a RC process, generate a response structure.
+        //! if responseFromRCCompiler is true it means that the response is an already-populated response struct
+        //! that was loaded from a response file and we should just append legacy SubIDs to it (responses are used INSTEAD of
+        //! heuristics).
+        //! otherwise it means that we know nothing about the files that were produced and should perform heuristics to determine.
+        //! productAssetType is what uuid type (or nulls) to apply to the generated products for when responseFromRCCompiler is false.
+        void ProcessRCResultFolder(const QString &dest, const AZ::Uuid& productAssetType, bool responseFromRCCompiler, AssetBuilderSDK::ProcessJobResponse &response);
+
+        void ProcessCopyJob(
+            const AssetBuilderSDK::ProcessJobRequest& request,
+            AZ::Uuid productAssetType,
+            bool outputProductDependencies,
+            const AssetBuilderSDK::JobCancelListener& jobCancelListener,
+            AssetBuilderSDK::ProcessJobResponse& response);
+
+        // overridable so we can unit-test override it.
+        virtual QFileInfoList GetFilesInDirectory(const QString& directoryPath);
+
+        // overridable so we can unit-test override it.
+        virtual bool SaveProcessJobRequestFile(const char* requestFileDir, const char* requestFileName, const AssetBuilderSDK::ProcessJobRequest& request);
+
+        // returns false only if there is a critical failure.
+        virtual bool LoadProcessJobResponseFile(const char* responseFileDir, const char* responseFileName, AssetBuilderSDK::ProcessJobResponse& response, bool& responseLoaded);
+
+        AZStd::unique_ptr<RCCompiler>           m_rcCompiler;
+        volatile bool                           m_isShuttingDown;
+        InternalRecognizerContainer             m_assetRecognizerDictionary;
+
+        QHash<QString, BuilderIdAndName>        m_builderById;
+
+        //! UUid for the internal recognizer for logging purposes since the this
+        //! class manages multiple internal build uuids
+        AZ::Uuid                                m_internalRecognizerBuilderUuid;
+
+    };
+} // namespace AssetProcessor

Code/Tools/AssetProcessor/native/resourcecompiler/RCCommon.cpp

@@ -0,0 +1,94 @@
+/*
+* All or portions of this file Copyright (c) Amazon.com, Inc. or its affiliates or
+* its licensors.
+*
+* For complete copyright and license terms please see the LICENSE at the root of this
+* distribution (the "License"). All use of this software is governed by the License,
+* or, if provided, by the license below or the license accompanying this file. Do not
+* remove or modify any license notices. This file is distributed on an "AS IS" BASIS,
+* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+*
+*/
+#include "RCCommon.h"
+#include <QHash>
+
+namespace AssetProcessor
+{
+    QueueElementID::QueueElementID(QString inputAssetName, QString platform, QString jobDescriptor)
+        : m_inputAssetName(inputAssetName)
+        , m_platform(platform)
+        , m_jobDescriptor(jobDescriptor)
+    {
+    }
+
+    QString QueueElementID::GetInputAssetName() const
+    {
+        return m_inputAssetName;
+    }
+    QString QueueElementID::GetPlatform() const
+    {
+        return m_platform;
+    }
+
+    QString QueueElementID::GetJobDescriptor() const
+    {
+        return m_jobDescriptor;
+    }
+
+    void QueueElementID::SetInputAssetName(QString inputAssetName)
+    {
+        m_inputAssetName = inputAssetName;
+    }
+
+    void QueueElementID::SetPlatform(QString platform)
+    {
+        m_platform = platform;
+    }
+
+    void QueueElementID::SetJobDescriptor(QString jobDescriptor)
+    {
+        m_jobDescriptor = jobDescriptor;
+    }
+
+    bool QueueElementID::operator==(const QueueElementID& other) const
+    {
+        // if this becomes a hotspot in profile, we could use CRCs or other boost to comparison here.  These classes are constructed rarely
+        // compared to how commonly they are compared with each other.
+        return (
+            (QString::compare(m_inputAssetName, other.m_inputAssetName, Qt::CaseSensitive) == 0) &&
+            (QString::compare(m_platform, other.m_platform, Qt::CaseInsensitive) == 0) &&
+            (QString::compare(m_jobDescriptor, other.m_jobDescriptor, Qt::CaseInsensitive) == 0)
+            );
+    }
+
+    bool QueueElementID::operator<(const QueueElementID& other) const
+    {
+        int compare = QString::compare(m_inputAssetName, other.m_inputAssetName, Qt::CaseSensitive);
+        if (compare != 0)
+        {
+            return (compare < 0);
+        }
+
+        compare = QString::compare(m_platform, other.m_platform, Qt::CaseInsensitive);
+        if (compare != 0)
+        {
+            return (compare < 0);
+        }
+
+        compare = QString::compare(m_jobDescriptor, other.m_jobDescriptor, Qt::CaseInsensitive);
+
+        if (compare != 0)
+        {
+            return (compare < 0);
+        }
+
+        // all three are equal, other is not less than this.
+        return false;
+    }
+
+    uint qHash(const AssetProcessor::QueueElementID& key, uint seed)
+    {
+        return qHash(key.GetInputAssetName().toLower() + key.GetPlatform().toLower() + key.GetJobDescriptor().toLower(), seed);
+    }
+} // end namespace AssetProcessor
+

Code/Tools/AssetProcessor/native/resourcecompiler/RCCommon.h

@@ -0,0 +1,65 @@
+/*
+* All or portions of this file Copyright (c) Amazon.com, Inc. or its affiliates or
+* its licensors.
+*
+* For complete copyright and license terms please see the LICENSE at the root of this
+* distribution (the "License"). All use of this software is governed by the License,
+* or, if provided, by the license below or the license accompanying this file. Do not
+* remove or modify any license notices. This file is distributed on an "AS IS" BASIS,
+* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+*
+*/
+#ifndef ASSETPROCESSOR_RCCOMMON_H
+#define ASSETPROCESSOR_RCCOMMON_H
+
+#include <QString>
+
+namespace AssetProcessor
+{
+    // RCCommon contains common structs used by the RC Job System
+
+    //! Job Exit codes.
+    // note that this is NOT a complete list of return codes, and RC.EXE itself may return unknown return codes here.
+
+    enum JobExitCodes : int
+    {
+        JobExitCode_Success = 0,
+        JobExitCode_Failed = -10,
+        JobExitCode_InvalidParams = -9,
+        JobExitCode_UnableToCreateTempDir = -8,
+        JobExitCode_CopyFailed = -6,
+        JobExitCode_Unknown = -1,
+        JobExitCode_RCNotFound = -4,
+        JobExitCode_RCCouldNotBeLaunched = -5,
+        JobExitCode_JobCancelled = -7,
+    };
+
+
+    //! Identifies a queued job uniquely.  Not a case sensitive compare
+    class QueueElementID
+    {
+    public:
+        QueueElementID() = default;
+        
+        ///! note that inputAssetName is a database name (so includes outputprefix), not a relative path.
+        QueueElementID(QString inputAssetName, QString platform, QString jobDescriptor);
+
+        
+        QString GetInputAssetName() const; ///< This is the database name, with output prefix.
+        QString GetPlatform() const;
+        QString GetJobDescriptor() const;
+        void SetInputAssetName(QString inputAssetName);
+        void SetPlatform(QString platform);
+        void SetJobDescriptor(QString jobDescriptor);
+        bool operator==(const QueueElementID& other) const;
+        bool operator<(const QueueElementID& other) const;
+
+    protected:
+        QString m_inputAssetName;
+        QString m_platform;
+        QString m_jobDescriptor;
+    };
+    uint qHash(const AssetProcessor::QueueElementID& key, uint seed = 0);
+} // namespace AssetProcessor
+
+#endif //ASSETPROCESSOR_RCQUEUESORTMODEL_H

Code/Tools/AssetProcessor/native/resourcecompiler/RCJobSortFilterProxyModel.cpp

@@ -0,0 +1,106 @@
+/*
+* All or portions of this file Copyright (c) Amazon.com, Inc. or its affiliates or
+* its licensors.
+*
+* For complete copyright and license terms please see the LICENSE at the root of this
+* distribution (the "License"). All use of this software is governed by the License,
+* or, if provided, by the license below or the license accompanying this file. Do not
+* remove or modify any license notices. This file is distributed on an "AS IS" BASIS,
+* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+*
+*/
+
+#include <native/resourcecompiler/RCJobSortFilterProxyModel.h>
+#include <native/resourcecompiler/JobsModel.h> //for jobsModel column enum
+
+#include <AzQtComponents/Components/FilteredSearchWidget.h>
+
+namespace AssetProcessor
+{
+    JobSortFilterProxyModel::JobSortFilterProxyModel(QObject* parent)
+        : QSortFilterProxyModel(parent)
+    {
+        setSortCaseSensitivity(Qt::CaseInsensitive);
+        setFilterCaseSensitivity(Qt::CaseInsensitive);
+    }
+
+    bool JobSortFilterProxyModel::filterAcceptsRow(int sourceRow, const QModelIndex& sourceParent) const
+    {
+        using namespace AzToolsFramework::AssetSystem;
+
+        const QModelIndex jobStateIndex = sourceModel()->index(sourceRow,
+            AssetProcessor::JobsModel::Column::ColumnStatus, sourceParent);
+
+        JobStatusInfo jobStatus = sourceModel()->data(jobStateIndex, AssetProcessor::JobsModel::statusRole)
+            .value<JobStatusInfo>();
+
+        if (jobStatus.m_status == JobStatus::Failed_InvalidSourceNameExceedsMaxLimit)
+        {
+            jobStatus.m_status = JobStatus::Failed;
+        }
+        
+        // Checks our custom filters. 
+        // true = passed our filter, send on to the default filter
+        // false = reject the row
+        auto filterFunc = [&]()
+        {
+            if (m_completedWithWarningsFilter && jobStatus.m_status == JobStatus::Completed && (jobStatus.m_errorCount > 0 || jobStatus.m_warningCount > 0))
+            {
+                return true;
+            }
+            
+            if (!m_activeTypeFilters.isEmpty() && m_activeTypeFilters.contains(jobStatus.m_status))
+            {
+                return true;
+            }
+
+            return false;
+        };
+
+        bool hasFilters = !m_activeTypeFilters.isEmpty() || m_completedWithWarningsFilter == true;
+        bool useDefaultFilter = hasFilters ? filterFunc() : true;
+
+        return useDefaultFilter ? QSortFilterProxyModel::filterAcceptsRow(sourceRow, sourceParent) : false;
+    }
+
+    void JobSortFilterProxyModel::OnJobStatusFilterChanged(const AzQtComponents::SearchTypeFilterList& activeTypeFilters)
+    {
+        m_completedWithWarningsFilter = false;
+        m_activeTypeFilters.clear();
+
+        for (const auto& filter : activeTypeFilters)
+        {
+            if (filter.metadata.canConvert<AzToolsFramework::AssetSystem::JobStatus>())
+            {
+                m_activeTypeFilters << filter.metadata.value<AzToolsFramework::AssetSystem::JobStatus>();
+            }
+            else if (filter.metadata.canConvert<AssetProcessor::CustomJobStatusFilter>())
+            {
+                m_completedWithWarningsFilter = filter.metadata.value<AssetProcessor::CustomJobStatusFilter>().m_completedWithWarnings;
+            }
+        }
+
+        invalidateFilter();
+    }
+
+    bool JobSortFilterProxyModel::lessThan(const QModelIndex& left, const QModelIndex& right) const
+    {
+        // Only the completed column has an override, because it displays time in a different format
+        // than what works best to sort.
+        if (left.column() != JobsModel::ColumnCompleted || right.column() != JobsModel::ColumnCompleted)
+        {
+            return QSortFilterProxyModel::lessThan(left, right);
+        }
+        QVariant leftTime = sourceModel()->data(left, JobsModel::SortRole);
+        QVariant rightTime = sourceModel()->data(right, JobsModel::SortRole);
+
+        if (leftTime.type() != QVariant::DateTime || rightTime.type() != QVariant::DateTime)
+        {
+            return QSortFilterProxyModel::lessThan(left, right);
+        }
+
+        return leftTime.toDateTime() < rightTime.toDateTime();
+    }
+} //namespace AssetProcessor
+
+

Code/Tools/AssetProcessor/native/resourcecompiler/RCJobSortFilterProxyModel.h

@@ -0,0 +1,74 @@
+/*
+* All or portions of this file Copyright (c) Amazon.com, Inc. or its affiliates or
+* its licensors.
+*
+* For complete copyright and license terms please see the LICENSE at the root of this
+* distribution (the "License"). All use of this software is governed by the License,
+* or, if provided, by the license below or the license accompanying this file. Do not
+* remove or modify any license notices. This file is distributed on an "AS IS" BASIS,
+* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+*
+*/
+#pragma once
+
+#if !defined(Q_MOC_RUN)
+#include <QSortFilterProxyModel>
+#include <AzCore/base.h>
+#endif
+
+namespace AzQtComponents
+{
+    struct SearchTypeFilter;
+    using SearchTypeFilterList = QVector<SearchTypeFilter>;
+}
+
+namespace AzToolsFramework
+{
+    namespace AssetSystem
+    {
+        enum class JobStatus;
+    }
+}
+
+namespace AssetProcessor
+{
+    struct CustomJobStatusFilter
+    {
+        CustomJobStatusFilter() = default;
+        CustomJobStatusFilter(bool completeWithWarnings)
+            : m_completedWithWarnings(completeWithWarnings)
+        {
+
+        }
+
+        bool m_completedWithWarnings = false;
+    };
+
+    struct JobStatusInfo
+    {
+        AzToolsFramework::AssetSystem::JobStatus m_status;
+        AZ::u32 m_warningCount;
+        AZ::u32 m_errorCount;
+    };
+
+    class JobSortFilterProxyModel
+        : public QSortFilterProxyModel
+    {
+        Q_OBJECT
+
+    public:
+        explicit JobSortFilterProxyModel(QObject* parent = nullptr);
+        void OnJobStatusFilterChanged(const AzQtComponents::SearchTypeFilterList& activeTypeFilters);
+
+    protected:
+        bool filterAcceptsRow(int sourceRow, const QModelIndex& sourceParent) const override;
+        bool lessThan(const QModelIndex& left, const QModelIndex& right) const override;
+
+    private:
+        QList<AzToolsFramework::AssetSystem::JobStatus> m_activeTypeFilters = {};
+        bool m_completedWithWarningsFilter = false;
+    };
+} // namespace AssetProcessor
+
+Q_DECLARE_METATYPE(AssetProcessor::CustomJobStatusFilter);
+Q_DECLARE_METATYPE(AssetProcessor::JobStatusInfo);

Code/Tools/AssetProcessor/native/resourcecompiler/RCQueueSortModel.cpp

@@ -0,0 +1,258 @@
+/*
+* All or portions of this file Copyright (c) Amazon.com, Inc. or its affiliates or
+* its licensors.
+*
+* For complete copyright and license terms please see the LICENSE at the root of this
+* distribution (the "License"). All use of this software is governed by the License,
+* or, if provided, by the license below or the license accompanying this file. Do not
+* remove or modify any license notices. This file is distributed on an "AS IS" BASIS,
+* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+*
+*/
+#include <native/resourcecompiler/RCQueueSortModel.h>
+#include "rcjoblistmodel.h"
+
+namespace AssetProcessor
+{
+    RCQueueSortModel::RCQueueSortModel(QObject* parent)
+        : QSortFilterProxyModel(parent)
+    {
+        // jobs assigned to "all" platforms are always active.
+        m_currentlyConnectedPlatforms.insert(QString("all"));
+    }
+
+    void RCQueueSortModel::AttachToModel(RCJobListModel* target)
+    {
+        if (target)
+        {
+            setDynamicSortFilter(true);
+            BusConnect();
+            m_sourceModel = target;
+
+            setSourceModel(target);
+            setSortRole(RCJobListModel::jobIndexRole);
+            sort(0);
+        }
+        else
+        {
+            BusDisconnect();
+            setSourceModel(nullptr);
+            m_sourceModel = nullptr;
+        }
+    }
+
+    RCJob* RCQueueSortModel::GetNextPendingJob()
+    {
+        if (m_dirtyNeedsResort)
+        {
+            setDynamicSortFilter(false);
+            QSortFilterProxyModel::sort(0);
+            setDynamicSortFilter(true);
+            m_dirtyNeedsResort = false;
+        }
+        RCJob* anyPendingJob = nullptr;
+        bool waitingOnCatalog = false; // If we find an asset thats waiting on the catalog, don't assume there's a cyclic dependency.  We'll wait until the catalog is updated and then check again.
+
+        for (int idx = 0; idx < rowCount(); ++idx)
+        {
+            QModelIndex parentIndex = mapToSource(index(idx, 0));
+            RCJob* actualJob = m_sourceModel->getItem(parentIndex.row());
+            if ((actualJob) && (actualJob->GetState() == RCJob::pending))
+            {
+                bool canProcessJob = true;
+                for (const JobDependencyInternal& jobDepedencyInternal : actualJob->GetJobDependencies())
+                {
+                    if (jobDepedencyInternal.m_jobDependency.m_type == AssetBuilderSDK::JobDependencyType::Order || jobDepedencyInternal.m_jobDependency.m_type == AssetBuilderSDK::JobDependencyType::OrderOnce)
+                    {
+                        const AssetBuilderSDK::JobDependency& jobDependency = jobDepedencyInternal.m_jobDependency;
+                        QueueElementID elementId(jobDependency.m_sourceFile.m_sourceFileDependencyPath.c_str(), jobDependency.m_platformIdentifier.c_str(), jobDependency.m_jobKey.c_str());
+
+                        if (m_sourceModel->isInFlight(elementId) || m_sourceModel->isInQueue(elementId))
+                        {
+                            canProcessJob = false;
+                            if (!anyPendingJob)
+                            {
+                                anyPendingJob = actualJob;
+                            }
+                        }
+                        else if(m_sourceModel->isWaitingOnCatalog(elementId))
+                        {
+                            canProcessJob = false;
+                            waitingOnCatalog = true;
+                        }
+                    }
+                }
+
+                if (canProcessJob)
+                {
+                    return actualJob;
+                }
+            }
+        }
+
+        // Either there are no jobs to do or there is a cyclic order job dependency.
+        if (anyPendingJob && m_sourceModel->jobsInFlight() == 0 && !waitingOnCatalog)
+        {
+            AZ_Warning(AssetProcessor::DebugChannel, false, " Cyclic job order dependency detected. Processing job (%s, %s, %s, %s) to unblock.",
+                anyPendingJob->GetJobEntry().m_pathRelativeToWatchFolder.toUtf8().data(), anyPendingJob->GetJobKey().toUtf8().data(),
+                anyPendingJob->GetJobEntry().m_platformInfo.m_identifier.c_str(), anyPendingJob->GetBuilderGuid().ToString<AZStd::string>().c_str());
+            return anyPendingJob;
+        }
+        else
+        {
+            return nullptr;
+        }
+    }
+
+    bool RCQueueSortModel::filterAcceptsRow(int source_row, const QModelIndex& source_parent) const
+    {
+        (void)source_parent;
+        RCJob* actualJob = m_sourceModel->getItem(source_row);
+        if (!actualJob)
+        {
+            return false;
+        }
+
+        if (actualJob->GetState() != RCJob::pending)
+        {
+            return false;
+        }
+
+        return true;
+    }
+
+    bool RCQueueSortModel::lessThan(const QModelIndex& left, const QModelIndex& right) const
+    {
+        RCJob* leftJob = m_sourceModel->getItem(left.row());
+        RCJob* rightJob = m_sourceModel->getItem(right.row());
+
+        // auto fail jobs always take priority to give user feedback asap.
+        bool autoFailLeft = leftJob->IsAutoFail();
+        bool autoFailRight = rightJob->IsAutoFail();
+        if (autoFailLeft)
+        {
+            if (!autoFailRight)
+            {
+                return true; // left before right
+            }
+        }
+        else if (autoFailRight)
+        {
+            return false; // right before left.
+        }
+
+        // first thing to check is in platform.
+        bool leftActive = m_currentlyConnectedPlatforms.contains(leftJob->GetPlatformInfo().m_identifier.c_str());
+        bool rightActive = m_currentlyConnectedPlatforms.contains(rightJob->GetPlatformInfo().m_identifier.c_str());
+
+        if (leftActive)
+        {
+            if (!rightActive)
+            {
+                return true; // left before right
+            }
+        }
+        else if (rightActive)
+        {
+            return false; // right before left.
+        }
+
+        // critical jobs take priority
+        if (leftJob->IsCritical())
+        {
+            if (!rightJob->IsCritical())
+            {
+                return true; // left wins.
+            }
+        }
+        else if (rightJob->IsCritical())
+        {
+            return false; // right wins
+        }
+
+        int leftJobEscalation = leftJob->JobEscalation();
+        int rightJobEscalation = rightJob->JobEscalation();
+
+        // This function, even though its called lessThan(), really is asking, does LEFT come before RIGHT
+        // The higher the escalation, the more important the request, and thus the sooner we want to process the job
+        // Which means if left has a higher escalation number than right, its LESS THAN right.
+        if (leftJobEscalation != rightJobEscalation)
+        {
+            return leftJobEscalation > rightJobEscalation;
+        }
+
+        // arbitrarily, lets have PC get done first since pc-format assets are what the editor uses.
+        if (leftJob->GetPlatformInfo().m_identifier != rightJob->GetPlatformInfo().m_identifier)
+        {
+            if (leftJob->GetPlatformInfo().m_identifier == AzToolsFramework::AssetSystem::GetHostAssetPlatform())
+            {
+                return true; // left wins.
+            }
+            if (rightJob->GetPlatformInfo().m_identifier == AzToolsFramework::AssetSystem::GetHostAssetPlatform())
+            {
+                return false; // right wins
+            }
+        }
+
+        int priorityLeft = leftJob->GetPriority();
+        int priorityRight = rightJob->GetPriority();
+
+        if (priorityLeft != priorityRight)
+        {
+            return priorityLeft > priorityRight;
+        }
+        
+        // if we get all the way down here it means we're dealing with two assets which are not
+        // in any compile groups, not a priority platform, not a priority type, priority platform, etc.
+        // we can arrange these any way we want, but must pick at least a stable order.
+        return leftJob->GetJobEntry().m_jobRunKey < rightJob->GetJobEntry().m_jobRunKey;
+    }
+
+    void RCQueueSortModel::AssetProcessorPlatformConnected(const AZStd::string platform)
+    {
+        QMetaObject::invokeMethod(this, "ProcessPlatformChangeMessage", Qt::QueuedConnection, Q_ARG(QString, QString::fromUtf8(platform.c_str())), Q_ARG(bool, true));
+    }
+
+    void RCQueueSortModel::AssetProcessorPlatformDisconnected(const AZStd::string platform)
+    {
+        QMetaObject::invokeMethod(this, "ProcessPlatformChangeMessage", Qt::QueuedConnection, Q_ARG(QString, QString::fromUtf8(platform.c_str())), Q_ARG(bool, false));
+    }
+
+    void RCQueueSortModel::ProcessPlatformChangeMessage(QString platformName, bool connected)
+    {
+        AZ_TracePrintf(AssetProcessor::DebugChannel, "RCQueueSortModel: Platform %s has %s.", platformName.toUtf8().data(), connected ? "connected" : "disconnected");
+        m_dirtyNeedsResort = true;
+        if (connected)
+        {
+            m_currentlyConnectedPlatforms.insert(platformName);
+        }
+        else
+        {
+            m_currentlyConnectedPlatforms.remove(platformName);
+        }
+    }
+    void RCQueueSortModel::AddJobIdEntry(AssetProcessor::RCJob* rcJob)
+    {
+        m_currentJobRunKeyToJobEntries[rcJob->GetJobEntry().m_jobRunKey] = rcJob;
+    }
+
+    void RCQueueSortModel::RemoveJobIdEntry(AssetProcessor::RCJob* rcJob)
+    {
+        m_currentJobRunKeyToJobEntries.erase(rcJob->GetJobEntry().m_jobRunKey);
+    }
+
+    void RCQueueSortModel::OnEscalateJobs(AssetProcessor::JobIdEscalationList jobIdEscalationList)
+    {
+        for (const auto& jobIdEscalationPair : jobIdEscalationList)
+        {
+            auto found = m_currentJobRunKeyToJobEntries.find(jobIdEscalationPair.first);
+
+            if (found != m_currentJobRunKeyToJobEntries.end())
+            {
+                m_sourceModel->UpdateJobEscalation(found->second, jobIdEscalationPair.second);
+            }
+        }
+    }
+
+} // end namespace AssetProcessor
+

Code/Tools/AssetProcessor/native/resourcecompiler/RCQueueSortModel.h

@@ -0,0 +1,88 @@
+/*
+* All or portions of this file Copyright (c) Amazon.com, Inc. or its affiliates or
+* its licensors.
+*
+* For complete copyright and license terms please see the LICENSE at the root of this
+* distribution (the "License"). All use of this software is governed by the License,
+* or, if provided, by the license below or the license accompanying this file. Do not
+* remove or modify any license notices. This file is distributed on an "AS IS" BASIS,
+* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+*
+*/
+#ifndef ASSETPROCESSOR_RCQUEUESORTMODEL_H
+#define ASSETPROCESSOR_RCQUEUESORTMODEL_H
+
+#if !defined(Q_MOC_RUN)
+#include <QSortFilterProxyModel>
+#include <QSet>
+#include <QString>
+
+
+#include "native/utilities/AssetUtilEBusHelper.h"
+#include <AzCore/std/containers/unordered_map.h>
+#include "native/assetprocessor.h"
+#endif
+
+class RCcontrollerUnitTests;
+
+namespace AssetProcessor
+{
+    class QueueElementID;
+    class RCJobListModel;
+    class RCJob;
+
+    //! This sort and filtering proxy model attaches to the raw RC job list
+    //! And presents it in the optimal order for processing rather than display.
+    //! The current desired order is
+    //!  * Critical (currently Copy) jobs for currently connected platforms
+    //!  * Jobs in Sync Compile Requests for currently connected platforms (with most recent requests first)
+    //!  * Jobs in Async Compile Lists for currently connected platforms
+    //!  * Remaining jobs in currently connected platforms, in priority order
+    //!  (The same, repeated, for unconnected platforms).
+    class RCQueueSortModel
+        : public QSortFilterProxyModel
+        , protected AssetProcessorPlatformBus::Handler
+    {
+        Q_OBJECT
+        friend class ::RCcontrollerUnitTests;
+    public:
+        explicit RCQueueSortModel(QObject* parent = 0);
+
+        void AttachToModel(RCJobListModel* target);
+        RCJob* GetNextPendingJob();
+
+        void AddJobIdEntry(AssetProcessor::RCJob* rcJob);
+        void RemoveJobIdEntry(AssetProcessor::RCJob* rcJob);
+
+
+        // implement QSortFilteRProxyModel:
+        bool filterAcceptsRow(int source_row, const QModelIndex& source_parent) const override;
+        bool lessThan(const QModelIndex& left, const QModelIndex& right) const override;
+
+    public Q_SLOTS:
+        void OnEscalateJobs(AssetProcessor::JobIdEscalationList jobIdEscalationList);
+
+
+    protected:
+
+        typedef AZStd::unordered_map<AZ::s64, AssetProcessor::RCJob*> JobRunKeyToRCJobMap;
+
+        JobRunKeyToRCJobMap m_currentJobRunKeyToJobEntries;
+
+        QSet<QString> m_currentlyConnectedPlatforms;
+        bool m_dirtyNeedsResort = false; // instead of constantly resorting, we resort only when someone wants to pull an element from us
+
+        // ---------------------------------------------------------
+        // AssetProcessorPlatformBus::Handler
+        void AssetProcessorPlatformConnected(const AZStd::string platform) override;
+        void AssetProcessorPlatformDisconnected(const AZStd::string platform) override;
+        // -----------
+
+        RCJobListModel* m_sourceModel;
+    private Q_SLOTS:
+
+        void ProcessPlatformChangeMessage(QString platformName, bool connected);
+    };
+} // namespace AssetProcessor
+
+#endif //ASSETPROCESSOR_RCQUEUESORTMODEL_H

Code/Tools/AssetProcessor/native/resourcecompiler/rccontroller.cpp

@@ -0,0 +1,431 @@
+/*
+* All or portions of this file Copyright (c) Amazon.com, Inc. or its affiliates or
+* its licensors.
+*
+* For complete copyright and license terms please see the LICENSE at the root of this
+* distribution (the "License"). All use of this software is governed by the License,
+* or, if provided, by the license below or the license accompanying this file. Do not
+* remove or modify any license notices. This file is distributed on an "AS IS" BASIS,
+* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+*
+*/
+
+#include "rccontroller.h"
+#include <native/resourcecompiler/RCCommon.h>
+#include <QTimer>
+#include <QThreadPool>
+
+
+
+namespace AssetProcessor
+{
+    RCController::RCController(int cfg_minJobs, int cfg_maxJobs, QObject* parent)
+        : QObject(parent)
+        , m_dispatchingJobs(false)
+        , m_shuttingDown(false)
+    {
+        AssetProcessorPlatformBus::Handler::BusConnect();
+
+        // Determine a good starting value for max jobs
+        int maxJobs = QThread::idealThreadCount();
+        if (maxJobs == -1)
+        {
+            maxJobs = 3;
+        }
+
+        maxJobs = qMax<int>(maxJobs - 1, 1);
+
+        // if the user has specified max jobs in the cfg file, then we obey their request
+        // regardless of whether they have chosen something bad or not - they would have had to explicitly
+        // pick this value (we ship with default 0 meaning auto), so if they've changed it, they intend it that way
+        m_maxJobs = cfg_maxJobs ? qMax(cfg_minJobs, cfg_maxJobs) :  maxJobs;
+
+        m_RCQueueSortModel.AttachToModel(&m_RCJobListModel);
+
+        // make sure that the global thread pool has enough slots to accomidate your request though, since 
+        // by default, the global thread pool has idealThreadCount() slots only.
+        // leave an extra slot for non-job work.
+        int currentMaxThreadCount = QThreadPool::globalInstance()->maxThreadCount();
+        int newMaxThreadCount = qMax<int>(currentMaxThreadCount, m_maxJobs + 1);
+        QThreadPool::globalInstance()->setMaxThreadCount(newMaxThreadCount);
+
+        QObject::connect(this, &RCController::EscalateJobs, &m_RCQueueSortModel, &AssetProcessor::RCQueueSortModel::OnEscalateJobs);
+    }
+
+    RCController::~RCController()
+    {
+        AssetProcessorPlatformBus::Handler::BusDisconnect();
+        m_RCQueueSortModel.AttachToModel(nullptr);
+    }
+
+    RCJobListModel* RCController::GetQueueModel()
+    {
+        return &m_RCJobListModel;
+    }
+
+    void RCController::StartJob(RCJob* rcJob)
+    {
+        Q_ASSERT(rcJob);
+        // request to be notified when job is done
+        QObject::connect(rcJob, &RCJob::Finished, this, [this, rcJob]()
+        {
+            FinishJob(rcJob);
+        }, Qt::QueuedConnection);
+
+        // Mark as "being processed" by moving to Processing list
+        m_RCJobListModel.markAsProcessing(rcJob);
+        m_RCJobListModel.markAsStarted(rcJob);
+        Q_EMIT JobStatusChanged(rcJob->GetJobEntry(), AzToolsFramework::AssetSystem::JobStatus::InProgress);
+        rcJob->Start();
+        Q_EMIT JobStarted(rcJob->GetJobEntry().m_pathRelativeToWatchFolder, QString::fromUtf8(rcJob->GetPlatformInfo().m_identifier.c_str()));
+    }
+
+    void RCController::QuitRequested()
+    {
+        m_shuttingDown = true;
+
+        if (m_RCJobListModel.jobsInFlight() == 0)
+        {
+            Q_EMIT ReadyToQuit(this);
+            return;
+        }
+
+        QTimer::singleShot(10, this, SLOT(QuitRequested()));
+    }
+
+    int RCController::NumberOfPendingCriticalJobsPerPlatform(QString platform)
+    {
+        return m_pendingCriticalJobsPerPlatform[platform.toLower()];
+    }
+
+    int RCController::NumberOfPendingJobsPerPlatform(QString platform)
+    {
+        return m_jobsCountPerPlatform[platform.toLower()];
+    }
+
+    void RCController::FinishJob(RCJob* rcJob)
+    {
+        m_RCQueueSortModel.RemoveJobIdEntry(rcJob);
+        QString platform = rcJob->GetPlatformInfo().m_identifier.c_str();
+        auto found = m_jobsCountPerPlatform.find(platform);
+        if (found != m_jobsCountPerPlatform.end())
+        {
+            int prevCount = found.value();
+            if (prevCount > 0)
+            {
+                int newCount = prevCount - 1;
+                m_jobsCountPerPlatform[platform] = newCount;
+                Q_EMIT JobsInQueuePerPlatform(platform, newCount);
+            }
+        }
+
+        if (rcJob->IsCritical())
+        {
+            int criticalJobsCount = m_pendingCriticalJobsPerPlatform[platform.toLower()] - 1;
+            m_pendingCriticalJobsPerPlatform[platform.toLower()] = criticalJobsCount;
+        }
+
+        if (rcJob->GetState() == RCJob::cancelled)
+        {
+            Q_EMIT FileCancelled(rcJob->GetJobEntry());
+        }
+        else if (rcJob->GetState() != RCJob::completed)
+        {
+            Q_EMIT FileFailed(rcJob->GetJobEntry());
+            Q_EMIT JobStatusChanged(rcJob->GetJobEntry(), AzToolsFramework::AssetSystem::JobStatus::Failed);
+        }
+        else
+        {
+            Q_EMIT FileCompiled(rcJob->GetJobEntry(), AZStd::move(rcJob->GetProcessJobResponse()));
+            Q_EMIT JobStatusChanged(rcJob->GetJobEntry(), AzToolsFramework::AssetSystem::JobStatus::Completed);
+        }
+        
+        // Move to Completed list which will mark as "completed"
+        // unless a different state has been set.
+        m_RCJobListModel.markAsCompleted(rcJob);
+
+        if (!m_dispatchingPaused)
+        {
+            Q_EMIT ActiveJobsCountChanged(aznumeric_cast<unsigned int>(m_RCJobListModel.itemCount()));
+        }
+
+        if (!m_shuttingDown)
+        {
+            // Start next job only if we are not shutting down
+            DispatchJobs();
+
+            // if there is no next job, and nothing is in flight, we are done.
+            if (IsIdle())
+            {
+                Q_EMIT BecameIdle();
+            }
+        }
+    }
+
+    bool RCController::IsIdle()
+    {
+        return ((!m_RCQueueSortModel.GetNextPendingJob()) && (m_RCJobListModel.jobsInFlight() == 0));
+    }
+
+    void RCController::JobSubmitted(JobDetails details)
+    {
+        AssetProcessor::QueueElementID checkFile(details.m_jobEntry.m_databaseSourceName, details.m_jobEntry.m_platformInfo.m_identifier.c_str(), details.m_jobEntry.m_jobKey);
+
+        if (m_RCJobListModel.isInQueue(checkFile))
+        {
+            AZ_TracePrintf(AssetProcessor::DebugChannel, "Job is already in queue and has not started yet - ignored [%s, %s, %s]\n", checkFile.GetInputAssetName().toUtf8().data(), checkFile.GetPlatform().toUtf8().data(), checkFile.GetJobDescriptor().toUtf8().data());
+            return;
+        }
+
+        if (m_RCJobListModel.isInFlight(checkFile))
+        {
+            // if the computed fingerprint is the same as the fingerprint of the in-flight job, this is okay.
+            int existingJobIndex = m_RCJobListModel.GetIndexOfProcessingJob(checkFile);
+            if (existingJobIndex != -1)
+            {
+                RCJob* job = m_RCJobListModel.getItem(existingJobIndex);
+                bool cancelJob = false;
+
+                if (job->GetJobEntry().m_computedFingerprint != details.m_jobEntry.m_computedFingerprint)
+                {
+                    AZ_TracePrintf(AssetProcessor::DebugChannel, "Cancelling Job [%s, %s, %s] with old FP %u, replacing with new FP %u \n", checkFile.GetInputAssetName().toUtf8().data(), checkFile.GetPlatform().toUtf8().data(), checkFile.GetJobDescriptor().toUtf8().data(), job->GetJobEntry().m_computedFingerprint, details.m_jobEntry.m_computedFingerprint);
+                    cancelJob = true;
+                }
+                else if(!job->GetJobDependencies().empty())
+                {
+                    // If a job has dependencies, it's very likely it was re-queued as a result of a dependency being changed
+                    // The in-flight job is probably going to fail at best, or use old data at worst, so cancel the in-flight job
+
+                    AZ_TracePrintf(AssetProcessor::DebugChannel, "Cancelling Job with dependencies [%s, %s, %s], replacing with re-queued job\n", 
+                        checkFile.GetInputAssetName().toUtf8().data(), checkFile.GetPlatform().toUtf8().data(), checkFile.GetJobDescriptor().toUtf8().data());
+                    cancelJob = true;
+                }
+                else
+                {
+                    AZ_TracePrintf(AssetProcessor::DebugChannel, "Job is already in progress but has the same computed fingerprint (%u) - ignored [%s, %s, %s]\n", details.m_jobEntry.m_computedFingerprint,  checkFile.GetInputAssetName().toUtf8().data(), checkFile.GetPlatform().toUtf8().data(), checkFile.GetJobDescriptor().toUtf8().data());
+                    return;
+                }
+
+                if(cancelJob)
+                {
+                    job->SetState(RCJob::JobState::cancelled);
+                    AssetBuilderSDK::JobCommandBus::Event(job->GetJobEntry().m_jobRunKey, &AssetBuilderSDK::JobCommandBus::Events::Cancel);
+                    m_RCJobListModel.UpdateRow(existingJobIndex);
+                }
+            }
+        }
+
+        RCJob* rcJob = new RCJob(&m_RCJobListModel);
+        rcJob->Init(details); // note - move operation.  From this point on you must use the job details to refer to it.
+        m_RCQueueSortModel.AddJobIdEntry(rcJob);
+        m_RCJobListModel.addNewJob(rcJob);
+        QString platformName = rcJob->GetPlatformInfo().m_identifier.c_str();// we need to get the actual platform from the rcJob
+        if (rcJob->IsCritical())
+        {
+            int criticalJobsCount = m_pendingCriticalJobsPerPlatform[platformName.toLower()] + 1;
+            m_pendingCriticalJobsPerPlatform[platformName.toLower()] = criticalJobsCount;
+        }
+        auto found = m_jobsCountPerPlatform.find(platformName);
+        if (found != m_jobsCountPerPlatform.end())
+        {
+            int newCount = found.value() + 1;
+            m_jobsCountPerPlatform[platformName] = newCount;
+        }
+        else
+        {
+            m_jobsCountPerPlatform[platformName] = 1;
+        }
+        Q_EMIT JobsInQueuePerPlatform(platformName, m_jobsCountPerPlatform[platformName]);
+        Q_EMIT JobStatusChanged(rcJob->GetJobEntry(), AzToolsFramework::AssetSystem::JobStatus::Queued);
+
+        if (!m_dispatchingPaused)
+        {
+            Q_EMIT ActiveJobsCountChanged(aznumeric_cast<unsigned int>(m_RCJobListModel.itemCount()));
+        }
+
+        // Start the job we just received if no job currently running
+        if ((!m_shuttingDown) && (!m_dispatchingJobs))
+        {
+            DispatchJobs();
+        }
+    }
+
+    void RCController::SetDispatchPaused(bool pause)
+    {
+        if (m_dispatchingPaused != pause)
+        {
+            m_dispatchingPaused = pause;
+            if (!pause)
+            {
+                if ((!m_shuttingDown) && (!m_dispatchingJobs))
+                {
+                    DispatchJobs();
+                    Q_EMIT ActiveJobsCountChanged(aznumeric_cast<unsigned int>(m_RCJobListModel.itemCount()));
+                }
+            }
+        }
+    }
+
+    void RCController::DispatchJobsImpl()
+    {
+        m_dispatchJobsQueued = false;
+        if (!m_dispatchingJobs)
+        {
+            m_dispatchingJobs = true;
+            RCJob* rcJob = m_RCQueueSortModel.GetNextPendingJob();
+            
+            while (m_RCJobListModel.jobsInFlight() < m_maxJobs && rcJob && !m_shuttingDown)
+            {
+                if (m_dispatchingPaused)
+                {
+                    // note, even if dispatching is "paused" we start all "auto fail jobs" so that user gets instant feedback on failure.
+                    if (!rcJob->IsAutoFail())
+                    {
+                        break;
+                    }
+                }
+
+                StartJob(rcJob);
+                rcJob = m_RCQueueSortModel.GetNextPendingJob();
+            }
+            m_dispatchingJobs = false;
+        }
+    }
+    void RCController::DispatchJobs()
+    {
+        if (!m_dispatchJobsQueued)
+        {
+            m_dispatchJobsQueued = true;
+            QMetaObject::invokeMethod(this, "DispatchJobsImpl", Qt::QueuedConnection);
+        }
+    }
+
+    void RCController::OnRequestCompileGroup(AssetProcessor::NetworkRequestID groupID, QString platform, QString searchTerm, AZ::Data::AssetId assetId, bool isStatusRequest, int searchType)
+    {
+        // someone has asked for a compile group to be created that conforms to that search term.
+        // the goal here is to use a heuristic to find any assets that match the search term and place them in a new group
+        // then respond with the appropriate response.
+
+        // lets do some minimal processing on the search term
+        AssetProcessor::JobIdEscalationList escalationList;
+        QSet<AssetProcessor::QueueElementID> results;
+
+        if (assetId.IsValid())
+        {
+            m_RCJobListModel.PerformUUIDSearch(assetId.m_guid, platform, results, escalationList, isStatusRequest);
+        }
+        else
+        {
+            m_RCJobListModel.PerformHeuristicSearch(AssetUtilities::NormalizeAndRemoveAlias(searchTerm), platform, results, escalationList, isStatusRequest, searchType);
+        }
+
+        if (results.isEmpty())
+        {
+            // nothing found
+            Q_EMIT CompileGroupCreated(groupID, AzFramework::AssetSystem::AssetStatus_Unknown);
+
+            AZ_TracePrintf(AssetProcessor::DebugChannel, "OnRequestCompileGroup:  %s - %s requested, but no matching source assets found.\n", searchTerm.toUtf8().constData(), assetId.ToString<AZStd::string>().c_str());
+        }
+        else
+        {
+            // it is not necessary to denote the search terms or list of results here because
+            // PerformHeursticSearch already prints out the results.
+            m_RCQueueSortModel.OnEscalateJobs(escalationList);
+            
+            m_activeCompileGroups.push_back(AssetCompileGroup());
+            m_activeCompileGroups.back().m_groupMembers.swap(results);
+            m_activeCompileGroups.back().m_requestID = groupID;
+
+            Q_EMIT CompileGroupCreated(groupID, AzFramework::AssetSystem::AssetStatus_Queued);
+        }
+    }
+
+    void RCController::OnEscalateJobsBySearchTerm(QString platform, QString searchTerm)
+    {
+        AssetProcessor::JobIdEscalationList escalationList;
+        QSet<AssetProcessor::QueueElementID> results;
+        m_RCJobListModel.PerformHeuristicSearch(AssetUtilities::NormalizeAndRemoveAlias(searchTerm), platform, results, escalationList, true);
+
+        if (!results.isEmpty())
+        {
+            // it is not necessary to denote the search terms or list of results here because
+            // PerformHeursticSearch already prints out the results.
+            m_RCQueueSortModel.OnEscalateJobs(escalationList);
+        }
+        // do not print a warning out when this fails, its fine for things to escalate jobs as a matter of course just to "make sure" they are escalated
+        // and its fine if none are in the build queue.
+    }
+
+    void RCController::OnEscalateJobsBySourceUUID(QString platform, AZ::Uuid sourceUuid)
+    {
+        AssetProcessor::JobIdEscalationList escalationList;
+        QSet<AssetProcessor::QueueElementID> results;
+        m_RCJobListModel.PerformUUIDSearch(sourceUuid, platform, results, escalationList, true);
+
+        if (!results.isEmpty())
+        {
+            for (const AssetProcessor::QueueElementID& result : results)
+            {
+                AZ_TracePrintf(AssetProcessor::DebugChannel, "OnEscalateJobsBySourceUUID:  %s --> %s\n", sourceUuid.ToString<AZStd::string>().c_str(), result.GetInputAssetName().toUtf8().constData());
+            }
+            m_RCQueueSortModel.OnEscalateJobs(escalationList);
+        }
+        // do not print a warning out when this fails, its fine for things to escalate jobs as a matter of course just to "make sure" they are escalated
+        // and its fine if none are in the build queue.
+    }
+
+    void RCController::OnJobComplete(JobEntry completeEntry, AzToolsFramework::AssetSystem::JobStatus state)
+    {
+        if (m_activeCompileGroups.empty())
+        {
+            return;
+        }
+
+        QueueElementID jobQueueId(completeEntry.m_databaseSourceName, completeEntry.m_platformInfo.m_identifier.c_str(), completeEntry.m_jobKey);
+
+        // only the 'completed' status means success:
+        bool statusSucceeded = (state == AzToolsFramework::AssetSystem::JobStatus::Completed);
+
+        // start at the end so that we can actually erase the compile groups and not skip any:
+        for (int groupIdx = m_activeCompileGroups.size() - 1; groupIdx >= 0; --groupIdx)
+        {
+            AssetCompileGroup& compileGroup = m_activeCompileGroups[groupIdx];
+            auto it = compileGroup.m_groupMembers.find(jobQueueId);
+            if (it != compileGroup.m_groupMembers.end())
+            {
+                compileGroup.m_groupMembers.erase(it);
+                if ((compileGroup.m_groupMembers.isEmpty()) || (!statusSucceeded))
+                {
+                    // if we get here, we're either empty (and succeeded) or we failed one and have now failed
+                    Q_EMIT CompileGroupFinished(compileGroup.m_requestID, statusSucceeded ? AzFramework::AssetSystem::AssetStatus_Compiled: AzFramework::AssetSystem::AssetStatus_Failed);
+                    m_activeCompileGroups.removeAt(groupIdx);
+                }
+            }
+        }
+    }
+    
+    void RCController::RemoveJobsBySource(QString relSourceFileDatabaseName)
+    {
+        // some jobs may have not been started yet, these need to be removed manually
+        AZStd::vector<RCJob*> pendingJobs;
+
+        m_RCJobListModel.EraseJobs(relSourceFileDatabaseName, pendingJobs);
+
+        // force finish all pending jobs
+        for (auto* rcJob : pendingJobs)
+        {
+            FinishJob(rcJob);
+        }
+    }
+
+    void RCController::OnAddedToCatalog(JobEntry jobEntry)
+    {
+        AssetProcessor::QueueElementID checkFile(jobEntry.m_databaseSourceName, jobEntry.m_platformInfo.m_identifier.c_str(), jobEntry.m_jobKey);
+
+        m_RCJobListModel.markAsCataloged(checkFile);
+
+        DispatchJobs();
+    }
+
+} // Namespace AssetProcessor
+

Code/Tools/AssetProcessor/native/resourcecompiler/rccontroller.h

@@ -0,0 +1,144 @@
+/*
+* All or portions of this file Copyright (c) Amazon.com, Inc. or its affiliates or
+* its licensors.
+*
+* For complete copyright and license terms please see the LICENSE at the root of this
+* distribution (the "License"). All use of this software is governed by the License,
+* or, if provided, by the license below or the license accompanying this file. Do not
+* remove or modify any license notices. This file is distributed on an "AS IS" BASIS,
+* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+*
+*/
+#ifndef RCCONTROLLER_H
+#define RCCONTROLLER_H
+
+#if !defined(Q_MOC_RUN)
+#include "RCCommon.h"
+
+#include <QObject>
+#include <QProcess>
+#include <QDir>
+#include <QList>
+#include "native/utilities/AssetUtilEBusHelper.h"
+
+#include "rcjoblistmodel.h"
+#include "RCQueueSortModel.h"
+
+#include <AzFramework/Asset/AssetProcessorMessages.h>
+#include <AzToolsFramework/API/EditorAssetSystemAPI.h>
+#endif
+
+class RCcontrollerUnitTests;
+
+namespace AssetProcessor
+{
+    /**
+     * The RCController class controls the receiving of job requests, adding them to the model,
+     * running RC and sending of responses
+     */
+    class RCController
+        : public QObject
+        , public AssetProcessorPlatformBus::Handler
+    {
+        friend class ::RCcontrollerUnitTests;
+        Q_OBJECT
+    public:
+        enum CommandType
+        {
+            cmdUnknown = 0,
+            cmdExecute,
+            cmdTerminate
+        };
+        RCController() = default;
+        explicit RCController(int minJobs, int maxJobs, QObject* parent = 0);
+        virtual ~RCController();
+
+        AssetProcessor::RCJobListModel* GetQueueModel();
+
+        void StartJob(AssetProcessor::RCJob* rcJob);
+        int NumberOfPendingCriticalJobsPerPlatform(QString platform);
+
+        void SetSystemRoot(const QDir& systemRoot);
+        int NumberOfPendingJobsPerPlatform(QString platform);
+        bool IsIdle();
+        bool IsPriorityCopyJob(AssetProcessor::RCJob* rcJob);
+    Q_SIGNALS:
+        void FileCompiled(JobEntry entry, AssetBuilderSDK::ProcessJobResponse response);
+        void FileFailed(JobEntry entry);
+        void FileCancelled(JobEntry entry);
+        //void AssetStatus(JobEntry jobEntry, AzFramework::AssetProcessor::AssetStatus status);
+        void RcError(QString error);
+        void ReadyToQuit(QObject* source); //After receiving QuitRequested, you must send this when its safe
+
+        ///! JobStarted will notify with a path name relative to the watch folder it was found in (not the database sourcename column)
+        void JobStarted(QString inputFile, QString platform);
+        void JobStatusChanged(JobEntry entry, AzToolsFramework::AssetSystem::JobStatus status);
+        void JobsInQueuePerPlatform(QString platform, int jobs);
+        void ActiveJobsCountChanged(unsigned int jobs); // This is the count of jobs which are either queued or inflight 
+
+        void BecameIdle();
+
+        //! This will be signalled upon compile group creation - or failure to do so (in which case status will be unknown)
+        void CompileGroupCreated(AssetProcessor::NetworkRequestID groupID, AzFramework::AssetSystem::AssetStatus status);
+        //! Once a compile group has an error or finished, this will be invoked.
+        void CompileGroupFinished(AssetProcessor::NetworkRequestID groupID, AzFramework::AssetSystem::AssetStatus status);
+
+        void EscalateJobs(AssetProcessor::JobIdEscalationList jobIdEscalationList);
+
+    public Q_SLOTS:
+        void JobSubmitted(JobDetails details);
+        void QuitRequested();
+
+        //! This will be called in order to create a compile group and start tracking it.
+        void OnRequestCompileGroup(AssetProcessor::NetworkRequestID groupID, QString platform, QString searchTerm, AZ::Data::AssetId assetId, bool isStatusRequest = true, int searchType = 0);
+
+        void OnEscalateJobsBySearchTerm(QString platform, QString searchTerm);
+        void OnEscalateJobsBySourceUUID(QString platform, AZ::Uuid sourceUuid);
+
+        void DispatchJobs();
+        void DispatchJobsImpl();
+
+        //! Pause or unpause dispatching, only necessary on startup to avoid thrashing and make sure no jobs jump the gun.
+        void SetDispatchPaused(bool pause);
+
+        //! All jobs which match this source will be cancelled or removed.  Note that relSourceFile should have any applicable output prefixes!
+        void RemoveJobsBySource(QString relSourceFileDatabaseName);
+
+        // when the AP is truly done with a particular job and its going to be deleted and nothing more cares about it,
+        // this function is called. this allows us to synchronize the various threads (catalog, queue, etc) to know that
+        // its completely done.
+        void OnJobComplete(JobEntry completeEntry, AzToolsFramework::AssetSystem::JobStatus status);
+        void OnAddedToCatalog(JobEntry jobEntry);
+
+    private:
+        void FinishJob(AssetProcessor::RCJob* rcJob);
+
+        unsigned int m_maxJobs;
+
+        bool m_dispatchingJobs = false;
+        bool m_shuttingDown = false;
+        bool m_dispatchingPaused = true;// dispatching starts out paused.
+        bool m_dispatchJobsQueued = false;
+
+        QMap<QString, int> m_jobsCountPerPlatform;// This stores the count of jobs per platform in the RC Queue
+        QMap<QString, int> m_pendingCriticalJobsPerPlatform;// This stores the count of pending critical jobs per platform in the RC Queue
+        AssetProcessor::RCJobListModel m_RCJobListModel;
+        AssetProcessor::RCQueueSortModel m_RCQueueSortModel;
+
+        //! An Asset Compile Group is a set of assets that we're tracking the compilation of
+        //! It consists of a whole bunch of assets and is considered to be "complete" when either one of the assets in the group fails
+        //! Or all assets in the group have finished.
+        class AssetCompileGroup
+        {
+        public:
+            AssetProcessor::NetworkRequestID m_requestID;
+            QSet<AssetProcessor::QueueElementID> m_groupMembers;
+        };
+
+        QList<AssetCompileGroup> m_activeCompileGroups;
+        
+    };
+} // namespace AssetProcessor
+
+
+#endif // RCCONTROLLER_H

Code/Tools/AssetProcessor/native/resourcecompiler/rcjob.cpp

@@ -0,0 +1,965 @@
+/*
+* All or portions of this file Copyright (c) Amazon.com, Inc. or its affiliates or
+* its licensors.
+*
+* For complete copyright and license terms please see the LICENSE at the root of this
+* distribution (the "License"). All use of this software is governed by the License,
+* or, if provided, by the license below or the license accompanying this file. Do not
+* remove or modify any license notices. This file is distributed on an "AS IS" BASIS,
+* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+*
+*/
+#include "rcjob.h"
+
+
+
+#include <AzToolsFramework/UI/Logging/LogLine.h>
+
+#include <native/utilities/BuilderManager.h>
+#include <native/utilities/ThreadHelper.h>
+
+#include <QtConcurrent/QtConcurrentRun>
+#include <QElapsedTimer>
+
+#include "native/utilities/JobDiagnosticTracker.h"
+
+
+namespace
+{
+    unsigned long s_jobSerial = 1;
+    bool s_typesRegistered = false;
+    // You have up to 60 minutes to finish processing an asset.
+    // This was increased from 10 to account for PVRTC compression
+    // taking up to an hour for large normal map textures, and should
+    // be reduced again once we move to the ASTC compression format, or
+    // find another solution to reduce processing times to be reasonable.
+    const unsigned int g_jobMaximumWaitTime = 1000 * 60 * 60;
+
+    const unsigned int g_sleepDurationForLockingAndFingerprintChecking = 100;
+
+    const unsigned int g_graceTimeBeforeLockingAndFingerprintChecking = 300;
+
+    const unsigned int g_timeoutInSecsForRetryingCopy = 30;
+
+    const char* const s_tempString = "%TEMP%";
+    const char* const s_jobLogFileName = "jobLog.xml";
+
+    bool MoveCopyFile(QString sourceFile, QString productFile, bool isCopyJob = false)
+    {
+        if (!isCopyJob && (AssetUtilities::MoveFileWithTimeout(sourceFile, productFile, g_timeoutInSecsForRetryingCopy)))
+        {
+            //We do not want to rename the file if it is a copy job
+            return true;
+        }
+        else if (AssetUtilities::CopyFileWithTimeout(sourceFile, productFile, g_timeoutInSecsForRetryingCopy))
+        {
+            // try to copy instead
+            return true;
+        }
+        AZ_TracePrintf(AssetBuilderSDK::ErrorWindow, "Failed to move OR copy file from Source directory: %s  to Destination Directory: %s", sourceFile.toUtf8().data(), productFile.toUtf8().data());
+        return false;
+    }
+}
+
+using namespace AssetProcessor;
+
+bool Params::IsValidParams() const
+{
+    return (!m_finalOutputDir.isEmpty());
+}
+
+bool RCParams::IsValidParams() const
+{
+    return (
+        (!m_rcExe.isEmpty()) &&
+        (!m_rootDir.isEmpty()) &&
+        (!m_inputFile.isEmpty()) &&
+        Params::IsValidParams()
+        );
+}
+
+namespace AssetProcessor
+{
+    RCJob::RCJob(QObject* parent)
+        : QObject(parent)
+        , m_timeCreated(QDateTime::currentDateTime())
+        , m_scanFolderID(0)
+    {
+        m_jobState = RCJob::pending;
+
+        if (!s_typesRegistered)
+        {
+            qRegisterMetaType<RCParams>("RCParams");
+            qRegisterMetaType<BuilderParams>("BuilderParams");
+            qRegisterMetaType<JobOutputInfo>("JobOutputInfo");
+            s_typesRegistered = true;
+        }
+    }
+
+    RCJob::~RCJob()
+    {
+    }
+
+    void RCJob::Init(JobDetails& details)
+    {
+        m_jobDetails = AZStd::move(details);
+        m_queueElementID = QueueElementID(GetJobEntry().m_databaseSourceName, GetPlatformInfo().m_identifier.c_str(), GetJobKey());
+    }
+
+    const JobEntry& RCJob::GetJobEntry() const
+    {
+        return m_jobDetails.m_jobEntry;
+    }
+
+    QDateTime RCJob::GetTimeCreated() const
+    {
+        return m_timeCreated;
+    }
+
+    void RCJob::SetTimeCreated(const QDateTime& timeCreated)
+    {
+        m_timeCreated = timeCreated;
+    }
+
+    QDateTime RCJob::GetTimeLaunched() const
+    {
+        return m_timeLaunched;
+    }
+
+    void RCJob::SetTimeLaunched(const QDateTime& timeLaunched)
+    {
+        m_timeLaunched = timeLaunched;
+    }
+
+    QDateTime RCJob::GetTimeCompleted() const
+    {
+        return m_timeCompleted;
+    }
+
+    void RCJob::SetTimeCompleted(const QDateTime& timeCompleted)
+    {
+        m_timeCompleted = timeCompleted;
+    }
+
+    AZ::u32 RCJob::GetOriginalFingerprint() const
+    {
+        return m_jobDetails.m_jobEntry.m_computedFingerprint;
+    }
+
+    void RCJob::SetOriginalFingerprint(unsigned int fingerprint)
+    {
+        m_jobDetails.m_jobEntry.m_computedFingerprint = fingerprint;
+    }
+
+    RCJob::JobState RCJob::GetState() const
+    {
+        return m_jobState;
+    }
+
+    void RCJob::SetState(const JobState& state)
+    {
+        bool wasPending = (m_jobState == pending);
+        m_jobState = state;
+
+        if ((wasPending)&&(m_jobState == cancelled))
+        {
+            // if we were pending (had not started yet) and we are now canceled, we still have to emit the finished signal
+            // so that all the various systems waiting for us can do their housekeeping.
+            Q_EMIT Finished();
+        }
+    }
+
+    void RCJob::SetJobEscalation(int jobEscalation)
+    {
+        m_JobEscalation = jobEscalation;
+    }
+
+    void RCJob::SetCheckExclusiveLock(bool value)
+    {
+        m_jobDetails.m_jobEntry.m_checkExclusiveLock = value;
+    }
+
+    QString RCJob::GetStateDescription(const RCJob::JobState& state)
+    {
+        switch (state)
+        {
+        case RCJob::pending:
+            return tr("Pending");
+        case RCJob::processing:
+            return tr("Processing");
+        case RCJob::completed:
+            return tr("Completed");
+        case RCJob::crashed:
+            return tr("Crashed");
+        case RCJob::terminated:
+            return tr("Terminated");
+        case RCJob::failed:
+            return tr("Failed");
+        case RCJob::cancelled:
+            return tr("Cancelled");
+        }
+        return QString();
+    }
+
+    const AZ::Uuid& RCJob::GetInputFileUuid() const
+    {
+        return m_jobDetails.m_jobEntry.m_sourceFileUUID;
+    }
+
+    QString RCJob::GetFinalOutputPath() const
+    {
+        return m_jobDetails.m_destinationPath;
+    }
+
+    const AssetBuilderSDK::PlatformInfo& RCJob::GetPlatformInfo() const
+    {
+        return m_jobDetails.m_jobEntry.m_platformInfo;
+    }
+
+    AssetBuilderSDK::ProcessJobResponse& RCJob::GetProcessJobResponse()
+    {
+        return m_processJobResponse;
+    }
+
+    void RCJob::PopulateProcessJobRequest(AssetBuilderSDK::ProcessJobRequest& processJobRequest)
+    {
+        processJobRequest.m_jobDescription.m_critical = IsCritical();
+        processJobRequest.m_jobDescription.m_additionalFingerprintInfo = m_jobDetails.m_extraInformationForFingerprinting;
+        processJobRequest.m_jobDescription.m_jobKey = GetJobKey().toUtf8().data();
+        processJobRequest.m_jobDescription.m_jobParameters = AZStd::move(m_jobDetails.m_jobParam);
+        processJobRequest.m_jobDescription.SetPlatformIdentifier(GetPlatformInfo().m_identifier.c_str());
+        processJobRequest.m_jobDescription.m_priority = GetPriority();
+        processJobRequest.m_platformInfo = GetPlatformInfo();
+        processJobRequest.m_builderGuid = GetBuilderGuid();
+        processJobRequest.m_sourceFile = GetJobEntry().m_pathRelativeToWatchFolder.toUtf8().data();
+        processJobRequest.m_sourceFileUUID = GetInputFileUuid();
+        processJobRequest.m_watchFolder = GetJobEntry().m_watchFolderPath.toUtf8().data();
+        processJobRequest.m_fullPath = GetJobEntry().GetAbsoluteSourcePath().toUtf8().data();
+        processJobRequest.m_jobId = GetJobEntry().m_jobRunKey;
+    }
+
+    QString RCJob::GetJobKey() const
+    {
+        return m_jobDetails.m_jobEntry.m_jobKey;
+    }
+
+    AZ::Uuid RCJob::GetBuilderGuid() const
+    {
+        return m_jobDetails.m_jobEntry.m_builderGuid;
+    }
+
+    bool RCJob::IsCritical() const
+    {
+        return m_jobDetails.m_critical;
+    }
+
+    bool RCJob::IsAutoFail() const
+    {
+        return m_jobDetails.m_autoFail;
+    }
+
+    int RCJob::GetPriority() const
+    {
+        return m_jobDetails.m_priority;
+    }
+
+    const AZStd::vector<AssetProcessor::JobDependencyInternal>& RCJob::GetJobDependencies()
+    {
+        return m_jobDetails.m_jobDependencyList;
+    }
+
+    void RCJob::Start()
+    {
+        // the following trace can be uncommented if there is a need to deeply inspect job running.
+        //AZ_TracePrintf(AssetProcessor::DebugChannel, "JobTrace Start(%i %s,%s,%s)\n", this, GetInputFileAbsolutePath().toUtf8().data(), GetPlatform().toUtf8().data(), GetJobKey().toUtf8().data());
+
+        AssetUtilities::QuitListener listener;
+        listener.BusConnect();
+        RCParams rc(this);
+        BuilderParams builderParams(this);
+
+        //Create the process job request
+        AssetBuilderSDK::ProcessJobRequest processJobRequest;
+        PopulateProcessJobRequest(processJobRequest);
+
+        builderParams.m_processJobRequest = processJobRequest;
+        builderParams.m_finalOutputDir = GetFinalOutputPath();
+        builderParams.m_assetBuilderDesc = m_jobDetails.m_assetBuilderDesc;
+
+        // when the job finishes, record the results and emit Finished()
+        connect(this, &RCJob::JobFinished, this, [this](AssetBuilderSDK::ProcessJobResponse result)
+            {
+                m_processJobResponse = AZStd::move(result);
+                switch (m_processJobResponse.m_resultCode)
+                {
+                case AssetBuilderSDK::ProcessJobResult_Crashed:
+                    {
+                        SetState(crashed);
+                    }
+                    break;
+                case AssetBuilderSDK::ProcessJobResult_Success:
+                    {
+                        SetState(completed);
+                    }
+                    break;
+                case AssetBuilderSDK::ProcessJobResult_Cancelled:
+                    {
+                        SetState(cancelled);
+                    }
+                    break;
+                default:
+                    {
+                        SetState(failed);
+                    }
+                    break;
+                }
+                Q_EMIT Finished();
+            });
+
+        if (!listener.WasQuitRequested())
+        {
+            QtConcurrent::run(&RCJob::ExecuteBuilderCommand, builderParams);
+        }
+        else
+        {
+            AZ_TracePrintf(AssetBuilderSDK::ErrorWindow, "Job canceled due to quit being requested.");
+            SetState(terminated);
+            Q_EMIT Finished();
+        }
+        listener.BusDisconnect();
+    }
+
+    void RCJob::ExecuteBuilderCommand(BuilderParams builderParams)
+    {
+        // Note: this occurs inside a worker thread.
+
+        // listen for the user quitting (CTRL-C or otherwise)
+        AssetUtilities::QuitListener listener;
+        listener.BusConnect();
+        QElapsedTimer ticker;
+        ticker.start();
+        AssetBuilderSDK::ProcessJobResponse result;
+
+        if (builderParams.m_rcJob->m_jobDetails.m_autoFail)
+        {
+            // if this is an auto-fail job, we should avoid doing any additional work besides the work required to fail the job and 
+            // write the details into its log.  This is because Auto-fail jobs have 'incomplete' job descriptors, and only exist to 
+            // force a job to fail with a reasonable log file stating the reason for failure.  An example of where it is useful to
+            // use auto-fail jobs is when, after compilation was successful, something goes wrong integrating the result into the
+            // cache.  (For example, files collide, or the product file name would be too long).  The job will have at that point
+            // already completed, the thread long gone, so we can 'append' to the log in this manner post-build by creating a new
+            // job that will automatically fail and ingest the old (success) log along with additional fail reasons and then fail.
+            AutoFailJob(builderParams);
+            result.m_resultCode = AssetBuilderSDK::ProcessJobResult_Failed;
+            Q_EMIT builderParams.m_rcJob->JobFinished(result);
+            return;
+        }
+
+        // We are adding a grace time before we check exclusive lock and validate the fingerprint of the file.
+        // This grace time should prevent multiple jobs from getting added to the queue if the source file is still updating.
+        qint64 milliSecsDiff = QDateTime::currentMSecsSinceEpoch() - builderParams.m_rcJob->GetJobEntry().m_computedFingerprintTimeStamp;
+        if (milliSecsDiff < g_graceTimeBeforeLockingAndFingerprintChecking)
+        {
+            QThread::msleep(aznumeric_cast<unsigned long>(g_graceTimeBeforeLockingAndFingerprintChecking - milliSecsDiff));
+        }
+        // Lock and unlock the source file to ensure it is not still open by another process.
+        // This prevents premature processing of some source files that are opened for writing, but are zero bytes for longer than the modification threshhold
+        QString inputFile = builderParams.m_rcJob->GetJobEntry().GetAbsoluteSourcePath();
+        if (builderParams.m_rcJob->GetJobEntry().m_checkExclusiveLock && QFile::exists(inputFile))
+        {
+            // We will only continue once we get exclusive lock on the source file
+            while (!AssetUtilities::CheckCanLock(inputFile))
+            {
+                QThread::msleep(g_sleepDurationForLockingAndFingerprintChecking);
+                if (listener.WasQuitRequested() || (ticker.elapsed() > g_jobMaximumWaitTime))
+                {
+                    result.m_resultCode = AssetBuilderSDK::ProcessJobResult_Cancelled;
+                    Q_EMIT builderParams.m_rcJob->JobFinished(result);
+                    return;
+                }
+            }
+        }
+
+        // We will only continue once the fingerprint of the file stops changing
+        unsigned int fingerprint = AssetUtilities::GenerateFingerprint(builderParams.m_rcJob->m_jobDetails);
+        while (fingerprint != builderParams.m_rcJob->GetOriginalFingerprint())
+        {
+            builderParams.m_rcJob->SetOriginalFingerprint(fingerprint);
+            QThread::msleep(g_sleepDurationForLockingAndFingerprintChecking);
+            
+            if (listener.WasQuitRequested() || (ticker.elapsed() > g_jobMaximumWaitTime))
+            {
+                result.m_resultCode = AssetBuilderSDK::ProcessJobResult_Cancelled;
+                Q_EMIT builderParams.m_rcJob->JobFinished(result);
+                return;
+            }
+
+            fingerprint = AssetUtilities::GenerateFingerprint(builderParams.m_rcJob->m_jobDetails);
+        }        
+
+        Q_EMIT builderParams.m_rcJob->BeginWork();
+        // We will actually start working on the job after this point and even if RcController gets the same job again, we will put it in the queue for processing
+        builderParams.m_rcJob->DoWork(result, builderParams, listener);
+        Q_EMIT builderParams.m_rcJob->JobFinished(result);
+    }
+
+    void RCJob::AutoFailJob(BuilderParams& builderParams)
+    {
+        // force the fail data to be captured to the log file.  
+        // because this is being executed in a thread worker, this won't stomp the main thread's job id.
+        AssetProcessor::SetThreadLocalJobId(builderParams.m_rcJob->GetJobEntry().m_jobRunKey);
+        AssetUtilities::JobLogTraceListener jobLogTraceListener(builderParams.m_rcJob->m_jobDetails.m_jobEntry);
+
+        QString sourceFullPath(builderParams.m_processJobRequest.m_fullPath.c_str());
+        auto failReason = builderParams.m_processJobRequest.m_jobDescription.m_jobParameters.find(AZ_CRC(AssetProcessor::AutoFailReasonKey));
+        if (failReason != builderParams.m_processJobRequest.m_jobDescription.m_jobParameters.end())
+        {
+            // you are allowed to have many lines in your fail reason.
+            AZ_Error(AssetBuilderSDK::ErrorWindow, false, "Failed processing %s", sourceFullPath.toUtf8().data());
+            AZStd::vector<AZStd::string> delimited;
+            AzFramework::StringFunc::Tokenize(failReason->second.c_str(), delimited, "\n");
+            for (const AZStd::string& token : delimited)
+            {
+                AZ_Error(AssetBuilderSDK::ErrorWindow, false, "%s", token.c_str());
+            }
+        }
+        else
+        {
+            // since we didn't have a custom auto-fail reason, add a token to the log file that will help with 
+            // forensic debugging to differentiate auto-fails from regular fails (although it should also be 
+            // obvious from the output in other ways)
+            AZ_TracePrintf("Debug", "(auto-failed)\n"); 
+        }
+        auto failLogFile = builderParams.m_processJobRequest.m_jobDescription.m_jobParameters.find(AZ_CRC(AssetProcessor::AutoFailLogFile));
+        if (failLogFile != builderParams.m_processJobRequest.m_jobDescription.m_jobParameters.end())
+        {
+            AzToolsFramework::Logging::LogLine::ParseLog(failLogFile->second.c_str(), failLogFile->second.size(),
+                [](AzToolsFramework::Logging::LogLine& target)
+            {
+                switch (target.GetLogType())
+                {
+                case AzToolsFramework::Logging::LogLine::TYPE_DEBUG:
+                    AZ_TracePrintf(target.GetLogWindow().c_str(), "%s", target.GetLogMessage().c_str());
+                    break;
+                case AzToolsFramework::Logging::LogLine::TYPE_MESSAGE:
+                    AZ_TracePrintf(target.GetLogWindow().c_str(), "%s", target.GetLogMessage().c_str());
+                    break;
+                case AzToolsFramework::Logging::LogLine::TYPE_WARNING:
+                    AZ_Warning(target.GetLogWindow().c_str(), false, "%s", target.GetLogMessage().c_str());
+                    break;
+                case AzToolsFramework::Logging::LogLine::TYPE_ERROR:
+                    AZ_Error(target.GetLogWindow().c_str(), false, "%s", target.GetLogMessage().c_str());
+                    break;
+                case AzToolsFramework::Logging::LogLine::TYPE_CONTEXT:
+                    AZ_TracePrintf(target.GetLogWindow().c_str(), " %s", target.GetLogMessage().c_str());
+                    break;
+                }
+            });
+        }
+
+        // note that this line below is printed out to be consistent with the output from a job that normally failed, so 
+        // applications reading log file will find it.
+        AZ_Error(AssetBuilderSDK::ErrorWindow, false, "Builder indicated that the job has failed.\n"); 
+        
+        if (builderParams.m_processJobRequest.m_jobDescription.m_jobParameters.find(AZ_CRC(AssetProcessor::AutoFailOmitFromDatabaseKey)) != builderParams.m_processJobRequest.m_jobDescription.m_jobParameters.end())
+        {
+            // we don't add Auto-fail jobs to the database if they have asked to be emitted.
+            builderParams.m_rcJob->m_jobDetails.m_jobEntry.m_addToDatabase = false;
+        }
+
+        AssetProcessor::SetThreadLocalJobId(0);
+    }
+
+
+    void RCJob::DoWork(AssetBuilderSDK::ProcessJobResponse& result, BuilderParams& builderParams, AssetUtilities::QuitListener& listener)
+    {
+        // Setting job id for logging purposes
+        AssetProcessor::SetThreadLocalJobId(builderParams.m_rcJob->GetJobEntry().m_jobRunKey);
+        AssetUtilities::JobLogTraceListener jobLogTraceListener(builderParams.m_rcJob->m_jobDetails.m_jobEntry);
+
+        {
+            AssetBuilderSDK::JobCancelListener JobCancelListener(builderParams.m_rcJob->m_jobDetails.m_jobEntry.m_jobRunKey);
+            result.m_resultCode = AssetBuilderSDK::ProcessJobResult_Failed; // failed by default
+
+            auto warningMessage = builderParams.m_processJobRequest.m_jobDescription.m_jobParameters.find(AZ_CRC(AssetProcessor::JobWarningKey));
+            if (warningMessage != builderParams.m_processJobRequest.m_jobDescription.m_jobParameters.end())
+            {
+                // you are allowed to have many lines in your warning message.
+                AZStd::vector<AZStd::string> delimited;
+                AzFramework::StringFunc::Tokenize(warningMessage->second.c_str(), delimited, "\n");
+                for (const AZStd::string& token : delimited)
+                {
+                    AZ_Warning(AssetBuilderSDK::WarningWindow, false, "%s", token.c_str());
+                }
+            }
+
+            // create a temporary directory for Builder to work in.
+            // lets make it as a subdir of a known temp dir
+
+            QString workFolder;
+
+            if (!AssetUtilities::CreateTempWorkspace(workFolder))
+            {
+                AZ_Error(AssetBuilderSDK::ErrorWindow, false, "Could not create temporary directory for Builder!\n");
+                result.m_resultCode = AssetBuilderSDK::ProcessJobResult_Failed;
+                Q_EMIT builderParams.m_rcJob->JobFinished(result);
+                return;
+            }
+
+            builderParams.m_processJobRequest.m_tempDirPath = AZStd::string(workFolder.toUtf8().data());
+
+            QString sourceFullPath(builderParams.m_processJobRequest.m_fullPath.c_str());
+            
+            if (sourceFullPath.length() >= AP_MAX_PATH_LEN)
+            {
+                AZ_Warning(AssetBuilderSDK::WarningWindow, false, "Source Asset: %s filepath length %d exceeds the maximum path length (%d) allowed.\n", sourceFullPath.toUtf8().data(), sourceFullPath.length(), AP_MAX_PATH_LEN);
+                result.m_resultCode = AssetBuilderSDK::ProcessJobResult_Failed;
+            }
+            else
+            {
+                if (!JobCancelListener.IsCancelled())
+                {
+                    bool runProcessJob = true;
+                    if (m_jobDetails.m_checkServer)
+                    {
+                        QFileInfo fileInfo(builderParams.m_processJobRequest.m_sourceFile.c_str());
+                        builderParams.m_serverKey = QString("%1_%2_%3_%4").arg(fileInfo.completeBaseName(), builderParams.m_processJobRequest.m_jobDescription.m_jobKey.c_str(), builderParams.m_processJobRequest.m_platformInfo.m_identifier.c_str()).arg(builderParams.m_rcJob->GetOriginalFingerprint());
+                        bool operationResult = false;
+                        if (AssetUtilities::InServerMode())
+                        {
+                            // sending process job command to the builder
+                            builderParams.m_assetBuilderDesc.m_processJobFunction(builderParams.m_processJobRequest, result);
+                            runProcessJob = false;
+                            if (result.m_resultCode == AssetBuilderSDK::ProcessJobResult_Success)
+                            {
+                                auto beforeStoreResult = BeforeStoringJobResult(builderParams, result);
+                                if (beforeStoreResult.IsSuccess())
+                                {
+                                    AssetProcessor::AssetServerBus::BroadcastResult(operationResult, &AssetProcessor::AssetServerBusTraits::StoreJobResult, builderParams, beforeStoreResult.GetValue());
+                                }
+                                else
+                                {
+                                    AZ_Warning(AssetBuilderSDK::WarningWindow, false, "Failed preparing store result for %s", builderParams.m_processJobRequest.m_sourceFile.c_str());
+                                }
+
+                                if (!operationResult)
+                                {
+                                    AZ_TracePrintf(AssetProcessor::DebugChannel, "Unable to save job (%s, %s, %s) with fingerprint (%u) to the server.\n",
+                                        builderParams.m_rcJob->GetJobEntry().m_pathRelativeToWatchFolder.toUtf8().data(), builderParams.m_rcJob->GetJobKey().toUtf8().data(),
+                                        builderParams.m_rcJob->GetPlatformInfo().m_identifier.c_str(), builderParams.m_rcJob->GetOriginalFingerprint());
+                                }
+                            }
+                        }
+                        else
+                        {
+                            // running as client, check with the server whether it has already
+                            // processed this asset, if not or if the operation fails then process locally
+                            AssetProcessor::AssetServerBus::BroadcastResult(operationResult, &AssetProcessor::AssetServerBusTraits::RetrieveJobResult, builderParams);
+
+                            if (operationResult)
+                            {
+                                operationResult = AfterRetrievingJobResult(builderParams, jobLogTraceListener, result);
+                            }
+                            else
+                            {
+                                AZ_TracePrintf(AssetProcessor::DebugChannel, "Unable to get job (%s, %s, %s) with fingerprint (%u) from the server. Processing locally.\n",
+                                    builderParams.m_rcJob->GetJobEntry().m_pathRelativeToWatchFolder.toUtf8().data(), builderParams.m_rcJob->GetJobKey().toUtf8().data(),
+                                    builderParams.m_rcJob->GetPlatformInfo().m_identifier.c_str(), builderParams.m_rcJob->GetOriginalFingerprint());
+                            }
+
+                            runProcessJob = !operationResult;
+                        }
+                    }
+
+                    if(runProcessJob)
+                    {
+                        result.m_outputProducts.clear();
+                        // sending process job command to the builder
+                        builderParams.m_assetBuilderDesc.m_processJobFunction(builderParams.m_processJobRequest, result);
+                    }
+                }
+            }
+
+            if (JobCancelListener.IsCancelled())
+            {
+                result.m_resultCode = AssetBuilderSDK::ProcessJobResult_Cancelled;
+            }
+        }
+
+        bool shouldRemoveTempFolder = true;
+
+        if (result.m_resultCode == AssetBuilderSDK::ProcessJobResult_Success)
+        {
+            // do a final check of this job to make sure its not making colliding subIds.
+            AZStd::unordered_set<AZ::u32> subIdsFound;
+            for (const AssetBuilderSDK::JobProduct& product : result.m_outputProducts)
+            {
+                if (!subIdsFound.insert(product.m_productSubID).second)
+                {
+                    // if this happens the element was already in the set.
+                    AZ_Error(AssetBuilderSDK::ErrorWindow, false, "The builder created more than one asset with the same subID (%u) when emitting product %s\n  Builders should set a unique m_productSubID value for each product, as this is used as part of the address of the asset.", product.m_productSubID, product.m_productFileName.c_str());
+                    result.m_resultCode = AssetBuilderSDK::ProcessJobResult_Failed;
+                    break;
+                }
+            }
+        }
+
+        if(result.m_resultCode == AssetBuilderSDK::ProcessJobResult_Success)
+        {
+            bool handledDependencies = true; // True in case there are no outputs
+
+            for (const AssetBuilderSDK::JobProduct& jobProduct : result.m_outputProducts)
+            {
+                handledDependencies = false; // False by default since there are outputs
+
+                if(jobProduct.m_dependenciesHandled)
+                {
+                    handledDependencies = true;
+                    break;
+                }
+            }
+
+            if(!handledDependencies)
+            {
+                AZ_Warning(AssetBuilderSDK::WarningWindow, false, "The builder (%s) has not indicated it handled outputting product dependencies for file %s.  This is a programmer error.", builderParams.m_assetBuilderDesc.m_name.c_str(), builderParams.m_processJobRequest.m_sourceFile.c_str());
+                AZ_Warning(AssetBuilderSDK::WarningWindow, false, "For builders that output AZ serialized types, it is recommended to use AssetBuilderSDK::OutputObject which will handle outputting product depenedencies and creating the JobProduct.  This is fine to use even if your builder never has product dependencies.");
+                AZ_Warning(AssetBuilderSDK::WarningWindow, false, "For builders that need custom depenedency parsing that cannot be handled by AssetBuilderSDK::OutputObject or ones that output non-AZ serialized types, add the dependencies to m_dependencies and m_pathDependencies on the JobProduct and then set m_dependenciesHandled to true.");
+                jobLogTraceListener.AddWarning();
+            }
+
+            WarningLevel warningLevel = WarningLevel::Default;
+            JobDiagnosticRequestBus::BroadcastResult(warningLevel, &JobDiagnosticRequestBus::Events::GetWarningLevel);
+            const bool hasErrors = jobLogTraceListener.GetErrorCount() > 0;
+            const bool hasWarnings = jobLogTraceListener.GetWarningCount() > 0;
+
+            if(warningLevel == WarningLevel::FatalErrors && hasErrors)
+            {
+                AZ_Error(AssetBuilderSDK::ErrorWindow, false, "Failing job, fatal errors setting is enabled");
+                result.m_resultCode = AssetBuilderSDK::ProcessJobResult_Failed;
+            }
+            else if(warningLevel == WarningLevel::FatalErrorsAndWarnings && (hasErrors || hasWarnings))
+            {
+                AZ_Error(AssetBuilderSDK::ErrorWindow, false, "Failing job, fatal errors and warnings setting is enabled");
+                result.m_resultCode = AssetBuilderSDK::ProcessJobResult_Failed;
+            }
+        }
+
+        switch (result.m_resultCode)
+        {
+        case AssetBuilderSDK::ProcessJobResult_Success:
+            // make sure there's no subid collision inside a job.
+            {
+                
+                if (!CopyCompiledAssets(builderParams, result))
+                {
+                    result.m_resultCode = AssetBuilderSDK::ProcessJobResult_Failed;
+                    shouldRemoveTempFolder = false;
+                }
+                shouldRemoveTempFolder = shouldRemoveTempFolder && !s_createRequestFileForSuccessfulJob;
+            }
+            break;
+
+        case AssetBuilderSDK::ProcessJobResult_Crashed:
+            AZ_TracePrintf(AssetBuilderSDK::ErrorWindow, "Builder indicated that its process crashed!");
+            break;
+
+        case AssetBuilderSDK::ProcessJobResult_Cancelled:
+            AZ_TracePrintf(AssetBuilderSDK::ErrorWindow, "Builder indicates that the job was cancelled.");
+            break;
+
+        case AssetBuilderSDK::ProcessJobResult_Failed:
+            AZ_TracePrintf(AssetBuilderSDK::ErrorWindow, "Builder indicated that the job has failed.");
+            shouldRemoveTempFolder = false;
+            break;
+        }
+
+        if ((shouldRemoveTempFolder) || (listener.WasQuitRequested()))
+        {
+            QDir workingDir(QString(builderParams.m_processJobRequest.m_tempDirPath.c_str()));
+            workingDir.removeRecursively();
+        }
+
+        // Setting the job id back to zero for error detection
+        AssetProcessor::SetThreadLocalJobId(0);
+        listener.BusDisconnect();
+
+        JobDiagnosticRequestBus::Broadcast(&JobDiagnosticRequestBus::Events::RecordDiagnosticInfo, builderParams.m_rcJob->GetJobEntry().m_jobRunKey, JobDiagnosticInfo(aznumeric_cast<AZ::u32>(jobLogTraceListener.GetWarningCount()), aznumeric_cast<AZ::u32>(jobLogTraceListener.GetErrorCount())));
+    }
+
+    bool RCJob::CopyCompiledAssets(BuilderParams& params, AssetBuilderSDK::ProcessJobResponse& response)
+    {
+        if (response.m_outputProducts.empty())
+        {
+            // early out here for performance - no need to do anything at all here so don't waste time with IsDir or Exists or anything.
+            return true;
+        }
+
+        QDir outputDirectory(params.m_finalOutputDir);
+        QString         tempFolder = params.m_processJobRequest.m_tempDirPath.c_str();
+        QDir            tempDir(tempFolder);
+
+        if (params.m_finalOutputDir.isEmpty())
+        {
+            AZ_Assert(false, "CopyCompiledAssets:  params.m_finalOutputDir is empty for an asset processor job.  This should not happen and is because of a recent code change.  Check history of any new builders or rcjob.cpp\n");
+            return false;
+        }
+
+        if (!tempDir.exists())
+        {
+            AZ_Assert(false, "PCopyCompiledAssets:  params.m_processJobRequest.m_tempDirPath is empty for an asset processor job.  This should not happen and is because of a recent code change!  Check history of RCJob.cpp and any new builder code changes.\n");
+            return false;
+        }
+
+        // if outputDirectory does not exist then create it
+        unsigned int waitTimeInSecs = 3;
+        if (!AssetUtilities::CreateDirectoryWithTimeout(outputDirectory, waitTimeInSecs))
+        {
+            AZ_TracePrintf(AssetBuilderSDK::ErrorWindow, "Failed to create output directory: %s\n", outputDirectory.absolutePath().toUtf8().data());
+            return false;
+        }
+
+        // copy the built products into the appropriate location in the real cache and update the job status accordingly.
+        // note that we go to the trouble of first doing all the checking for disk space and existence of the source files
+        // before we notify the AP or start moving any of the files so that failures cause the least amount of damage possible.
+
+        // this vector is a set of pairs where the first of each pair is the source file (absolute) we intend to copy
+        // and  the second is the product destination we intend to copy it to.
+        QList< QPair<QString, QString> > outputsToCopy;
+        outputsToCopy.reserve(static_cast<int>(response.m_outputProducts.size()));
+        qint64 totalFileSizeRequired = 0;
+
+        for (AssetBuilderSDK::JobProduct& product : response.m_outputProducts)
+        {
+            // each Output Product communicated by the builder will either be
+            // * a relative path, which means we assume its relative to the temp folder, and we attempt to move the file
+            // * an absolute path in the temp folder, and we attempt to move also
+            // * an absolute path outside the temp folder, in which we assume you'd like to just copy a file somewhere.
+
+            QString outputProduct = QString::fromUtf8(product.m_productFileName.c_str()); // could be a relative path.
+            QFileInfo fileInfo(outputProduct);
+
+            if (fileInfo.isRelative())
+            {
+                // we assume that its relative to the TEMP folder.
+                fileInfo = QFileInfo(tempDir.absoluteFilePath(outputProduct));
+            }
+
+            QString absolutePathOfSource = fileInfo.absoluteFilePath();
+            QString outputFilename = fileInfo.fileName();
+            QString productFile = AssetUtilities::NormalizeFilePath(outputDirectory.filePath(outputFilename.toLower()));
+
+            // Don't make productFile all lowercase for case-insensitive as this
+            // breaks macOS. The case is already setup properly when the job
+            // was created.
+
+            if (productFile.length() >= AP_MAX_PATH_LEN)
+            {
+                AZ_Error(AssetBuilderSDK::ErrorWindow, false, "Cannot copy file: Product '%s' path length (%d) exceeds the max path length (%d) allowed on disk\n", productFile.toUtf8().data(), productFile.length(), AP_MAX_PATH_LEN);
+                return false;
+            }
+            
+            QFileInfo inFile(absolutePathOfSource);
+            if (!inFile.exists())
+            {
+                AZ_Error(AssetBuilderSDK::ErrorWindow, false, "Cannot copy file - product file with absolute path '%s' attempting to save into cache could not be found", absolutePathOfSource.toUtf8().constData());
+                return false;
+            }
+            
+            totalFileSizeRequired += inFile.size();
+            outputsToCopy.push_back(qMakePair(absolutePathOfSource, productFile));
+            
+            // also update the product file name to be the final resting place of this product in the cache (normalized!)
+            product.m_productFileName = AssetUtilities::NormalizeFilePath(productFile).toUtf8().constData();
+        }
+
+        // now we can check if there's enough space for ALL the files before we copy any.
+        bool hasSpace = false;
+        AssetProcessor::DiskSpaceInfoBus::BroadcastResult(hasSpace, &AssetProcessor::DiskSpaceInfoBusTraits::CheckSufficientDiskSpace, outputDirectory.absolutePath(), totalFileSizeRequired, false);
+
+        if (!hasSpace)
+        {
+            AZ_Error(AssetProcessor::ConsoleChannel, false, "Cannot save file to cache, not enough disk space to save all the products of %s.  Total needed: %lli bytes", params.m_processJobRequest.m_sourceFile.c_str(), totalFileSizeRequired);
+            return false;
+        }
+
+        // if we get here, we are good to go in terms of disk space and sources existing, so we make the best attempt we can.
+        // first, we broadcast the name of ALL of the outputs we are about to change:
+        for (const QPair<QString, QString>& filePair : outputsToCopy)
+        {
+            const QString& productAbsolutePath = filePair.second;
+            // note that this absolute path is a real file system path, and the following API requires normalized paths:
+            QString normalized = AssetUtilities::NormalizeFilePath(productAbsolutePath);
+            AssetProcessor::ProcessingJobInfoBus::Broadcast(&AssetProcessor::ProcessingJobInfoBus::Events::BeginCacheFileUpdate, normalized.toUtf8().constData());
+        }
+
+        // after we do the above notify its important that we do not early exit this function without undoing those locks.
+
+        bool anyFileFailed = false;
+
+        for (const QPair<QString, QString>& filePair : outputsToCopy)
+        {
+            const QString& sourceAbsolutePath = filePair.first;
+            const QString& productAbsolutePath = filePair.second;
+
+            bool isCopyJob = !(sourceAbsolutePath.startsWith(tempFolder, Qt::CaseInsensitive));
+
+            if (!MoveCopyFile(sourceAbsolutePath, productAbsolutePath, isCopyJob)) // this has its own traceprintf for failure
+            {
+                // MoveCopyFile will have output to the log.  No need to double output here.
+                anyFileFailed = true;
+                continue;
+            }
+            
+            //we now ensure that the file is writable - this is just a warning if it fails, not a complete failure.
+            if (!AssetUtilities::MakeFileWritable(productAbsolutePath))
+            {
+                AZ_TracePrintf(AssetBuilderSDK::WarningWindow, "Unable to change permission for the file: %s.\n", productAbsolutePath.toUtf8().data());
+            }
+        }
+
+        // once we're done, regardless of success or failure, we 'unlock' those files for further process.
+        // if we failed, also re-trigger them to rebuild (the bool param at the end of the ebus call)
+        for (const QPair<QString, QString>& filePair : outputsToCopy)
+        {
+            const QString& productAbsolutePath = filePair.second;
+            // note that this absolute path is a real file system path, and the following API requires normalized paths:
+            QString normalized = AssetUtilities::NormalizeFilePath(productAbsolutePath);
+            AssetProcessor::ProcessingJobInfoBus::Broadcast(&AssetProcessor::ProcessingJobInfoBus::Events::EndCacheFileUpdate, normalized.toUtf8().constData(), anyFileFailed);
+        }
+        
+        return !anyFileFailed;
+    }
+
+    AZ::Outcome<AZStd::vector<AZStd::string>> RCJob::BeforeStoringJobResult(const BuilderParams& builderParams, AssetBuilderSDK::ProcessJobResponse jobResponse)
+    {
+        AZStd::string normalizedTempFolderPath = builderParams.m_processJobRequest.m_tempDirPath;
+        AzFramework::StringFunc::Path::Normalize(normalizedTempFolderPath);
+
+        AZStd::vector<AZStd::string> sourceFiles;
+        for (AssetBuilderSDK::JobProduct& product : jobResponse.m_outputProducts)
+        {
+            // Try to handle Absolute paths within the temp folder
+            if (!AzFramework::StringFunc::Replace(product.m_productFileName, normalizedTempFolderPath.c_str(), s_tempString))
+            {
+                // From CopyCompiledAssets:
+
+                // each Output Product communicated by the builder will either be
+                // * a relative path, which means we assume its relative to the temp folder, and we attempt to move the file
+                // * an absolute path in the temp folder, and we attempt to move also
+                // * an absolute path outside the temp folder, in which we assume you'd like to just copy a file somewhere.
+
+                // We need to handle case 3 here (Case 2 was above, case 1 is treated as relative within temp)
+                // If the path was not absolute within the temp folder and not relative it should be an absolute path beneath our source (Including the source)
+                // meaning a copy job which needs to be added to our archive.  
+                if (!AzFramework::StringFunc::Path::IsRelative(product.m_productFileName.c_str()))
+                {
+                    AZStd::string sourceFile{ builderParams.m_rcJob->GetJobEntry().GetAbsoluteSourcePath().toUtf8().data() };
+                    AzFramework::StringFunc::Path::Normalize(sourceFile);
+                    AzFramework::StringFunc::Path::StripFullName(sourceFile);
+
+                    AzFramework::StringFunc::Path::Normalize(product.m_productFileName);
+                    size_t sourcePathPos = product.m_productFileName.find(sourceFile.c_str());
+                    if(sourcePathPos != AZStd::string::npos)
+                    {
+                        sourceFiles.push_back(product.m_productFileName.substr(sourceFile.size()).c_str());
+                        AzFramework::StringFunc::Path::Join(s_tempString, product.m_productFileName.substr(sourceFile.size()).c_str(), product.m_productFileName);
+                    }
+                    else
+                    {
+                        AZ_Warning(AssetBuilderSDK::WarningWindow, false, "Failed to find source path %s or temp path %s in non relative path in %s", sourceFile.c_str(), normalizedTempFolderPath.c_str(), product.m_productFileName.c_str());
+                    }
+                }
+            }
+        }
+        AZStd::string responseFilePath;
+        AzFramework::StringFunc::Path::ConstructFull(builderParams.m_processJobRequest.m_tempDirPath.c_str(), AssetBuilderSDK::s_processJobResponseFileName, responseFilePath, true);
+        //Save ProcessJobResponse to disk
+        if (!AZ::Utils::SaveObjectToFile(responseFilePath, AZ::DataStream::StreamType::ST_XML, &jobResponse))
+        {
+            return AZ::Failure();
+        }
+        AzToolsFramework::AssetSystem::JobInfo jobInfo;
+        AzToolsFramework::AssetSystem::AssetJobLogResponse jobLogResponse;
+        jobInfo.m_sourceFile = builderParams.m_rcJob->GetJobEntry().m_databaseSourceName.toUtf8().data();
+        jobInfo.m_platform = builderParams.m_rcJob->GetPlatformInfo().m_identifier.c_str();
+        jobInfo.m_jobKey = builderParams.m_rcJob->GetJobKey().toUtf8().data();
+        jobInfo.m_builderGuid = builderParams.m_rcJob->GetBuilderGuid();
+        jobInfo.m_jobRunKey = builderParams.m_rcJob->GetJobEntry().m_jobRunKey;
+        jobInfo.m_watchFolder = builderParams.m_processJobRequest.m_watchFolder;
+        AssetUtilities::ReadJobLog(jobInfo, jobLogResponse);
+
+        //Save joblog to disk
+        AZStd::string jobLogFilePath;
+        AzFramework::StringFunc::Path::ConstructFull(builderParams.m_processJobRequest.m_tempDirPath.c_str(), s_jobLogFileName, jobLogFilePath, true);
+        if (!AZ::Utils::SaveObjectToFile(jobLogFilePath, AZ::DataStream::StreamType::ST_XML, &jobLogResponse))
+        {
+            return AZ::Failure();
+        }
+        return AZ::Success(sourceFiles);
+    }
+
+    bool RCJob::AfterRetrievingJobResult(const BuilderParams& builderParams, AssetUtilities::JobLogTraceListener& jobLogTraceListener, AssetBuilderSDK::ProcessJobResponse& jobResponse)
+    {
+        AZStd::string responseFilePath;
+        AzFramework::StringFunc::Path::ConstructFull(builderParams.m_processJobRequest.m_tempDirPath.c_str(), AssetBuilderSDK::s_processJobResponseFileName, responseFilePath, true);
+        if (!AZ::Utils::LoadObjectFromFileInPlace(responseFilePath.c_str(), jobResponse))
+        {
+            return false;
+        }
+
+        //Ensure that ProcessJobResponse have the correct absolute paths 
+        for (AssetBuilderSDK::JobProduct& product : jobResponse.m_outputProducts)
+        {
+            AzFramework::StringFunc::Replace(product.m_productFileName, s_tempString, builderParams.m_processJobRequest.m_tempDirPath.c_str(), s_tempString);
+        }
+
+        AZStd::string jobLogFilePath;
+        AzFramework::StringFunc::Path::ConstructFull(builderParams.m_processJobRequest.m_tempDirPath.c_str(), s_jobLogFileName, jobLogFilePath, true);
+        AzToolsFramework::AssetSystem::AssetJobLogResponse jobLogResponse;
+
+        if (!AZ::Utils::LoadObjectFromFileInPlace(jobLogFilePath.c_str(), jobLogResponse))
+        {
+            return false;
+        }
+
+        if (!jobLogResponse.m_isSuccess)
+        {
+            AZ_TracePrintf(AssetProcessor::DebugChannel, "Job log request was unsuccessful for job (%s, %s, %s) from the server.\n",
+                builderParams.m_rcJob->GetJobEntry().m_pathRelativeToWatchFolder.toUtf8().data(), builderParams.m_rcJob->GetJobKey().toUtf8().data(),
+                builderParams.m_rcJob->GetPlatformInfo().m_identifier.c_str());
+
+            if(jobLogResponse.m_jobLog.find("No log file found") != AZStd::string::npos)
+            {
+                AZ_TracePrintf(AssetProcessor::DebugChannel, "Unable to find job log from the server. This could happen if you are trying to use the server cache with a copy job,\
+please check the assetprocessorplatformconfig.ini file and ensure that server cache is disabled for the job.\n");
+            }
+            
+            return false;
+        }
+
+        // writing server logs
+        AZ_TracePrintf(AssetProcessor::DebugChannel, "------------SERVER BEGIN----------\n");
+        AzToolsFramework::Logging::LogLine::ParseLog(jobLogResponse.m_jobLog.c_str(), jobLogResponse.m_jobLog.size(),
+            [&jobLogTraceListener](AzToolsFramework::Logging::LogLine& line)
+        {
+            jobLogTraceListener.AppendLog(line);
+        });
+        AZ_TracePrintf(AssetProcessor::DebugChannel, "------------SERVER END----------\n");
+        return true;
+    }
+
+    AZStd::string BuilderParams::GetTempJobDirectory() const
+    {
+        return m_processJobRequest.m_tempDirPath;
+    }
+
+    QString BuilderParams::GetServerKey() const
+    {
+        return m_serverKey;
+    }
+
+} // namespace AssetProcessor
+
+
+//////////////////////////////////////////////////////////////////////////
+

Code/Tools/AssetProcessor/native/resourcecompiler/rcjob.h

@@ -0,0 +1,238 @@
+/*
+* All or portions of this file Copyright (c) Amazon.com, Inc. or its affiliates or
+* its licensors.
+*
+* For complete copyright and license terms please see the LICENSE at the root of this
+* distribution (the "License"). All use of this software is governed by the License,
+* or, if provided, by the license below or the license accompanying this file. Do not
+* remove or modify any license notices. This file is distributed on an "AS IS" BASIS,
+* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+*
+*/
+#ifndef RCJOB_H
+#define RCJOB_H
+
+#if !defined(Q_MOC_RUN)
+#include <QObject>
+#include <QString>
+#include <QDateTime>
+#include <QStringList>
+#include <AzCore/base.h>
+#include "RCCommon.h"
+#include "native/utilities/PlatformConfiguration.h"
+#include <AzCore/Math/Uuid.h>
+#include <AssetBuilderSDK/AssetBuilderSDK.h>
+#include "native/assetprocessor.h"
+#include <AzToolsFramework/AssetDatabase/AssetDatabaseConnection.h>
+#include <QFileInfoList>
+#endif
+
+namespace AssetProcessor
+{
+    struct AssetRecognizer;
+    class RCJob;
+
+    //! Params Base class
+    struct Params
+    {
+        Params(AssetProcessor::RCJob* job = nullptr)
+            : m_rcJob(job)
+        {}
+        virtual ~Params() = default;
+
+        AssetProcessor::RCJob* m_rcJob;
+        QString m_finalOutputDir;
+
+        Params(const Params&) = default;
+
+        virtual bool IsValidParams() const;
+    };
+
+    //! RCParams contains info that is required by the rc
+    struct RCParams
+        : public Params
+    {
+        QString m_rootDir;
+        QString m_rcExe;
+        QString m_inputFile;
+        QString m_platformIdentifier;
+        QString m_params;
+
+        RCParams(AssetProcessor::RCJob* job = nullptr)
+            : Params(job)
+        {}
+
+        RCParams(const RCParams&) = default;
+
+        bool IsValidParams() const override;
+    };
+
+    //! BuilderParams contains info that is required by the builders
+    struct BuilderParams
+        : public Params
+    {
+        AssetBuilderSDK::ProcessJobRequest m_processJobRequest;
+        AssetBuilderSDK::AssetBuilderDesc  m_assetBuilderDesc;
+        QString m_serverKey;
+
+        BuilderParams(AssetProcessor::RCJob* job = nullptr)
+            : Params(job)
+        {}
+
+        BuilderParams(const BuilderParams&) = default;
+
+        AZStd::string GetTempJobDirectory() const;
+        QString GetServerKey() const;
+    };
+
+    //! JobOutputInfo is used to store job related messages.
+    //! Messages can be an error or just some information.
+    struct JobOutputInfo
+    {
+        QString m_windowName; // window name is used to specify whether it is an error or not
+        QString m_message;// the actual message
+
+        JobOutputInfo() = default;
+
+        JobOutputInfo(QString window, QString message)
+            : m_windowName(window)
+            , m_message(message)
+        {
+        }
+    };
+
+
+    /**
+    * The RCJob class contains all the necessary information about a single RC job
+    */
+    class RCJob
+        : public QObject
+    {
+        Q_OBJECT
+
+    public:
+        enum JobState
+        {
+            pending,
+            processing,
+            completed,
+            crashed,
+            terminated,
+            cancelled,
+            failed,
+        };
+
+        explicit RCJob(QObject* parent = 0);
+
+        virtual ~RCJob();
+
+        void Init(JobDetails& details);
+
+        QString Params() const;
+        QString CommandLine() const;
+
+        QDateTime GetTimeCreated() const;
+        void SetTimeCreated(const QDateTime& timeCreated);
+
+        QDateTime GetTimeLaunched() const;
+        void SetTimeLaunched(const QDateTime& timeLaunched);
+
+        QDateTime GetTimeCompleted() const;
+        void SetTimeCompleted(const QDateTime& timeCompleted);
+
+        void SetOriginalFingerprint(AZ::u32 originalFingerprint);
+        AZ::u32 GetOriginalFingerprint() const;
+
+        QString GetConsoleOutput() const;
+        void SetConsoleOutput(QString rcOut);
+
+        JobState GetState() const;
+        void SetState(const JobState& state);
+
+        const AZ::Uuid& GetInputFileUuid() const;
+
+        QString GetDestination() const;
+
+        //! the final output path is where the actual outputs are copied when processing succeeds
+        //! this will be in the asset cache, in the gamename / platform / gamename folder.
+        QString GetFinalOutputPath() const;
+
+        const AssetProcessor::AssetRecognizer* GetRecognizer() const;
+        void SetRecognizer(const AssetProcessor::AssetRecognizer* value);
+
+        const AssetBuilderSDK::PlatformInfo& GetPlatformInfo() const;
+
+        // intentionally non-const to move.
+        AssetBuilderSDK::ProcessJobResponse& GetProcessJobResponse();
+
+        const JobEntry& GetJobEntry() const;
+
+        void Start();
+
+        const QueueElementID& GetElementID() const { return m_queueElementID; }
+
+        const int JobEscalation() { return m_JobEscalation; }
+
+        void SetJobEscalation(int jobEscalation);
+
+        void SetCheckExclusiveLock(bool value);
+
+    Q_SIGNALS:
+        //! This signal will be emitted when we make sure that no other application has a lock on the source file 
+        //! and also that the fingerprint of the source file is stable and not changing.
+        //! This will basically indicate that we are starting to perform work on the current job
+        void BeginWork();
+        void Finished();
+        void JobFinished(AssetBuilderSDK::ProcessJobResponse result);
+
+    public:
+        static QString GetStateDescription(const JobState& state);
+        static void ExecuteBuilderCommand(BuilderParams builderParams);
+        static void AutoFailJob(BuilderParams& builderParams);
+        static bool CopyCompiledAssets(BuilderParams& params, AssetBuilderSDK::ProcessJobResponse& response);
+        //! This method will save the processJobResponse and the job log to the temp directory as xml files.
+        //! We will be modifying absolute paths in processJobResponse before saving it to the disk.
+        static AZ::Outcome<AZStd::vector<AZStd::string>> BeforeStoringJobResult(const BuilderParams& builderParams, AssetBuilderSDK::ProcessJobResponse jobResponse);
+        //! This method will retrieve the processJobResponse and the job log from the temp directory.
+        //! This method is also responsible for emitting the server job logs to the local job log file.
+        static bool AfterRetrievingJobResult(const BuilderParams& builderParams, AssetUtilities::JobLogTraceListener& jobLogTraceListener, AssetBuilderSDK::ProcessJobResponse& jobResponse);
+
+        QString GetJobKey() const;
+        AZ::Uuid GetBuilderGuid() const;
+        bool IsCritical() const;
+        bool IsAutoFail() const;
+        int GetPriority() const;
+        const AZStd::vector<JobDependencyInternal>& GetJobDependencies();
+
+    protected:
+        //! DoWork ensure that the job is ready for being processing and than makes the actual builder call   
+        virtual void DoWork(AssetBuilderSDK::ProcessJobResponse& result, BuilderParams& builderParams, AssetUtilities::QuitListener& listener);
+        void PopulateProcessJobRequest(AssetBuilderSDK::ProcessJobRequest& processJobRequest);
+
+    private:
+        JobDetails m_jobDetails;
+        JobState m_jobState;
+
+        QueueElementID m_queueElementID; // cached to prevent lots of construction of this all over the place
+
+        int m_JobEscalation = AssetProcessor::JobEscalation::Default; // Escalation indicates how important the job is and how soon it needs processing, the greater the number the greater the escalation  
+
+        QDateTime m_timeCreated;
+        QDateTime m_timeLaunched;
+        QDateTime m_timeCompleted;
+
+        unsigned int m_exitCode = 0;
+
+        AzToolsFramework::AssetDatabase::ProductDatabaseEntryContainer m_products;
+
+        AssetBuilderSDK::ProcessJobResponse m_processJobResponse;
+
+        AZ::u32 m_scanFolderID;
+    };
+} // namespace AssetProcessor
+
+Q_DECLARE_METATYPE(AssetProcessor::BuilderParams);
+Q_DECLARE_METATYPE(AssetProcessor::JobOutputInfo);
+Q_DECLARE_METATYPE(AssetProcessor::RCParams);
+
+#endif // RCJOB_H

Code/Tools/AssetProcessor/native/resourcecompiler/rcjoblistmodel.cpp

@@ -0,0 +1,543 @@
+/*
+* All or portions of this file Copyright (c) Amazon.com, Inc. or its affiliates or
+* its licensors.
+*
+* For complete copyright and license terms please see the LICENSE at the root of this
+* distribution (the "License"). All use of this software is governed by the License,
+* or, if provided, by the license below or the license accompanying this file. Do not
+* remove or modify any license notices. This file is distributed on an "AS IS" BASIS,
+* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+*
+*/
+#include "rcjoblistmodel.h"
+
+
+// uncomment this in order to add additional verbose log output for this class.
+// can drastically slow down function since this class is a hotspot
+// #define DEBUG_RCJOB_MODEL
+
+namespace AssetProcessor
+{
+    RCJobListModel::RCJobListModel(QObject* parent)
+        : QAbstractItemModel(parent)
+    {
+
+    }
+
+    int RCJobListModel::rowCount(const QModelIndex& parent) const
+    {
+        if (parent.isValid())
+        {
+            return 0;
+        }
+        return itemCount();
+    }
+
+    QModelIndex RCJobListModel::parent(const QModelIndex& /*index*/) const
+    {
+        return QModelIndex();
+    }
+
+    QModelIndex RCJobListModel::index(int row, int column, const QModelIndex& parent) const
+    {
+        if (row >= rowCount(parent) || column >= columnCount(parent))
+        {
+            return QModelIndex();
+        }
+        return createIndex(row, column);
+    }
+
+    int RCJobListModel::columnCount(const QModelIndex& parent) const
+    {
+        return parent.isValid() ? 0 : Column::Max;
+    }
+
+    QVariant RCJobListModel::headerData(int section, Qt::Orientation orientation, int role) const
+    {
+        if (orientation == Qt::Horizontal && role == Qt::DisplayRole)
+        {
+            switch (section)
+            {
+            case ColumnState:
+                return tr("State");
+            case ColumnJobId:
+                return tr("Job Id");
+            case ColumnCommand:
+                return tr("Asset");
+            case ColumnCompleted:
+                return tr("Completed");
+            case ColumnPlatform:
+                return tr("Platform");
+            default:
+                break;
+            }
+        }
+
+        return QAbstractItemModel::headerData(section, orientation, role);
+    }
+
+    unsigned int RCJobListModel::jobsInFlight() const
+    {
+        return m_jobsInFlight.size();
+    }
+
+
+    void RCJobListModel::UpdateJobEscalation(AssetProcessor::RCJob* rcJob, int jobEscalation)
+    {
+        for (int idx = 0; idx < rowCount(); ++idx)
+        {
+            RCJob* job = getItem(idx);
+            if (job == rcJob)
+            {
+                rcJob->SetJobEscalation(jobEscalation);
+                Q_EMIT dataChanged(index(idx, 0), index(idx, columnCount() - 1));
+                break;
+            }
+        }
+    }
+
+    void RCJobListModel::UpdateRow(int jobIndex)
+    {
+        Q_EMIT dataChanged(index(jobIndex, 0), index(jobIndex, columnCount() - 1));
+    }
+
+    QVariant RCJobListModel::data(const QModelIndex& index, int role) const
+    {
+        if (!index.isValid())
+        {
+            return QVariant();
+        }
+
+        if (index.row() >= itemCount())
+        {
+            return QVariant();
+        }
+
+        switch (role)
+        {
+        case jobIndexRole:
+            return getItem(index.row())->GetJobEntry().m_jobRunKey;
+        case stateRole:
+            return RCJob::GetStateDescription(getItem(index.row())->GetState());
+        case displayNameRole:
+            return getItem(index.row())->GetJobEntry().m_pathRelativeToWatchFolder;
+        case timeCreatedRole:
+            return getItem(index.row())->GetTimeCreated().toString("hh:mm:ss.zzz");
+        case timeLaunchedRole:
+            return getItem(index.row())->GetTimeLaunched().toString("hh:mm:ss.zzz");
+        case timeCompletedRole:
+            return getItem(index.row())->GetTimeCompleted().toString("hh:mm:ss.zzz");
+
+        case Qt::DisplayRole:
+            switch (index.column())
+            {
+            case ColumnJobId:
+                return getItem(index.row())->GetJobEntry().m_jobRunKey;
+            case ColumnState:
+                return RCJob::GetStateDescription(getItem(index.row())->GetState());
+            case ColumnCommand:
+                return getItem(index.row())->GetJobEntry().m_pathRelativeToWatchFolder;
+            case ColumnCompleted:
+                return getItem(index.row())->GetTimeCompleted().toString("hh:mm:ss.zzz");
+            case ColumnPlatform:
+                return QString::fromUtf8(getItem(index.row())->GetPlatformInfo().m_identifier.c_str());
+            default:
+                break;
+            }
+        default:
+            break;
+        }
+
+        return QVariant();
+    }
+
+    int RCJobListModel::itemCount() const
+    {
+        return aznumeric_caster(m_jobs.size());
+    }
+
+    RCJob* RCJobListModel::getItem(int index) const
+    {
+        if (index >= 0 && index < m_jobs.size())
+        {
+            return m_jobs[index];
+        }
+
+        return nullptr; //invalid index
+    }
+
+    bool RCJobListModel::isEmpty()
+    {
+        return m_jobs.empty();
+    }
+
+    void RCJobListModel::addNewJob(RCJob* rcJob)
+    {
+        int posForInsert = aznumeric_caster(m_jobs.size());
+        beginInsertRows(QModelIndex(), posForInsert, posForInsert);
+
+        m_jobs.push_back(rcJob);
+#if defined(DEBUG_RCJOB_MODEL)
+        AZ_TracePrintf(AssetProcessor::DebugChannel, "JobTrace AddNewJob(%i  %s,%s,%s)\n", rcJob, rcJob->GetInputFileAbsolutePath().toUtf8().constData(), rcJob->GetPlatformInfo().m_identifier.c_str(), rcJob->GetJobKey().toUtf8().constData());
+#endif
+
+        bool isPending = false;
+        if (rcJob->GetState() == RCJob::pending)
+        {
+            m_jobsInQueueLookup.insert(rcJob->GetElementID(), rcJob);
+            isPending = true;
+        }
+        endInsertRows();
+    }
+
+    void RCJobListModel::markAsProcessing(RCJob* rcJob)
+    {
+#if defined(DEBUG_RCJOB_MODEL)
+        AZ_TracePrintf(AssetProcessor::DebugChannel, "JobTrace markAsProcessing(%i %s,%s,%s)\n", rcJob, rcJob->GetJobEntry().m_databaseSourceName.toUtf8().constData(), rcJob->GetPlatformInfo().m_identifier.c_str(), rcJob->GetJobKey().toUtf8().constData());
+#endif
+
+        rcJob->SetState(RCJob::processing);
+        rcJob->SetTimeLaunched(QDateTime::currentDateTime());
+
+        m_jobsInFlight.insert(rcJob);
+
+        for(size_t jobIndex = m_jobs.size() - 1; jobIndex >= 0; --jobIndex)
+        {
+            if(m_jobs[jobIndex] == rcJob)
+            {
+                Q_EMIT dataChanged(index(aznumeric_caster(jobIndex), 0, QModelIndex()), index(aznumeric_caster(jobIndex), 0, QModelIndex()));
+                return;
+            }
+        }
+        
+        AZ_TracePrintf(AssetProcessor::DebugChannel, "JobTrace jobIndex == -1!!! (%i %s,%s,%s)\n",
+            rcJob, rcJob->GetJobEntry().m_databaseSourceName.toUtf8().constData(),
+            rcJob->GetPlatformInfo().m_identifier.c_str(),
+            rcJob->GetJobKey().toUtf8().constData());
+        AZ_Assert(false, "Job not found!!!");
+    }
+
+    void RCJobListModel::markAsStarted(RCJob* rcJob)
+    {
+#if defined(DEBUG_RCJOB_MODEL)
+        AZ_TracePrintf(AssetProcessor::DebugChannel, "JobTrace markAsStarted(%i %s,%s,%s)\n", rcJob, rcJob->GetInputFileAbsolutePath().toUtf8().constData(), rcJob->GetPlatformInfo().m_identifier.c_str(), rcJob->GetJobKey().toUtf8().constData());
+#endif
+
+        auto foundInQueue = m_jobsInQueueLookup.find(rcJob->GetElementID());
+        while ((foundInQueue != m_jobsInQueueLookup.end()) && (foundInQueue.value() == rcJob))
+        {
+            foundInQueue = m_jobsInQueueLookup.erase(foundInQueue);
+        }
+    }
+
+    void RCJobListModel::markAsCompleted(RCJob* rcJob)
+    {
+#if defined(DEBUG_RCJOB_MODEL)
+        AZ_TracePrintf(AssetProcessor::DebugChannel, "JobTrace markAsCompleted(%i %s,%s,%s)\n", rcJob, rcJob->GetInputFileAbsolutePath().toUtf8().constData(), rcJob->GetPlatformInfo().m_identifier.c_str(), rcJob->GetJobKey().toUtf8().constData());
+#endif
+
+        rcJob->SetTimeCompleted(QDateTime::currentDateTime());
+
+        auto foundInQueue = m_jobsInQueueLookup.find(rcJob->GetElementID());
+        while ((foundInQueue != m_jobsInQueueLookup.end()) && (foundInQueue.value() == rcJob))
+        {
+            foundInQueue = m_jobsInQueueLookup.erase(foundInQueue);
+        }
+
+        for (size_t jobIndex = m_jobs.size() - 1; jobIndex >= 0; --jobIndex)
+        {
+            if(m_jobs[jobIndex] == rcJob)
+            {
+                m_jobsInFlight.remove(rcJob);
+
+                // remove it from the list and delete it - there is a separate model that keeps track for the GUI so no need to keep jobs around.
+                {
+#if defined(DEBUG_RCJOB_MODEL)
+                    AZ_TracePrintf(AssetProcessor::DebugChannel, "JobTrace =>JobCompleted(%i %s,%s,%s)\n", rcJob, rcJob->GetJobEntry().m_databaseSourceName.toUtf8().constData(), rcJob->GetPlatformInfo().m_identifier.c_str(), rcJob->GetJobKey().toUtf8().constData());
+#endif
+                    beginRemoveRows(QModelIndex(), aznumeric_caster(jobIndex), aznumeric_caster(jobIndex));
+                    m_jobs.erase(m_jobs.begin() + jobIndex);
+                    endRemoveRows();
+
+                    // Only completed jobs need to wait on a catalog write
+                    if (rcJob->GetState() == RCJob::completed)
+                    {
+                        const auto& id = rcJob->GetElementID();
+
+                        auto itr = m_finishedJobsNotInCatalog.find(id);
+                        if (itr != m_finishedJobsNotInCatalog.end())
+                        {
+                            itr.value()++;
+                        }
+                        else
+                        {
+                            m_finishedJobsNotInCatalog.insert(id, 1);
+                        }
+                    }
+
+                    rcJob->deleteLater();
+                }
+
+                return;
+            }
+        }
+
+        AZ_TracePrintf(AssetProcessor::DebugChannel, "JobTrace jobIndex == -1!!! (%i %s,%s,%s)\n", rcJob, rcJob->GetJobEntry().m_databaseSourceName.toUtf8().constData(), rcJob->GetPlatformInfo().m_identifier.c_str(), rcJob->GetJobKey().toUtf8().constData());
+        AZ_Assert(false, "Job not found!!!");
+    }
+
+    void RCJobListModel::markAsCataloged(const AssetProcessor::QueueElementID& check)
+    {
+        auto itr = m_finishedJobsNotInCatalog.find(check);
+
+        if(itr == m_finishedJobsNotInCatalog.end())
+        {
+            AZ_Assert(false, "Attempting to mark a job as written to the catalog before the job has been put in the waiting queue! %s", check.GetInputAssetName().toUtf8().constData());
+            return;
+        }
+
+        itr.value()--;
+
+        if (itr.value() == 0)
+        {
+            m_finishedJobsNotInCatalog.erase(itr);
+        }
+    }
+
+    bool RCJobListModel::isInFlight(const AssetProcessor::QueueElementID& check) const
+    {
+        for (auto rcJob : m_jobsInFlight)
+        {
+            if (check == rcJob->GetElementID())
+            {
+                return true;
+            }
+        }
+        return false;
+    }
+
+    int RCJobListModel::GetIndexOfProcessingJob(const QueueElementID& elementId)
+    {
+        for (int idx = 0; idx < rowCount(); ++idx)
+        {
+            RCJob* job = getItem(idx);
+            if (job->GetState() == RCJob::processing && job->GetElementID() == elementId)
+            {
+                return idx;
+                break;
+            }
+        }
+
+        return -1; // invalid index
+    }
+
+    void RCJobListModel::EraseJobs(QString sourceFileDatabaseName, AZStd::vector<RCJob*>& pendingJobs)
+    {
+        for (int jobIdx = 0; jobIdx < rowCount(); ++jobIdx)
+        {
+            RCJob* job = getItem(jobIdx);
+            if (QString::compare(job->GetJobEntry().m_databaseSourceName, sourceFileDatabaseName, Qt::CaseInsensitive) == 0)
+            {
+                const QueueElementID& target = job->GetElementID();
+                if ((isInQueue(target)) || (isInFlight(target)))
+                {
+                    // Its important that this still follows the 'cancelled' flow, so that other parts of the code can update their "in progress" and other maps.
+                    AZ_TracePrintf(AssetProcessor::DebugChannel, "Cancelling Job [%s, %s, %s] because the source file no longer exists.\n", target.GetInputAssetName().toUtf8().data(), target.GetPlatform().toUtf8().data(), target.GetJobDescriptor().toUtf8().data());
+
+                    // if a job is pending, it was never started and thus will never enter Finished state, 
+                    // so simply changing its state to cancelled is not enough, collect them and return to rccontroller to process manually
+                    if (job->GetState() == RCJob::JobState::pending)
+                    {
+                        pendingJobs.push_back(job);
+                    }
+
+                    job->SetState(RCJob::JobState::cancelled);
+                    AssetBuilderSDK::JobCommandBus::Event(job->GetJobEntry().m_jobRunKey, &AssetBuilderSDK::JobCommandBus::Events::Cancel);
+                    UpdateRow(jobIdx);
+                }
+            }
+        }
+
+    }
+
+    bool RCJobListModel::isInQueue(const AssetProcessor::QueueElementID& check) const
+    {
+        return m_jobsInQueueLookup.contains(check);
+    }
+
+    bool RCJobListModel::isWaitingOnCatalog(const QueueElementID& check) const
+    {
+        return m_finishedJobsNotInCatalog.contains(check);
+    }
+
+    void RCJobListModel::PerformHeuristicSearch(QString searchTerm, QString platform, QSet<QueueElementID>& found, AssetProcessor::JobIdEscalationList& escalationList, bool isStatusRequest, int searchRules)
+    {
+        int escalationValue = 0;
+        if (isStatusRequest)
+        {
+            escalationValue = AssetProcessor::JobEscalation::ProcessAssetRequestStatusEscalation;
+        }
+        else
+        {
+            escalationValue = AssetProcessor::JobEscalation::ProcessAssetRequestSyncEscalation;
+        }
+
+        // try to narrowly exact-match the search term in case the search term refers to a specific actual source file:
+        for (const RCJob* rcJob : m_jobs)
+        {
+            if ((platform != rcJob->GetPlatformInfo().m_identifier.c_str()) || (rcJob->GetState() != RCJob::pending))
+            {
+                continue;
+            }
+            QString input = rcJob->GetJobEntry().m_pathRelativeToWatchFolder;
+            if (input.endsWith(searchTerm, Qt::CaseInsensitive))
+            {
+                AZ_TracePrintf(AssetProcessor::DebugChannel, "Job Queue: Heuristic search found exact match (%s,%s,%s).\n", rcJob->GetJobEntry().m_databaseSourceName.toUtf8().constData(), rcJob->GetPlatformInfo().m_identifier.c_str(), rcJob->GetJobKey().toUtf8().constData());
+                found.insert(QueueElementID(rcJob->GetJobEntry().m_databaseSourceName, platform, rcJob->GetJobKey()));
+                escalationList.append(qMakePair(rcJob->GetJobEntry().m_jobRunKey, escalationValue));
+            }
+        }
+
+        for (const RCJob* rcJob : m_jobsInFlight)
+        {
+            if (platform != rcJob->GetPlatformInfo().m_identifier.c_str())
+            {
+                continue;
+            }
+            QString input = rcJob->GetJobEntry().m_pathRelativeToWatchFolder;
+            if (input.endsWith(searchTerm, Qt::CaseInsensitive))
+            {
+                AZ_TracePrintf(AssetProcessor::DebugChannel, "Job Queue: Heuristic search found exact match (%s,%s,%s).\n", rcJob->GetJobEntry().m_databaseSourceName.toUtf8().constData(), rcJob->GetPlatformInfo().m_identifier.c_str(), rcJob->GetJobKey().toUtf8().constData());
+                found.insert(QueueElementID(rcJob->GetJobEntry().m_databaseSourceName, platform, rcJob->GetJobKey()));
+            }
+        }
+
+        if (!found.isEmpty() || searchRules == AzFramework::AssetSystem::RequestAssetStatus::SearchType::Exact)
+        {
+            return;
+        }
+
+        // broaden the heuristic.  Try without extensions - that is, ignore everything after the dot.
+        // if there are dashes, ignore them also.  This is how you match "blah.dds" to actually mean "blah.tif"
+        // since we have no idea what products will be generated by a source still in the queue until it runs
+        int dotIndex = searchTerm.lastIndexOf('.');
+        QStringRef searchTermWithNoExtension = searchTerm.midRef(0, dotIndex);
+
+        if (dotIndex != -1)
+        {
+            for (const auto& rcJob : m_jobs)
+            {
+                if ((platform != rcJob->GetPlatformInfo().m_identifier.c_str()) || (rcJob->GetState() != RCJob::pending))
+                {
+                    continue;
+                }
+                QString input = rcJob->GetJobEntry().m_pathRelativeToWatchFolder;
+                dotIndex = input.lastIndexOf('.');
+                if (dotIndex != -1)
+                {
+                    QStringRef testref = input.midRef(0, dotIndex);
+                    if (testref.endsWith(searchTermWithNoExtension, Qt::CaseInsensitive))
+                    {
+                        AZ_TracePrintf(AssetProcessor::DebugChannel, "Job Queue: Heuristic search found broad match (%s,%s,%s).\n", rcJob->GetJobEntry().m_pathRelativeToWatchFolder.toUtf8().constData(), rcJob->GetPlatformInfo().m_identifier.c_str(), rcJob->GetJobKey().toUtf8().constData());
+                        found.insert(QueueElementID(rcJob->GetJobEntry().m_databaseSourceName, platform, rcJob->GetJobKey()));
+                        escalationList.append(qMakePair(rcJob->GetJobEntry().m_jobRunKey, escalationValue));
+                    }
+                }
+            }
+
+            for (const RCJob* rcJob : m_jobsInFlight)
+            {
+                if (platform != rcJob->GetPlatformInfo().m_identifier.c_str())
+                {
+                    continue;
+                }
+                QString input = rcJob->GetJobEntry().m_pathRelativeToWatchFolder;
+                dotIndex = input.lastIndexOf('.');
+                if (dotIndex != -1)
+                {
+                    QStringRef testref = input.midRef(0, dotIndex);
+                    if (testref.endsWith(searchTermWithNoExtension, Qt::CaseInsensitive))
+                    {
+                        AZ_TracePrintf(AssetProcessor::DebugChannel, "Job Queue: Heuristic search found broad match (%s,%s,%s).\n", rcJob->GetJobEntry().m_databaseSourceName.toUtf8().constData(), rcJob->GetPlatformInfo().m_identifier.c_str(), rcJob->GetJobKey().toUtf8().constData());
+                        found.insert(QueueElementID(rcJob->GetJobEntry().m_databaseSourceName, platform, rcJob->GetJobKey()));
+                    }
+                }
+            }
+        }
+
+        if (!found.isEmpty())
+        {
+            return;
+        }
+
+        // broaden the heuristic further.  Eliminate anything after the last underscore in the file name
+        // (so blahblah_diff.dds just becomes blahblah) and then allow anything which has that file somewhere in it.
+
+        int slashIndex = searchTerm.lastIndexOf('/');
+        int dashIndex = searchTerm.lastIndexOf('_');
+        QStringRef searchTermWithNoSuffix = searchTermWithNoExtension;
+        if ((dashIndex != -1) && (slashIndex == -1) || (dashIndex > slashIndex))
+        {
+            searchTermWithNoSuffix = searchTermWithNoSuffix.mid(0, dashIndex);
+        }
+
+        for (const auto& rcJob : m_jobs)
+        {
+            if ((platform != rcJob->GetPlatformInfo().m_identifier.c_str()) || (rcJob->GetState() != RCJob::pending))
+            {
+                continue;
+            }
+            QString input = rcJob->GetJobEntry().m_pathRelativeToWatchFolder;
+            if (input.contains(searchTermWithNoSuffix, Qt::CaseInsensitive)) //notice here that we use simply CONTAINS instead of endswith - this can potentially be very broad!
+            {
+                AZ_TracePrintf(AssetProcessor::DebugChannel, "Job Queue: Heuristic search found ultra-broad match (%s,%s,%s).\n", rcJob->GetJobEntry().m_databaseSourceName.toUtf8().constData(), rcJob->GetPlatformInfo().m_identifier.c_str(), rcJob->GetJobKey().toUtf8().constData());
+                found.insert(QueueElementID(rcJob->GetJobEntry().m_databaseSourceName, platform, rcJob->GetJobKey()));
+                escalationList.append(qMakePair(rcJob->GetJobEntry().m_jobRunKey, escalationValue));
+            }
+        }
+
+        for (const RCJob* rcJob : m_jobsInFlight)
+        {
+            if (platform != rcJob->GetPlatformInfo().m_identifier.c_str())
+            {
+                continue;
+            }
+            
+            QString input = rcJob->GetJobEntry().m_pathRelativeToWatchFolder;
+            if (input.contains(searchTermWithNoSuffix, Qt::CaseInsensitive)) //notice here that we use simply CONTAINS instead of endswith - this can potentially be very broad!
+            {
+                AZ_TracePrintf(AssetProcessor::DebugChannel, "Job Queue: Heuristic search found ultra-broad match (%s,%s,%s).\n", rcJob->GetJobEntry().m_databaseSourceName.toUtf8().constData(), rcJob->GetPlatformInfo().m_identifier.c_str(), rcJob->GetJobKey().toUtf8().constData());
+                found.insert(QueueElementID(rcJob->GetJobEntry().m_databaseSourceName, platform, rcJob->GetJobKey()));
+            }
+        }
+    }
+
+
+    void RCJobListModel::PerformUUIDSearch(AZ::Uuid searchUuid, QString platform, QSet<QueueElementID>& found, AssetProcessor::JobIdEscalationList& escalationList, bool isStatusRequest)
+    {
+        int escalationValue = 0;
+        if (isStatusRequest)
+        {
+            escalationValue = AssetProcessor::JobEscalation::ProcessAssetRequestStatusEscalation;
+        }
+        else
+        {
+            escalationValue = AssetProcessor::JobEscalation::ProcessAssetRequestSyncEscalation;
+        }
+
+        for (const RCJob* rcJob : m_jobs)
+        {
+            if ((platform != rcJob->GetPlatformInfo().m_identifier.c_str()) || (rcJob->GetState() != RCJob::pending))
+            {
+                continue;
+            }
+
+            if (rcJob->GetJobEntry().m_sourceFileUUID == searchUuid)
+            {
+                found.insert(QueueElementID(rcJob->GetJobEntry().m_databaseSourceName, platform, rcJob->GetJobKey()));
+                escalationList.append(qMakePair(rcJob->GetJobEntry().m_jobRunKey, escalationValue));
+            }
+        }
+    }
+}// namespace AssetProcessor
+

Code/Tools/AssetProcessor/native/resourcecompiler/rcjoblistmodel.h

@@ -0,0 +1,118 @@
+/*
+* All or portions of this file Copyright (c) Amazon.com, Inc. or its affiliates or
+* its licensors.
+*
+* For complete copyright and license terms please see the LICENSE at the root of this
+* distribution (the "License"). All use of this software is governed by the License,
+* or, if provided, by the license below or the license accompanying this file. Do not
+* remove or modify any license notices. This file is distributed on an "AS IS" BASIS,
+* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+*
+*/
+#ifndef RCQUEUEMODEL_H
+#define RCQUEUEMODEL_H
+
+#if !defined(Q_MOC_RUN)
+#include "RCCommon.h"
+
+#include <QAbstractItemModel>
+#include <QObject>
+#include <QQueue>
+#include <QMultiMap>
+
+#include "rcjob.h"
+#endif
+
+namespace AssetProcessor
+{
+    class QueueElementID;
+}
+
+class RCcontrollerUnitTests;
+
+namespace AssetProcessor
+{
+    /**
+     * The RCJobListModel class contains lists of RC jobs
+     */
+    class RCJobListModel
+        : public QAbstractItemModel
+    {
+        friend class ::RCcontrollerUnitTests;
+        Q_OBJECT
+
+    public:
+        enum DataRoles
+        {
+            jobIndexRole = Qt::UserRole + 1,
+            stateRole,
+            displayNameRole,
+            timeCreatedRole,
+            timeLaunchedRole,
+            timeCompletedRole,
+            jobDataRole,
+        };
+
+        enum Column
+        {
+            ColumnState,
+            ColumnJobId,
+            ColumnCommand,
+            ColumnCompleted,
+            ColumnPlatform,
+            Max
+        };
+
+        explicit RCJobListModel(QObject* parent = 0);
+
+
+        QModelIndex parent(const QModelIndex&) const override;
+        QModelIndex index(int row, int column, const QModelIndex& parent = QModelIndex()) const override;
+        int columnCount(const QModelIndex& parent = QModelIndex()) const override;
+        int rowCount(const QModelIndex& parent = QModelIndex()) const override;
+        QVariant headerData(int section, Qt::Orientation orientation, int role) const override;
+        QVariant data(const QModelIndex& index, int role) const;
+
+        void markAsProcessing(RCJob* rcJob);
+        void markAsStarted(RCJob* rcJob);
+        void markAsCompleted(RCJob* rcJob);
+        void markAsCataloged(const AssetProcessor::QueueElementID& check);
+        unsigned int jobsInFlight() const;
+
+        void UpdateJobEscalation(AssetProcessor::RCJob* rcJob, int jobPrioririty);
+        void UpdateRow(int jobIndex);
+
+        bool isEmpty();
+        void addNewJob(RCJob* rcJob);
+
+        bool isInFlight(const QueueElementID& check) const;
+        bool isInQueue(const QueueElementID& check) const;
+        bool isWaitingOnCatalog(const QueueElementID& check) const;
+
+        void PerformHeuristicSearch(QString searchTerm, QString platform, QSet<QueueElementID>& found, AssetProcessor::JobIdEscalationList& escalationList, bool isStatusRequest, int searchRules = 0);
+        void PerformUUIDSearch(AZ::Uuid searchUuid, QString platform, QSet<QueueElementID>& found, AssetProcessor::JobIdEscalationList& escalationList, bool isStatusRequest);
+
+        int itemCount() const;
+        RCJob* getItem(int index) const;
+        int GetIndexOfProcessingJob(const QueueElementID& elementId);
+
+        ///! EraseJobs expects the database name of the source file.  (So with outputprefix)
+        void EraseJobs(QString sourceFileDatabaseName, AZStd::vector<RCJob*>& pendingJobs);
+
+    private:
+
+        AZStd::vector<RCJob*> m_jobs;
+        QSet<RCJob*> m_jobsInFlight;
+
+        // Keeps track of jobs waiting on the APM thread to finish writing out to the catalog
+        // This prevents job dependencies from starting before the dependent job is actually done
+        // Since the jobs aren't uniquely identified, and the APM thread can fall behind, we keep track of how many have finished
+        QHash<QueueElementID, int> m_finishedJobsNotInCatalog;
+
+        // profiler showed much of our time was spent in IsInQueue.
+        QMultiMap<QueueElementID, RCJob*> m_jobsInQueueLookup;
+    };
+} // namespace AssetProcessor
+
+
+#endif // RCQUEUEMODEL_H