Asset Processor: Remove gem loading from AP (#6488)

* AssetBuilder sends builder registration network message to AP

Signed-off-by: amzn-mike <80125227+amzn-mike@users.noreply.github.com>

* Add AP activating status message

Signed-off-by: amzn-mike <80125227+amzn-mike@users.noreply.github.com>

* First builder handles registration.

Fixed deadlock caused by AP and AssetBuilder waiting on each other when registering by moving AP builder start code to a thread

Signed-off-by: amzn-mike <80125227+amzn-mike@users.noreply.github.com>

* Clean up external builder registration

Signed-off-by: amzn-mike <80125227+amzn-mike@users.noreply.github.com>

* Add thread description for builder manager idle thread

Signed-off-by: amzn-mike <80125227+amzn-mike@users.noreply.github.com>

* Remove gem loading

Signed-off-by: amzn-mike <80125227+amzn-mike@users.noreply.github.com>

* Clean up builder registration and remove unused functions

Signed-off-by: amzn-mike <80125227+amzn-mike@users.noreply.github.com>

* Remove PostActivate call from batch application since it will be called after builders are registered

Signed-off-by: amzn-mike <80125227+amzn-mike@users.noreply.github.com>

* Removal external builder dependency scanning since we no longer support builder dlls

Signed-off-by: amzn-mike <80125227+amzn-mike@users.noreply.github.com>

* Fix missing bus disconnect

Signed-off-by: amzn-mike <80125227+amzn-mike@users.noreply.github.com>

* Remove unused variable

Signed-off-by: amzn-mike <80125227+amzn-mike@users.noreply.github.com>

* Moved AP-AssetBuilder specific types into AssetBuilder.Static library.  Also removed some unused/old code

Signed-off-by: amzn-mike <80125227+amzn-mike@users.noreply.github.com>
This commit is contained in:
amzn-mike
2022-01-04 14:34:56 -06:00
committed by GitHub
parent 052e282208
commit 8ee384f436
20 changed files with 659 additions and 727 deletions
@@ -85,7 +85,7 @@ namespace AssetProcessor
enum AssetCatalogStatus
{
RequiresSaving,
RequiresSaving,
UpToDate
};
@@ -213,11 +213,11 @@ namespace AssetProcessor
bool m_critical = false;
int m_priority = -1;
// indicates whether we need to check the server first for the outputs of this job
// indicates whether we need to check the server first for the outputs of this job
// before we start processing locally
bool m_checkServer = false;
// Indicates whether this job needs to be processed irrespective of whether its fingerprint got modified or not.
// Indicates whether this job needs to be processed irrespective of whether its fingerprint got modified or not.
bool m_autoProcessJob = false;
AssetBuilderSDK::AssetBuilderDesc m_assetBuilderDesc;
@@ -251,9 +251,9 @@ namespace AssetProcessor
JobDetails() = default;
};
//! JobDesc struct is used for identifying jobs that need to be processed again
//! because of job dependency declared on them by other jobs
//! JobDesc struct is used for identifying jobs that need to be processed again
//! because of job dependency declared on them by other jobs
struct JobDesc
{
AZStd::string m_databaseSourceName;
@@ -283,7 +283,7 @@ namespace AssetProcessor
}
};
//! JobIndentifier is an internal structure that store all the data that can uniquely identify a job
//! JobIndentifier is an internal structure that store all the data that can uniquely identify a job
struct JobIndentifier
{
JobDesc m_jobDesc;
@@ -165,7 +165,7 @@ void MainWindow::Activate()
ui->connectionTreeView->header()->resizeSection(ConnectionManager::PortColumn, 60);
ui->connectionTreeView->header()->resizeSection(ConnectionManager::PlatformColumn, 60);
ui->connectionTreeView->header()->resizeSection(ConnectionManager::AutoConnectColumn, 60);
ui->connectionTreeView->header()->setStretchLastSection(false);
connect(ui->connectionTreeView->selectionModel(), &QItemSelectionModel::selectionChanged, this, &MainWindow::OnConnectionSelectionChanged);
@@ -189,12 +189,12 @@ void MainWindow::Activate()
ui->allowListAllowedListConnectionsListView->setModel(&m_allowedListAddresses);
connect(ui->allowedListRejectedConnectionsListView, &QListView::clicked, this, &MainWindow::OnRejectedConnectionsListViewClicked);
ui->allowedListRejectedConnectionsListView->setModel(&m_rejectedAddresses);
connect(ui->allowedListEnableCheckBox, &QCheckBox::toggled, this, &MainWindow::OnAllowedListCheckBoxToggled);
connect(ui->allowedListAddHostNameToolButton, &QToolButton::clicked, this, &MainWindow::OnAddHostNameAllowedListButtonClicked);
connect(ui->allowedListAddIPToolButton, &QPushButton::clicked, this, &MainWindow::OnAddIPAllowedListButtonClicked);
connect(ui->allowedListToAllowedListToolButton, &QPushButton::clicked, this, &MainWindow::OnToAllowedListButtonClicked);
connect(ui->allowedListToRejectedListToolButton, &QToolButton::clicked, this, &MainWindow::OnToRejectedListButtonClicked);
@@ -204,7 +204,7 @@ void MainWindow::Activate()
QRegExpValidator* hostNameValidator = new QRegExpValidator(validHostName, this);
ui->allowedListAddHostNameLineEdit->setValidator(hostNameValidator);
QRegExpValidator* ipValidator = new QRegExpValidator(validIP, this);
ui->allowedListAddIPLineEdit->setValidator(ipValidator);
@@ -235,7 +235,7 @@ void MainWindow::Activate()
m_logSortFilterProxy->setSourceModel(m_logsModel);
m_logSortFilterProxy->setFilterKeyColumn(AzToolsFramework::Logging::LogTableModel::ColumnMessage);
m_logSortFilterProxy->setFilterCaseSensitivity(Qt::CaseInsensitive);
ui->jobLogTableView->setModel(m_logSortFilterProxy);
ui->jobLogTableView->setItemDelegate(new AzToolsFramework::Logging::LogTableItemDelegate(ui->jobLogTableView));
ui->jobLogTableView->setExpandOnSelection();
@@ -400,7 +400,7 @@ void MainWindow::Activate()
bool zeroAnalysisModeFromSettings = settings.value("EnableZeroAnalysis", QVariant(true)).toBool();
settings.endGroup();
QObject::connect(ui->modtimeSkippingCheckBox, &QCheckBox::stateChanged, this,
QObject::connect(ui->modtimeSkippingCheckBox, &QCheckBox::stateChanged, this,
[this](int newCheckState)
{
bool newOption = newCheckState == Qt::Checked ? true : false;
@@ -543,7 +543,7 @@ void MainWindow::OnAddConnection(bool /*checked*/)
m_guiApplicationManager->GetConnectionManager()->addUserConnection();
}
void MainWindow::OnAllowedListConnectionsListViewClicked()
void MainWindow::OnAllowedListConnectionsListViewClicked()
{
ui->allowedListRejectedConnectionsListView->clearSelection();
}
@@ -553,7 +553,7 @@ void MainWindow::OnRejectedConnectionsListViewClicked()
ui->allowListAllowedListConnectionsListView->clearSelection();
}
void MainWindow::OnAllowedListCheckBoxToggled()
void MainWindow::OnAllowedListCheckBoxToggled()
{
if (!ui->allowedListEnableCheckBox->isChecked())
{
@@ -588,7 +588,7 @@ void MainWindow::OnAllowedListCheckBoxToggled()
ui->allowedListToAllowedListToolButton->setEnabled(true);
ui->allowedListToRejectedListToolButton->setEnabled(true);
}
m_guiApplicationManager->GetConnectionManager()->AllowedListingEnabled(ui->allowedListEnableCheckBox->isChecked());
}
@@ -858,7 +858,7 @@ void MainWindow::OnAssetProcessorStatusChanged(const AssetProcessor::AssetProces
text = tr("Working, analyzing jobs remaining %1, processing jobs remaining %2...").arg(m_createJobCount).arg(m_processJobsCount);
ui->timerContainerWidget->setVisible(false);
ui->productAssetDetailsPanel->SetScanQueueEnabled(false);
IntervalAssetTabFilterRefresh();
}
else
@@ -877,7 +877,7 @@ void MainWindow::OnAssetProcessorStatusChanged(const AssetProcessor::AssetProces
break;
case AssetProcessorStatus::Processing_Jobs:
CheckStartProcessTimers();
m_processJobsCount = entry.m_count;
m_processJobsCount = entry.m_count;
if (m_processJobsCount + m_createJobCount > 0)
{
@@ -983,7 +983,7 @@ void MainWindow::ApplyConfig()
ui->jobLogTableView->header()->resizeSection(AzToolsFramework::Logging::LogTableModel::ColumnType, m_config.logTypeColumnWidth);
}
MainWindow::LogSortFilterProxy::LogSortFilterProxy(QObject* parentOjbect) : QSortFilterProxyModel(parentOjbect)
MainWindow::LogSortFilterProxy::LogSortFilterProxy(QObject* parentOjbect) : QSortFilterProxyModel(parentOjbect)
{
}
@@ -1302,7 +1302,7 @@ void MainWindow::ShowJobViewContextMenu(const QPoint& pos)
ui->sourceAssetDetailsPanel->GoToSource(item->m_elementId.GetInputAssetName().toUtf8().constData());
});
QString productMenuTitle(tr("View product asset..."));
QString productMenuTitle(tr("View product asset..."));
if (item->m_jobState != AzToolsFramework::AssetSystem::JobStatus::Completed)
{
QString disabledActionTooltip(tr("Only completed jobs are available in the Assets tab."));
@@ -1610,7 +1610,7 @@ void MainWindow::ShowProductAssetContextMenu(const QPoint& pos)
{
AzQtComponents::ShowFileOnDesktop(pathToProduct.GetValue());
}
});
QString fileOrFolder(cachedAsset->getChildCount() > 0 ? tr("folder") : tr("file"));
@@ -231,44 +231,6 @@ bool ApplicationManager::InitiatedShutdown() const
return m_duringShutdown;
}
void ApplicationManager::GetExternalBuilderFileList(QStringList& externalBuilderModules)
{
externalBuilderModules.clear();
static const char* builder_folder_name = "Builders";
// LY_ASSET_BUILDERS is defined by the CMakeLists.txt. The asset builders add themselves to a variable that
// is populated to allow selective building of those asset builder targets.
// This allows left over Asset builders in the output directory to not be loaded by the AssetProcessor
#if !defined(LY_ASSET_BUILDERS)
#error LY_ASSET_BUILDERS was not defined for ApplicationManager.cpp
#endif
QDir builderDir = QDir::toNativeSeparators(QString(this->m_frameworkApp.GetExecutableFolder()));
builderDir.cd(QString(builder_folder_name));
if (builderDir.exists())
{
AZStd::vector<AZStd::string> tokens;
AZ::StringFunc::Tokenize(AZStd::string_view(LY_ASSET_BUILDERS), tokens, ',');
AZStd::string builderLibrary;
for (const AZStd::string& token : tokens)
{
QString assetBuilderPath(token.c_str());
if (builderDir.exists(assetBuilderPath))
{
externalBuilderModules.push_back(builderDir.absoluteFilePath(assetBuilderPath));
}
}
}
if (externalBuilderModules.empty())
{
AZ_TracePrintf(AssetProcessor::ConsoleChannel, "AssetProcessor was unable to locate any external builders\n");
}
}
QDir ApplicationManager::GetSystemRoot() const
{
return m_systemRoot;
@@ -459,15 +421,6 @@ void ApplicationManager::PopulateApplicationDependencies()
m_filesOfInterest.push_back(dir.absoluteFilePath(pathWithPlatformExtension));
}
// Get the external builder modules to add to the files of interest
QStringList builderModuleFileList;
GetExternalBuilderFileList(builderModuleFileList);
for (const QString& builderModuleFile : builderModuleFileList)
{
m_filesOfInterest.push_back(builderModuleFile);
}
QDir assetRoot;
AssetUtilities::ComputeAssetRoot(assetRoot);
@@ -139,7 +139,7 @@ protected:
void RegisterObjectForQuit(QObject* source, bool insertInFront = false);
bool NeedRestart() const;
void addRunningThread(AssetProcessor::ThreadWorker* thread);
template<class BuilderClass>
void RegisterInternalBuilder(const QString& builderName);
@@ -151,9 +151,6 @@ protected:
bool m_duringStartup = true;
AssetProcessorAZApplication m_frameworkApp;
QCoreApplication* m_qApp = nullptr;
//! Get the list of external builder files for this asset processor
void GetExternalBuilderFileList(QStringList& externalBuilderModules);
virtual void Reflect() = 0;
virtual const char* GetLogBaseName() = 0;
@@ -25,6 +25,7 @@
#include <native/InternalBuilders/SettingsRegistryBuilder.h>
#include <AzToolsFramework/Application/Ticker.h>
#include <AzToolsFramework/ToolsFileUtils/ToolsFileUtils.h>
#include <AssetBuilder/AssetBuilderStatic.h>
#include <iostream>
@@ -32,9 +33,6 @@
#include <QElapsedTimer>
//! Amount of time to wait between checking the status of the AssetBuilder process
static const int s_MaximumSleepTimeMS = 10;
//! CreateJobs will wait up to 2 minutes before timing out
//! This shouldn't need to be so high but very large slices can take a while to process currently
//! This should be reduced down to something more reasonable after slice jobs are sped up
@@ -64,6 +62,7 @@ ApplicationManagerBase::~ApplicationManagerBase()
AZ::Debug::TraceMessageBus::Handler::BusDisconnect();
AssetProcessor::AssetBuilderRegistrationBus::Handler::BusDisconnect();
AssetBuilderSDK::AssetBuilderBus::Handler::BusDisconnect();
AssetProcessor::AssetBuilderInfoBus::Handler::BusDisconnect();
if (m_settingsRegistryBuilder)
{
@@ -192,7 +191,7 @@ void ApplicationManagerBase::InitAssetProcessorManager()
{
m_assetProcessorManager->SetEnableModtimeSkippingFeature(true);
}
if (commandLine->HasSwitch(Command_enableQueryLogging.m_switch))
{
m_assetProcessorManager->SetQueryLogging(true);
@@ -206,7 +205,7 @@ void ApplicationManagerBase::InitAssetProcessorManager()
{
m_dependencyScanPattern = commandLine->GetSwitchValue(Command_dsp.m_switch, 0).c_str();
}
m_fileDependencyScanPattern = "*";
if (commandLine->HasSwitch(Command_fileDependencyScanPattern.m_switch))
@@ -327,7 +326,7 @@ void ApplicationManagerBase::InitAssetCatalog()
AssetProcessor::AssetCatalog* catalog = new AssetCatalog(assetCatalogHelper, m_platformConfiguration);
// Using a direct connection so we know the catalog has been updated before continuing on with code might depend on the asset being in the catalog
connect(m_assetProcessorManager, &AssetProcessorManager::AssetMessage, catalog, &AssetCatalog::OnAssetMessage, Qt::DirectConnection);
connect(m_assetProcessorManager, &AssetProcessorManager::AssetMessage, catalog, &AssetCatalog::OnAssetMessage, Qt::DirectConnection);
connect(m_assetProcessorManager, &AssetProcessorManager::SourceQueued, catalog, &AssetCatalog::OnSourceQueued);
connect(m_assetProcessorManager, &AssetProcessorManager::SourceFinished, catalog, &AssetCatalog::OnSourceFinished);
connect(m_assetProcessorManager, &AssetProcessorManager::PathDependencyResolved, catalog, &AssetCatalog::OnDependencyResolved);
@@ -379,12 +378,12 @@ void ApplicationManagerBase::InitAssetScanner()
QObject::connect(m_assetScanner, &AssetScanner::FilesFound, [this](QSet<AssetFileInfo> files) { m_fileStateCache->AddInfoSet(files); });
QObject::connect(m_assetScanner, &AssetScanner::FoldersFound, [this](QSet<AssetFileInfo> files) { m_fileStateCache->AddInfoSet(files); });
QObject::connect(m_assetScanner, &AssetScanner::ExcludedFound, [this](QSet<AssetFileInfo> files) { m_fileStateCache->AddInfoSet(files); });
// file table
QObject::connect(m_assetScanner, &AssetScanner::AssetScanningStatusChanged, m_fileProcessor.get(), &FileProcessor::OnAssetScannerStatusChange);
QObject::connect(m_assetScanner, &AssetScanner::FilesFound, m_fileProcessor.get(), &FileProcessor::AssessFilesFromScanner);
QObject::connect(m_assetScanner, &AssetScanner::FoldersFound, m_fileProcessor.get(), &FileProcessor::AssessFoldersFromScanner);
}
void ApplicationManagerBase::DestroyAssetScanner()
@@ -591,6 +590,51 @@ void ApplicationManagerBase::InitConnectionManager()
}, AZStd::placeholders::_1, AZStd::placeholders::_2, AZStd::placeholders::_3, AZStd::placeholders::_4)
);
m_connectionManager->RegisterService(
AssetBuilder::BuilderRegistrationRequest::MessageType,
[this](unsigned int /*connId*/, unsigned int /*type*/, unsigned int /*serial*/, QByteArray payload, QString)
{
AssetBuilder::BuilderRegistrationRequest registrationRequest;
if (m_builderRegistrationComplete)
{
return;
}
m_builderRegistrationComplete = true;
if (AssetProcessor::UnpackMessage(payload, registrationRequest))
{
for (const auto& builder : registrationRequest.m_builders)
{
AssetBuilderSDK::AssetBuilderDesc desc;
desc.m_name = builder.m_name;
desc.m_patterns = builder.m_patterns;
desc.m_version = builder.m_version;
desc.m_analysisFingerprint = builder.m_analysisFingerprint;
desc.m_flags = builder.m_flags;
desc.m_busId = builder.m_busId;
desc.m_flagsByJobKey = builder.m_flagsByJobKey;
desc.m_productsToKeepOnFailure = builder.m_productsToKeepOnFailure;
// Builders registered this way are always external builders
desc.m_builderType = AssetBuilderSDK::AssetBuilderDesc::AssetBuilderType::External;
RegisterBuilderInformation(desc);
}
QTimer::singleShot(
0, this,
[this]()
{
if (!PostActivate())
{
QuitRequested();
}
});
}
});
//You can get Asset Processor Current State
using AzFramework::AssetSystem::RequestAssetProcessorStatus;
auto GetState = [this](unsigned int connId, unsigned int, unsigned int serial, QByteArray payload, QString)
@@ -633,11 +677,11 @@ void ApplicationManagerBase::InitConnectionManager()
AssetProcessorPlatformStatusRequest requestMessage;
if (AssetProcessor::UnpackMessage(payload, requestMessage))
{
AzToolsFramework::AssetSystemRequestBus::BroadcastResult(responseMessage.m_isPlatformEnabled,
AzToolsFramework::AssetSystemRequestBus::BroadcastResult(responseMessage.m_isPlatformEnabled,
&AzToolsFramework::AssetSystemRequestBus::Events::IsAssetPlatformEnabled, requestMessage.m_platform.c_str());
}
AssetProcessor::ConnectionBus::Event(connId,
AssetProcessor::ConnectionBus::Event(connId,
&AssetProcessor::ConnectionBus::Events::SendResponse, serial, responseMessage);
});
@@ -653,11 +697,11 @@ void ApplicationManagerBase::InitConnectionManager()
if (AssetProcessor::UnpackMessage(payload, requestMessage))
{
const char* platformIdentifier = requestMessage.m_platform.c_str();
responseMessage.m_numberOfPendingJobs =
responseMessage.m_numberOfPendingJobs =
GetRCController()->NumberOfPendingJobsPerPlatform(platformIdentifier);
}
AssetProcessor::ConnectionBus::Event(connId,
AssetProcessor::ConnectionBus::Event(connId,
&AssetProcessor::ConnectionBus::Events::SendResponse, serial, responseMessage);
});
}
@@ -696,7 +740,7 @@ void ApplicationManagerBase::InitAssetRequestHandler(AssetProcessor::AssetReques
QObject::connect(GetAssetProcessorManager(), &AssetProcessorManager::SendAssetExistsResponse, m_assetRequestHandler, &AssetRequestHandler::OnRequestAssetExistsResponse);
QObject::connect(GetAssetProcessorManager(), &AssetProcessorManager::FenceFileDetected, m_assetRequestHandler, &AssetRequestHandler::OnFenceFileDetected);
// connect the Asset Request Handler to RC:
QObject::connect(m_assetRequestHandler, &AssetRequestHandler::RequestCompileGroup, GetRCController(), &RCController::OnRequestCompileGroup);
QObject::connect(m_assetRequestHandler, &AssetRequestHandler::RequestEscalateAssetBySearchTerm, GetRCController(), &RCController::OnEscalateJobsBySearchTerm);
@@ -840,14 +884,6 @@ bool ApplicationManagerBase::Run()
return false;
}
bool startedSuccessfully = true;
if (!PostActivate())
{
QuitRequested();
startedSuccessfully = false;
}
AZ_Printf(AssetProcessor::ConsoleChannel, "Asset Processor Batch Processing Started.\n");
AZ_Printf(AssetProcessor::ConsoleChannel, "-----------------------------------------\n");
QElapsedTimer allAssetsProcessingTimer;
@@ -867,7 +903,7 @@ bool ApplicationManagerBase::Run()
RemoveOldTempFolders();
Destroy();
return (startedSuccessfully && FailedAssetsCount() == 0);
return FailedAssetsCount() == 0;
}
void ApplicationManagerBase::HandleFileRelocation() const
@@ -899,7 +935,7 @@ void ApplicationManagerBase::HandleFileRelocation() const
while(!m_sourceControlReady)
{
// We need to wait for source control to be ready before continuing
if (printCounter % 10 == 0)
{
AZ_TracePrintf(AssetProcessor::ConsoleChannel, "Waiting for Source Control connection\n");
@@ -1129,7 +1165,7 @@ void ApplicationManagerBase::CheckForIdle()
TryScanProductDependencies();
TryHandleFileRelocation();
// since we are shutting down, we save the registry and then we quit.
AZ_Printf(AssetProcessor::ConsoleChannel, "No assets remain in the build queue. Saving the catalog, and then shutting down.\n");
// stop accepting any further idle messages, as we will shut down - don't want this function to repeat!
@@ -1173,7 +1209,7 @@ void ApplicationManagerBase::InitBuilderManager()
{
m_builderManager->ConnectionLost(connId);
});
}
void ApplicationManagerBase::ShutdownBuilderManager()
@@ -1207,7 +1243,7 @@ void ApplicationManagerBase::ShutDownAssetDatabase()
AzToolsFramework::AssetDatabase::AssetDatabaseRequests::Bus::Handler::BusDisconnect();
}
void ApplicationManagerBase::InitFileProcessor()
void ApplicationManagerBase::InitFileProcessor()
{
AssetProcessor::ThreadController<AssetProcessor::FileProcessor>* fileProcessorHelper = new AssetProcessor::ThreadController<AssetProcessor::FileProcessor>();
@@ -1298,22 +1334,13 @@ bool ApplicationManagerBase::Activate()
}
InitBuilderConfiguration();
m_isCurrentlyLoadingGems = true;
if (!ActivateModules())
{
// ActivateModules reports any errors it encounters.
m_isCurrentlyLoadingGems = false;
return false;
}
m_isCurrentlyLoadingGems = false;
PopulateApplicationDependencies();
InitAssetProcessorManager();
AssetBuilderSDK::InitializeSerializationContext();
AssetBuilderSDK::InitializeBehaviorContext();
AssetBuilder::InitializeSerializationContext();
InitFileStateCache();
InitFileProcessor();
@@ -1341,7 +1368,7 @@ bool ApplicationManagerBase::Activate()
RegisterObjectForQuit(m_rcController);
m_connectionsToRemoveOnShutdown << QObject::connect(
m_assetProcessorManager, &AssetProcessor::AssetProcessorManager::AssetProcessorManagerIdleState,
m_assetProcessorManager, &AssetProcessor::AssetProcessorManager::AssetProcessorManagerIdleState,
this, [this](bool state)
{
if (state)
@@ -1362,7 +1389,7 @@ bool ApplicationManagerBase::Activate()
});
m_connectionsToRemoveOnShutdown << QObject::connect(
this, &ApplicationManagerBase::CheckAssetProcessorManagerIdleState,
this, &ApplicationManagerBase::CheckAssetProcessorManagerIdleState,
m_assetProcessorManager, &AssetProcessor::AssetProcessorManager::CheckAssetProcessorIdleState);
MakeActivationConnections();
@@ -1376,6 +1403,22 @@ bool ApplicationManagerBase::Activate()
return false;
}
}
AssetProcessor::AssetProcessorStatusEntry entry(AssetProcessor::AssetProcessorStatus::Initializing_Builders, 0, QString());
Q_EMIT AssetProcessorStatusChanged(entry);
AZStd::thread_desc desc;
desc.m_name = "Builder Component Registration";
AZStd::thread builderRegistrationThread(
desc,
[]()
{
AssetProcessor::BuilderRef builder;
AssetProcessor::BuilderManagerBus::BroadcastResult(builder, &AssetProcessor::BuilderManagerBus::Events::GetBuilder, true);
});
builderRegistrationThread.detach();
return true;
}
@@ -1384,11 +1427,6 @@ bool ApplicationManagerBase::PostActivate()
m_connectionManager->LoadConnections();
InitializeInternalBuilders();
if (!InitializeExternalBuilders())
{
AZ_Error("AssetProcessor", false, "AssetProcessor is closing. Failed to initialize and load all the external builders. Please ensure that Builders_Temp directory is not read-only. Please see log for more information.\n");
return false;
}
Q_EMIT OnBuildersRegistered();
@@ -1401,7 +1439,7 @@ bool ApplicationManagerBase::PostActivate()
AZ::SystemTickBus::Broadcast(&AZ::SystemTickEvents::OnSystemTick);
});
// now that everything is up and running, we start scanning. Before this, we don't want file events to start percolating through the
// now that everything is up and running, we start scanning. Before this, we don't want file events to start percolating through the
// asset system.
GetAssetScanner()->StartScan();
@@ -1425,124 +1463,20 @@ bool ApplicationManagerBase::InitializeInternalBuilders()
return result;
}
bool ApplicationManagerBase::InitializeExternalBuilders()
{
AssetProcessor::AssetProcessorStatusEntry entry(AssetProcessor::AssetProcessorStatus::Initializing_Builders);
Q_EMIT AssetProcessorStatusChanged(entry);
QCoreApplication::processEvents(QEventLoop::AllEvents);
// Get the list of external build modules (full paths)
QStringList fileList;
GetExternalBuilderFileList(fileList);
for (const QString& filePath : fileList)
{
if (QLibrary::isLibrary(filePath))
{
AssetProcessor::ExternalModuleAssetBuilderInfo* externalAssetBuilderInfo = new AssetProcessor::ExternalModuleAssetBuilderInfo(filePath);
AssetProcessor::AssetBuilderType assetBuilderType = externalAssetBuilderInfo->Load();
AZ_TracePrintf(AssetProcessor::ConsoleChannel, "AssetProcessor is loading library %s\n", filePath.toUtf8().data());
if (assetBuilderType == AssetProcessor::AssetBuilderType::None)
{
AZ_Warning(AssetProcessor::DebugChannel, false, "Non-builder DLL was found in Builders directory %s, skipping. \n", filePath.toUtf8().data());
delete externalAssetBuilderInfo;
continue;
}
if (assetBuilderType == AssetProcessor::AssetBuilderType::Invalid)
{
AZ_Warning(AssetProcessor::DebugChannel, false, "AssetProcessor was not able to load the library: %s\n", filePath.toUtf8().data());
delete externalAssetBuilderInfo;
return false;
}
AZ_TracePrintf(AssetProcessor::ConsoleChannel, "Initializing and registering builder %s\n", externalAssetBuilderInfo->GetName().toUtf8().data());
m_currentExternalAssetBuilder = externalAssetBuilderInfo;
externalAssetBuilderInfo->Initialize();
m_currentExternalAssetBuilder = nullptr;
m_externalAssetBuilders.push_back(externalAssetBuilderInfo);
}
}
// Also init external builders which may be inside of Gems
AzToolsFramework::ToolsApplicationRequestBus::Broadcast(
&AzToolsFramework::ToolsApplicationRequests::CreateAndAddEntityFromComponentTags,
AZStd::vector<AZ::Crc32>({ AssetBuilderSDK::ComponentTags::AssetBuilder }), "AssetBuilders Entity");
return true;
}
bool ApplicationManagerBase::WaitForBuilderExit(AzFramework::ProcessWatcher* processWatcher, AssetBuilderSDK::JobCancelListener* jobCancelListener, AZ::u32 processTimeoutLimitInSeconds)
{
AZ::u32 exitCode = 0;
bool finishedOK = false;
QElapsedTimer ticker;
ProcessCommunicatorTracePrinter tracer(processWatcher->GetCommunicator(), "AssetBuilder");
ticker.start();
while (!finishedOK)
{
AZStd::this_thread::sleep_for(AZStd::chrono::milliseconds(s_MaximumSleepTimeMS));
tracer.Pump();
if (ticker.elapsed() > processTimeoutLimitInSeconds * 1000 || (jobCancelListener && jobCancelListener->IsCancelled()))
{
break;
}
if (!processWatcher->IsProcessRunning(&exitCode))
{
finishedOK = true; // we either cant wait for it, or it finished.
break;
}
}
tracer.Pump(); // empty whats left if possible.
if (processWatcher->IsProcessRunning(&exitCode))
{
processWatcher->TerminateProcess(1);
}
if (exitCode != 0)
{
AZ_Error(AssetProcessor::ConsoleChannel, false, "AssetBuilder exited with error code %d", exitCode);
return false;
}
else if (jobCancelListener && jobCancelListener->IsCancelled())
{
AZ_TracePrintf(AssetProcessor::DebugChannel, "AssetBuilder was terminated. There was a request to cancel the job.\n");
return false;
}
else if (!finishedOK)
{
AZ_Error(AssetProcessor::ConsoleChannel, false, "AssetBuilder failed to terminate within %d seconds", processTimeoutLimitInSeconds);
return false;
}
return true;
}
void ApplicationManagerBase::RegisterBuilderInformation(const AssetBuilderSDK::AssetBuilderDesc& builderDesc)
{
// Create Job Function validation
AZ_Error(AssetProcessor::ConsoleChannel,
builderDesc.m_createJobFunction,
"Create Job Function (m_createJobFunction) for %s builder is empty.\n",
builderDesc.m_name.c_str());
if (!builderDesc.IsExternalBuilder())
{
// Create Job Function validation
AZ_Error(
AssetProcessor::ConsoleChannel, builderDesc.m_createJobFunction,
"Create Job Function (m_createJobFunction) for %s builder is empty.\n", builderDesc.m_name.c_str());
// Process Job Function validation
AZ_Error(AssetProcessor::ConsoleChannel,
builderDesc.m_processJobFunction,
"Process Job Function (m_processJobFunction) for %s builder is empty.\n",
builderDesc.m_name.c_str());
// Process Job Function validation
AZ_Error(
AssetProcessor::ConsoleChannel, builderDesc.m_processJobFunction,
"Process Job Function (m_processJobFunction) for %s builder is empty.\n", builderDesc.m_name.c_str());
}
// Bus ID validation
AZ_Error(AssetProcessor::ConsoleChannel,
@@ -1550,67 +1484,66 @@ void ApplicationManagerBase::RegisterBuilderInformation(const AssetBuilderSDK::A
"Bus ID for %s builder is empty.\n",
builderDesc.m_name.c_str());
// This is an external builder registering, we will want to track its builder desc since it can register multiple ones
AZStd::string builderFilePath;
if (m_currentExternalAssetBuilder)
{
m_currentExternalAssetBuilder->RegisterBuilderDesc(builderDesc.m_busId);
builderFilePath = m_currentExternalAssetBuilder->GetModuleFullPath().toUtf8().data();
}
AssetBuilderSDK::AssetBuilderDesc modifiedBuilderDesc = builderDesc;
// Allow for overrides defined in a BuilderConfig.ini file to update our code defined default values
AssetProcessor::BuilderConfigurationRequestBus::Broadcast(&AssetProcessor::BuilderConfigurationRequests::UpdateBuilderDescriptor, builderDesc.m_name, modifiedBuilderDesc);
if (builderDesc.IsExternalBuilder())
{
// We're going to override the createJob function so we can run it externally in AssetBuilder, rather than having it run inside the AP
modifiedBuilderDesc.m_createJobFunction = [builderFilePath](const AssetBuilderSDK::CreateJobsRequest& request, AssetBuilderSDK::CreateJobsResponse& response)
// We're going to override the createJob function so we can run it externally in AssetBuilder, rather than having it run
// inside the AP
modifiedBuilderDesc.m_createJobFunction =
[](const AssetBuilderSDK::CreateJobsRequest& request, AssetBuilderSDK::CreateJobsResponse& response)
{
AssetProcessor::BuilderRef builderRef;
AssetProcessor::BuilderManagerBus::BroadcastResult(builderRef, &AssetProcessor::BuilderManagerBusTraits::GetBuilder, false);
if (builderRef)
{
AssetProcessor::BuilderRef builderRef;
AssetProcessor::BuilderManagerBus::BroadcastResult(builderRef, &AssetProcessor::BuilderManagerBusTraits::GetBuilder);
int retryCount = 0;
AssetProcessor::BuilderRunJobOutcome result;
if (builderRef)
do
{
int retryCount = 0;
AssetProcessor::BuilderRunJobOutcome result;
do
{
retryCount++;
result = builderRef->RunJob<AssetBuilderSDK::CreateJobsNetRequest, AssetBuilderSDK::CreateJobsNetResponse>(request, response, s_MaximumCreateJobsTimeSeconds, "create", builderFilePath, nullptr);
} while (result == AssetProcessor::BuilderRunJobOutcome::LostConnection && retryCount <= AssetProcessor::RetriesForJobNetworkError);
}
else
{
AZ_Error("AssetProcessor", false, "Failed to retrieve a valid builder to process job");
}
};
retryCount++;
result = builderRef->RunJob<AssetBuilder::CreateJobsNetRequest, AssetBuilder::CreateJobsNetResponse>(
request, response, s_MaximumCreateJobsTimeSeconds, "create", "", nullptr);
} while (result == AssetProcessor::BuilderRunJobOutcome::LostConnection &&
retryCount <= AssetProcessor::RetriesForJobNetworkError);
}
else
{
AZ_Error("AssetProcessor", false, "Failed to retrieve a valid builder to process job");
}
};
// Also override the processJob function to run externally
modifiedBuilderDesc.m_processJobFunction = [builderFilePath](const AssetBuilderSDK::ProcessJobRequest& request, AssetBuilderSDK::ProcessJobResponse& response)
modifiedBuilderDesc.m_processJobFunction =
[](const AssetBuilderSDK::ProcessJobRequest& request, AssetBuilderSDK::ProcessJobResponse& response)
{
AssetBuilderSDK::JobCancelListener jobCancelListener(request.m_jobId);
AssetProcessor::BuilderRef builderRef;
AssetProcessor::BuilderManagerBus::BroadcastResult(builderRef, &AssetProcessor::BuilderManagerBusTraits::GetBuilder, false);
if (builderRef)
{
AssetBuilderSDK::JobCancelListener jobCancelListener(request.m_jobId);
int retryCount = 0;
AssetProcessor::BuilderRunJobOutcome result;
AssetProcessor::BuilderRef builderRef;
AssetProcessor::BuilderManagerBus::BroadcastResult(builderRef, &AssetProcessor::BuilderManagerBusTraits::GetBuilder);
if (builderRef)
do
{
int retryCount = 0;
AssetProcessor::BuilderRunJobOutcome result;
do
{
retryCount++;
result = builderRef->RunJob<AssetBuilderSDK::ProcessJobNetRequest, AssetBuilderSDK::ProcessJobNetResponse>(request, response, s_MaximumProcessJobsTimeSeconds, "process", builderFilePath, &jobCancelListener, request.m_tempDirPath);
} while (result == AssetProcessor::BuilderRunJobOutcome::LostConnection && retryCount <= AssetProcessor::RetriesForJobNetworkError);
}
else
{
AZ_Error("AssetProcessor", false, "Failed to retrieve a valid builder to process job");
}
};
retryCount++;
result = builderRef->RunJob<AssetBuilder::ProcessJobNetRequest, AssetBuilder::ProcessJobNetResponse>(
request, response, s_MaximumProcessJobsTimeSeconds, "process", "", &jobCancelListener, request.m_tempDirPath);
} while (result == AssetProcessor::BuilderRunJobOutcome::LostConnection &&
retryCount <= AssetProcessor::RetriesForJobNetworkError);
}
else
{
AZ_Error("AssetProcessor", false, "Failed to retrieve a valid builder to process job");
}
};
}
if (m_builderDescMap.find(modifiedBuilderDesc.m_busId) != m_builderDescMap.end())
@@ -1768,7 +1701,7 @@ bool ApplicationManagerBase::CheckSufficientDiskSpace(const QString& savePath, q
[[maybe_unused]] bool result = AzToolsFramework::ToolsFileUtils::GetFreeDiskSpace(savePath, bytesFree);
AZ_Assert(result, "Unable to determine the amount of free space on drive containing path (%s).", savePath.toUtf8().constData());
if (bytesFree < requiredSpace + s_ReservedDiskSpaceInBytes)
{
if (shutdownIfInsufficient)
@@ -1806,8 +1739,8 @@ void ApplicationManagerBase::RemoveOldTempFolders()
return;
}
// We will remove old temp folders if either their modified time is older than the cutoff time or
// if the total number of temp folders have exceeded the maximum number of temp folders.
// We will remove old temp folders if either their modified time is older than the cutoff time or
// if the total number of temp folders have exceeded the maximum number of temp folders.
QFileInfoList entries = root.entryInfoList(QDir::Dirs | QDir::NoDotAndDotDot, QDir::Time); // sorting by modification time
int folderCount = 0;
bool removeFolder = false;
@@ -1821,9 +1754,9 @@ void ApplicationManagerBase::RemoveOldTempFolders()
// Since we are sorting the folders list from latest to oldest, we will either be in a state where we have to delete all the remaining folders or not
// because either we have reached the folder limit or reached the cutoff date limit.
removeFolder = removeFolder || (folderCount++ >= s_MaximumTempFolders) ||
removeFolder = removeFolder || (folderCount++ >= s_MaximumTempFolders) ||
(entry.lastModified() < cutoffTime);
if (removeFolder)
{
QDir dir(entry.absoluteFilePath());
@@ -1837,8 +1770,6 @@ void ApplicationManagerBase::ConnectivityStateChanged(const AzToolsFramework::So
Q_EMIT SourceControlReady();
}
void ApplicationManagerBase::OnAssetProcessorManagerIdleState(bool isIdle)
{
// these can come in during shutdown.
@@ -149,7 +149,6 @@ protected:
void CreateQtApplication() override;
bool InitializeInternalBuilders();
bool InitializeExternalBuilders();
void InitBuilderManager();
void ShutdownBuilderManager();
bool InitAssetDatabase();
@@ -173,8 +172,6 @@ protected:
AssetProcessor::AssetCatalog* GetAssetCatalog() const { return m_assetCatalog; }
static bool WaitForBuilderExit(AzFramework::ProcessWatcher* processWatcher, AssetBuilderSDK::JobCancelListener* jobCancelListener, AZ::u32 processTimeoutLimitInSeconds);
ApplicationServer* m_applicationServer = nullptr;
ConnectionManager* m_connectionManager = nullptr;
@@ -218,6 +215,8 @@ protected:
AZStd::shared_ptr<AssetProcessor::InternalRecognizerBasedBuilder> m_internalBuilder;
AZStd::shared_ptr<AssetProcessor::SettingsRegistryBuilder> m_settingsRegistryBuilder;
bool m_builderRegistrationComplete = false;
// Builder description map based on the builder id
AZStd::unordered_map<AZ::Uuid, AssetBuilderSDK::AssetBuilderDesc> m_builderDescMap;
@@ -231,7 +230,7 @@ protected:
AZStd::list<AssetProcessor::ExternalModuleAssetBuilderInfo*> m_externalAssetBuilders;
AssetProcessor::ExternalModuleAssetBuilderInfo* m_currentExternalAssetBuilder = nullptr;
QAtomicInt m_connectionsAwaitingAssetCatalogSave = 0;
int m_remainingAPMJobs = 0;
bool m_assetProcessorManagerIsReady = false;
@@ -16,6 +16,7 @@
#include <native/utilities/AssetBuilderInfo.h>
#include <QCoreApplication>
#include <AzCore/Settings/SettingsRegistryMergeUtils.h>
#include <AssetBuilder/AssetBuilderStatic.h>
namespace AssetProcessor
{
@@ -138,7 +139,7 @@ namespace AssetProcessor
}
}
bool Builder::Start()
bool Builder::Start(bool doRegistration)
{
// Get the current BinXXX folder based on the current running AP
QString applicationDir = QCoreApplication::instance()->applicationDirPath();
@@ -155,7 +156,7 @@ namespace AssetProcessor
return false;
}
const AZStd::vector<AZStd::string> params = BuildParams("resident", buildersFolder.c_str(), UuidString(), "", "");
const AZStd::vector<AZStd::string> params = BuildParams("resident", buildersFolder.c_str(), UuidString(), "", "", doRegistration);
m_processWatcher = LaunchProcess(fullExePathString.c_str(), params);
@@ -179,7 +180,7 @@ namespace AssetProcessor
return !m_processWatcher || (m_processWatcher && m_processWatcher->IsProcessRunning(exitCode));
}
AZStd::vector<AZStd::string> Builder::BuildParams(const char* task, const char* moduleFilePath, const AZStd::string& builderGuid, const AZStd::string& jobDescriptionFile, const AZStd::string& jobResponseFile) const
AZStd::vector<AZStd::string> Builder::BuildParams(const char* task, const char* moduleFilePath, const AZStd::string& builderGuid, const AZStd::string& jobDescriptionFile, const AZStd::string& jobResponseFile, bool doRegistration) const
{
QDir projectCacheRoot;
AssetUtilities::ComputeProjectCacheRoot(projectCacheRoot);
@@ -200,6 +201,11 @@ namespace AssetProcessor
params.emplace_back(AZStd::string::format(R"(-engine-path="%s")", enginePath.c_str()));
params.emplace_back(AZStd::string::format("-port=%d", portNumber));
if(doRegistration)
{
params.emplace_back("--register");
}
if (moduleFilePath && moduleFilePath[0])
{
params.emplace_back(AZStd::string::format(R"(-module="%s")", moduleFilePath));
@@ -232,7 +238,7 @@ namespace AssetProcessor
{
AzFramework::ProcessLauncher::ProcessLaunchInfo processLaunchInfo;
processLaunchInfo.m_processExecutableString = fullExePath;
AZStd::vector<AZStd::string> commandLineArray{ fullExePath };
commandLineArray.insert(commandLineArray.end(), params.begin(), params.end());
processLaunchInfo.m_commandlineParameters = AZStd::move(commandLineArray);
@@ -350,17 +356,19 @@ namespace AssetProcessor
BuilderManager::BuilderManager(ConnectionManager* connectionManager)
{
using namespace AZStd::placeholders;
connectionManager->RegisterService(AssetBuilderSDK::BuilderHelloRequest::MessageType(), AZStd::bind(&BuilderManager::IncomingBuilderPing, this, _1, _2, _3, _4, _5));
connectionManager->RegisterService(AssetBuilder::BuilderHelloRequest::MessageType(), AZStd::bind(&BuilderManager::IncomingBuilderPing, this, _1, _2, _3, _4, _5));
// Setup a background thread to pump the idle builders so they don't get blocked trying to output to stdout/err
m_pollingThread = AZStd::thread([this]()
AZStd::thread_desc desc;
desc.m_name = "BuilderManager Idle Pump";
m_pollingThread = AZStd::thread(desc, [this]()
{
while (!m_quitListener.WasQuitRequested())
{
while (!m_quitListener.WasQuitRequested())
{
PumpIdleBuilders();
AZStd::this_thread::sleep_for(AZStd::chrono::milliseconds(s_IdleBuilderPumpingDelayMS));
}
});
PumpIdleBuilders();
AZStd::this_thread::sleep_for(AZStd::chrono::milliseconds(s_IdleBuilderPumpingDelayMS));
}
});
m_quitListener.BusConnect();
BusConnect();
@@ -399,8 +407,8 @@ namespace AssetProcessor
void BuilderManager::IncomingBuilderPing(AZ::u32 connId, AZ::u32 /*type*/, AZ::u32 serial, QByteArray payload, QString platform)
{
AssetBuilderSDK::BuilderHelloRequest requestPing;
AssetBuilderSDK::BuilderHelloResponse responsePing;
AssetBuilder::BuilderHelloRequest requestPing;
AssetBuilder::BuilderHelloResponse responsePing;
if (!AZ::Utils::LoadObjectFromBufferInPlace(payload.data(), payload.length(), requestPing))
{
@@ -476,7 +484,7 @@ namespace AssetProcessor
return builder;
}
BuilderRef BuilderManager::GetBuilder()
BuilderRef BuilderManager::GetBuilder(bool doRegistration)
{
AZStd::shared_ptr<Builder> newBuilder;
BuilderRef builderRef;
@@ -484,27 +492,30 @@ namespace AssetProcessor
{
AZStd::unique_lock<AZStd::mutex> lock(m_buildersMutex);
for (auto itr = m_builders.begin(); itr != m_builders.end(); )
if (!doRegistration)
{
auto& builder = itr->second;
if (!builder->m_busy)
for (auto itr = m_builders.begin(); itr != m_builders.end();)
{
builder->PumpCommunicator();
auto& builder = itr->second;
if (builder->IsValid())
if (!builder->m_busy)
{
return BuilderRef(builder);
builder->PumpCommunicator();
if (builder->IsValid())
{
return BuilderRef(builder);
}
else
{
itr = m_builders.erase(itr);
}
}
else
{
itr = m_builders.erase(itr);
++itr;
}
}
else
{
++itr;
}
}
AZ_TracePrintf("BuilderManager", "Starting new builder for job request\n");
@@ -516,7 +527,7 @@ namespace AssetProcessor
builderRef = BuilderRef(newBuilder);
}
if (!newBuilder->Start())
if (!newBuilder->Start(doRegistration))
{
AZ_Error("BuilderManager", false, "Builder failed to start");
@@ -39,7 +39,7 @@ namespace AssetProcessor
virtual ~BuilderManagerBusTraits() = default;
//! Returns a builder for doing work
virtual BuilderRef GetBuilder() = 0;
virtual BuilderRef GetBuilder(bool doRegistration) = 0;
};
using BuilderManagerBus = AZ::EBus<BuilderManagerBusTraits>;
@@ -98,12 +98,12 @@ namespace AssetProcessor
private:
//! Starts the builder process and waits for it to connect
bool Start();
bool Start(bool doRegistration);
//! Sets the connection id and signals that the builder has connected
void SetConnection(AZ::u32 connId);
AZStd::vector<AZStd::string> BuildParams(const char* task, const char* moduleFilePath, const AZStd::string& builderGuid, const AZStd::string& jobDescriptionFile, const AZStd::string& jobResponseFile) const;
AZStd::vector<AZStd::string> BuildParams(const char* task, const char* moduleFilePath, const AZStd::string& builderGuid, const AZStd::string& jobDescriptionFile, const AZStd::string& jobResponseFile, bool doRegistration) const;
AZStd::unique_ptr<AzFramework::ProcessWatcher> LaunchProcess(const char* fullExePath, const AZStd::vector<AZStd::string>& params) const;
//! Waits for the builder exe to send the job response and pumps stdout/err
@@ -169,7 +169,7 @@ namespace AssetProcessor
void ConnectionLost(AZ::u32 connId);
//BuilderManagerBus
BuilderRef GetBuilder() override;
BuilderRef GetBuilder(bool doRegistration) override;
private:
@@ -50,7 +50,7 @@ namespace AssetProcessor
if (!netResponse.m_response.Succeeded() || s_createRequestFileForSuccessfulJob)
{
// we write the request out to disk for failure or debugging
// we write the request out to disk for failure or debugging
if (!DebugWriteRequestFile(tempFolderPath.c_str(), request, task, modulePath))
{
return BuilderRunJobOutcome::FailedToWriteDebugRequest;
@@ -83,7 +83,7 @@ namespace AssetProcessor
return false;
}
auto params = BuildParams(task.c_str(), modulePath.c_str(), "", jobRequestFile, jobResponseFile);
auto params = BuildParams(task.c_str(), modulePath.c_str(), "", jobRequestFile, jobResponseFile, false);
AZStd::string paramString;
AZ::StringFunc::Join(paramString, params.begin(), params.end(), " ");
@@ -318,16 +318,8 @@ bool GUIApplicationManager::Run()
qApp->setQuitOnLastWindowClosed(false);
QTimer::singleShot(0, this, [this]()
{
if (!PostActivate())
{
QuitRequested();
m_startedSuccessfully = false;
}
});
m_duringStartup = false;
m_startedSuccessfully = true;
int resultCode = qApp->exec(); // this blocks until the last window is closed.
@@ -483,6 +475,7 @@ bool GUIApplicationManager::PostActivate()
{
if (!ApplicationManagerBase::PostActivate())
{
m_startedSuccessfully = false;
return false;
}