From 8ee384f436a1092cafd3a7bdd7128bbbea0208f9 Mon Sep 17 00:00:00 2001 From: amzn-mike <80125227+amzn-mike@users.noreply.github.com> Date: Tue, 4 Jan 2022 14:34:56 -0600 Subject: [PATCH] Asset Processor: Remove gem loading from AP (#6488) * AssetBuilder sends builder registration network message to AP Signed-off-by: amzn-mike <80125227+amzn-mike@users.noreply.github.com> * Add AP activating status message Signed-off-by: amzn-mike <80125227+amzn-mike@users.noreply.github.com> * First builder handles registration. Fixed deadlock caused by AP and AssetBuilder waiting on each other when registering by moving AP builder start code to a thread Signed-off-by: amzn-mike <80125227+amzn-mike@users.noreply.github.com> * Clean up external builder registration Signed-off-by: amzn-mike <80125227+amzn-mike@users.noreply.github.com> * Add thread description for builder manager idle thread Signed-off-by: amzn-mike <80125227+amzn-mike@users.noreply.github.com> * Remove gem loading Signed-off-by: amzn-mike <80125227+amzn-mike@users.noreply.github.com> * Clean up builder registration and remove unused functions Signed-off-by: amzn-mike <80125227+amzn-mike@users.noreply.github.com> * Remove PostActivate call from batch application since it will be called after builders are registered Signed-off-by: amzn-mike <80125227+amzn-mike@users.noreply.github.com> * Removal external builder dependency scanning since we no longer support builder dlls Signed-off-by: amzn-mike <80125227+amzn-mike@users.noreply.github.com> * Fix missing bus disconnect Signed-off-by: amzn-mike <80125227+amzn-mike@users.noreply.github.com> * Remove unused variable Signed-off-by: amzn-mike <80125227+amzn-mike@users.noreply.github.com> * Moved AP-AssetBuilder specific types into AssetBuilder.Static library. Also removed some unused/old code Signed-off-by: amzn-mike <80125227+amzn-mike@users.noreply.github.com> --- .../AssetBuilder/AssetBuilderApplication.cpp | 2 + .../AssetBuilder/AssetBuilderComponent.cpp | 133 ++++--- .../AssetBuilder/AssetBuilderComponent.h | 13 +- .../AssetBuilder/AssetBuilderStatic.cpp | 182 +++++++++ .../AssetBuilder/AssetBuilderStatic.h | 140 +++++++ .../AssetBuilder/CMakeLists.txt | 17 + .../asset_builder_static_files.cmake | 12 + .../AssetBuilderSDK/AssetBuilderSDK.cpp | 168 -------- .../AssetBuilderSDK/AssetBuilderSDK.h | 164 +------- Code/Tools/AssetProcessor/CMakeLists.txt | 1 + .../AssetProcessor/native/assetprocessor.h | 16 +- .../AssetProcessor/native/ui/MainWindow.cpp | 30 +- .../native/utilities/ApplicationManager.cpp | 47 --- .../native/utilities/ApplicationManager.h | 5 +- .../utilities/ApplicationManagerBase.cpp | 359 +++++++----------- .../native/utilities/ApplicationManagerBase.h | 7 +- .../native/utilities/BuilderManager.cpp | 67 ++-- .../native/utilities/BuilderManager.h | 8 +- .../native/utilities/BuilderManager.inl | 4 +- .../utilities/GUIApplicationManager.cpp | 11 +- 20 files changed, 659 insertions(+), 727 deletions(-) create mode 100644 Code/Tools/AssetProcessor/AssetBuilder/AssetBuilderStatic.cpp create mode 100644 Code/Tools/AssetProcessor/AssetBuilder/AssetBuilderStatic.h create mode 100644 Code/Tools/AssetProcessor/AssetBuilder/asset_builder_static_files.cmake diff --git a/Code/Tools/AssetProcessor/AssetBuilder/AssetBuilderApplication.cpp b/Code/Tools/AssetProcessor/AssetBuilder/AssetBuilderApplication.cpp index da2edcc51f..074138b324 100644 --- a/Code/Tools/AssetProcessor/AssetBuilder/AssetBuilderApplication.cpp +++ b/Code/Tools/AssetProcessor/AssetBuilder/AssetBuilderApplication.cpp @@ -37,6 +37,7 @@ #include #include #include +#include namespace AssetBuilder { @@ -165,6 +166,7 @@ void AssetBuilderApplication::StartCommon(AZ::Entity* systemEntity) AssetBuilderSDK::InitializeSerializationContext(); AssetBuilderSDK::InitializeBehaviorContext(); + AssetBuilder::InitializeSerializationContext(); // the asset builder app never writes source files, only assets, so there is no need to do any kind of asset upgrading AZ::Data::AssetManager::Instance().SetAssetInfoUpgradingEnabled(false); diff --git a/Code/Tools/AssetProcessor/AssetBuilder/AssetBuilderComponent.cpp b/Code/Tools/AssetProcessor/AssetBuilder/AssetBuilderComponent.cpp index 78244ab02c..21b8c31cf4 100644 --- a/Code/Tools/AssetProcessor/AssetBuilder/AssetBuilderComponent.cpp +++ b/Code/Tools/AssetProcessor/AssetBuilder/AssetBuilderComponent.cpp @@ -34,6 +34,7 @@ #include #include #include +#include // Command-line parameter options: static const char* const s_paramHelp = "help"; // Print help information. @@ -51,10 +52,10 @@ static const char* const s_paramDebugCreate = "debug_create"; // Debug mode for static const char* const s_paramDebugProcess = "debug_process"; // Debug mode for the process job of the specified file. static const char* const s_paramPlatformTags = "tags"; // Additional list of tags to add platform tag list. static const char* const s_paramPlatform = "platform"; // Platform to use +static const char* const s_paramRegisterBuilders = "register"; // Indicates the AP is starting up and requesting a list of registered builders // Task modes: static const char* const s_taskResident = "resident"; // stays up and running indefinitely, accepting jobs via network connection -static const char* const s_taskRegisterBuilder = "register"; // outputs all the builder descriptors static const char* const s_taskCreateJob = "create"; // runs a builders createJobs function static const char* const s_taskProcessJob = "process"; // runs processJob function static const char* const s_taskDebug = "debug"; // runs a one shot job in a fake environment for a specified file. @@ -204,6 +205,40 @@ void AssetBuilderComponent::Reflect(AZ::ReflectContext* context) } } +bool AssetBuilderComponent::DoHelloPing() +{ + using namespace AssetBuilder; + + BuilderHelloRequest request; + BuilderHelloResponse response; + + AZStd::string id; + + if (!GetParameter(s_paramId, id)) + { + return false; + } + + request.m_uuid = AZ::Uuid::CreateString(id.c_str()); + + AZ_TracePrintf( + "AssetBuilderComponent", "RunInResidentMode: Pinging asset processor with the builder UUID %s\n", + request.m_uuid.ToString().c_str()); + + bool result = AzFramework::AssetSystem::SendRequest(request, response); + + AZ_Error("AssetBuilder", result, "Failed to send hello request to Asset Processor"); + // This error is only shown if we successfully got a response AND the response explicitly indicates the AP rejected the builder + AZ_Error("AssetBuilder", !result || response.m_accepted, "Asset Processor rejected connection request"); + + if (result) + { + AZ_TracePrintf("AssetBuilder", "Builder ID: %s\n", response.m_uuid.ToString().c_str()); + } + + return result; +} + bool AssetBuilderComponent::Run() { AZ_TracePrintf("AssetBuilderComponent", "Run: Parsing command line.\n"); @@ -217,8 +252,8 @@ bool AssetBuilderComponent::Run() } AZStd::string task; - AZStd::string debugFile; + if (GetParameter(s_paramDebug, debugFile, false)) { task = s_taskDebug; @@ -256,11 +291,13 @@ bool AssetBuilderComponent::Run() AZ_TracePrintf("AssetBuilderComponent", "Run: Connecting back to Asset Processor...\n"); bool connectedToAssetProcessor = ConnectToAssetProcessor(); //AP connection is required to access the asset catalog - AZ_Error("AssetBuilder", connectedToAssetProcessor, "Failed to establish a network connection to the AssetProcessor. Use -help for options.");; + AZ_Error("AssetBuilder", connectedToAssetProcessor, "Failed to establish a network connection to the AssetProcessor. Use -help for options."); + + bool registerBuilders = commandLine->GetNumSwitchValues(s_paramRegisterBuilders) > 0; IBuilderApplication* builderApplication = AZ::Interface::Get(); - if(!builderApplication) + if (!builderApplication) { AZ_Error("AssetBuilder", false, "Failed to retreive IBuilderApplication interface"); return false; @@ -274,7 +311,7 @@ bool AssetBuilderComponent::Run() { if (task == s_taskResident) { - result = RunInResidentMode(); + result = RunInResidentMode(registerBuilders); } else if (task == s_taskDebug) { @@ -370,43 +407,46 @@ bool AssetBuilderComponent::ConnectToAssetProcessor() ////////////////////////////////////////////////////////////////////////// -bool AssetBuilderComponent::RunInResidentMode() +bool AssetBuilderComponent::SendRegisteredBuildersToAp() { - using namespace AssetBuilderSDK; + AssetBuilder::BuilderRegistrationRequest registrationRequest; + + for (const auto& [uuid, desc] : m_assetBuilderDescMap) + { + AssetBuilder::BuilderRegistration registration; + + registration.m_name = desc->m_name; + registration.m_analysisFingerprint = desc->m_analysisFingerprint; + registration.m_flags = desc->m_flags; + registration.m_flagsByJobKey = desc->m_flagsByJobKey; + registration.m_version = desc->m_version; + registration.m_busId = desc->m_busId; + registration.m_patterns = desc->m_patterns; + registration.m_productsToKeepOnFailure = desc->m_productsToKeepOnFailure; + + registrationRequest.m_builders.push_back(AZStd::move(registration)); + } + + bool result = SendRequest(registrationRequest); + + AZ_Error("AssetBuilder", result, "Failed to send builder registration request to Asset Processor"); + + return result; +} + +bool AssetBuilderComponent::RunInResidentMode(bool sendRegistration) +{ + using namespace AssetBuilder; using namespace AZStd::placeholders; AZ_TracePrintf("AssetBuilderComponent", "RunInResidentMode: Starting resident mode (waiting for commands to arrive)\n"); - AZStd::string port, id, builderFolder; - - if (!GetParameter(s_paramId, id) - || !GetParameter(s_paramModule, builderFolder)) - { - return false; - } - - if (!LoadBuilders(builderFolder)) - { - return false; - } - AzFramework::SocketConnection::GetInstance()->AddMessageHandler(CreateJobsNetRequest::MessageType(), AZStd::bind(&AssetBuilderComponent::CreateJobsResidentHandler, this, _1, _2, _3, _4)); AzFramework::SocketConnection::GetInstance()->AddMessageHandler(ProcessJobNetRequest::MessageType(), AZStd::bind(&AssetBuilderComponent::ProcessJobResidentHandler, this, _1, _2, _3, _4)); - BuilderHelloRequest request; - BuilderHelloResponse response; + bool result = DoHelloPing() && ((sendRegistration && SendRegisteredBuildersToAp()) || !sendRegistration); - request.m_uuid = AZ::Uuid::CreateString(id.c_str()); - - AZ_TracePrintf("AssetBuilderComponent", "RunInResidentMode: Pinging asset processor with the builder UUID %s\n", request.m_uuid.ToString().c_str()); - - bool result = AzFramework::AssetSystem::SendRequest(request, response); - - AZ_Error("AssetBuilder", result, "Failed to send hello request to Asset Processor"); - // This error is only shown if we successfully got a response AND the response explicitly indicates the AP rejected the builder - AZ_Error("AssetBuilder", !result || response.m_accepted, "Asset Processor rejected connection request"); - - if (result && response.m_accepted) + if (result) { m_running = true; @@ -415,7 +455,6 @@ bool AssetBuilderComponent::RunInResidentMode() AzFramework::EngineConnectionEvents::Bus::Handler::BusConnect(); // Listen for disconnects - AZ_TracePrintf("AssetBuilder", "Builder ID: %s\n", response.m_uuid.ToString().c_str()); AZ_TracePrintf("AssetBuilder", "Resident mode ready\n"); m_mainEvent.acquire(); AZ_TracePrintf("AssetBuilder", "Shutting down\n"); @@ -736,11 +775,7 @@ bool AssetBuilderComponent::RunOneShotTask(const AZStd::string& task) AZ::StringFunc::Path::Normalize(inputFilePath); AZ::StringFunc::Path::Normalize(outputFilePath); - if (task == s_taskRegisterBuilder) - { - return HandleRegisterBuilder(inputFilePath, outputFilePath); - } - else if (task == s_taskCreateJob) + if (task == s_taskCreateJob) { auto func = [this](const AssetBuilderSDK::CreateJobsRequest& request, AssetBuilderSDK::CreateJobsResponse& response) { @@ -896,7 +931,7 @@ void AssetBuilderComponent::JobThread() { case JobType::Create: { - using namespace AssetBuilderSDK; + using namespace AssetBuilder; auto* netRequest = azrtti_cast(job->m_netRequest.get()); auto* netResponse = azrtti_cast(job->m_netResponse.get()); @@ -922,7 +957,7 @@ void AssetBuilderComponent::JobThread() } case JobType::Process: { - using namespace AssetBuilderSDK; + using namespace AssetBuilder; AZ_TracePrintf("AssetBuilder", "Running processJob task\n"); @@ -981,14 +1016,14 @@ void AssetBuilderComponent::JobThread() void AssetBuilderComponent::CreateJobsResidentHandler(AZ::u32 /*typeId*/, AZ::u32 serial, const void* data, AZ::u32 dataLength) { - using namespace AssetBuilderSDK; + using namespace AssetBuilder; ResidentJobHandler(serial, data, dataLength, JobType::Create); } void AssetBuilderComponent::ProcessJobResidentHandler(AZ::u32 /*typeId*/, AZ::u32 serial, const void* data, AZ::u32 dataLength) { - using namespace AssetBuilderSDK; + using namespace AssetBuilder; ResidentJobHandler(serial, data, dataLength, JobType::Process); } @@ -1018,18 +1053,6 @@ bool AssetBuilderComponent::HandleTask(const AZStd::string& inputFilePath, const return true; } -bool AssetBuilderComponent::HandleRegisterBuilder(const AZStd::string& /*inputFilePath*/, const AZStd::string& outputFilePath) const -{ - AssetBuilderSDK::RegisterBuilderResponse response; - - for (const auto& pair : m_assetBuilderDescMap) - { - response.m_assetBuilderDescList.push_back(*pair.second); - } - - return AZ::Utils::SaveObjectToFile(outputFilePath, AZ::DataStream::ST_XML, &response); -} - void AssetBuilderComponent::UpdateResultCode(const AssetBuilderSDK::ProcessJobRequest& request, AssetBuilderSDK::ProcessJobResponse& response) const { if (request.m_jobDescription.m_failOnError) diff --git a/Code/Tools/AssetProcessor/AssetBuilder/AssetBuilderComponent.h b/Code/Tools/AssetProcessor/AssetBuilder/AssetBuilderComponent.h index 2014be41aa..2172e9093e 100644 --- a/Code/Tools/AssetProcessor/AssetBuilder/AssetBuilderComponent.h +++ b/Code/Tools/AssetProcessor/AssetBuilder/AssetBuilderComponent.h @@ -46,6 +46,7 @@ class AssetBuilderComponent public: AZ_COMPONENT(AssetBuilderComponent, "{04332899-5d73-4d41-86b7-b1017d349673}") static void Reflect(AZ::ReflectContext* context); + bool DoHelloPing(); AssetBuilderComponent() = default; ~AssetBuilderComponent() override = default; @@ -64,7 +65,7 @@ public: void RegisterBuilderInformation(const AssetBuilderSDK::AssetBuilderDesc& builderDesc) override; void RegisterComponentDescriptor(AZ::ComponentDescriptor* descriptor) override; - + //EngineConnectionEvents Handler void Disconnected(AzFramework::SocketConnection* connection) override; @@ -98,12 +99,13 @@ protected: static const char* GetLibraryExtension(); bool ConnectToAssetProcessor(); + bool SendRegisteredBuildersToAp(); bool LoadBuilders(const AZStd::string& builderFolder); bool LoadBuilder(const AZStd::string& filePath); void UnloadBuilders(); //! Hooks up net job request handling and keeps the AssetBuilder running indefinitely - bool RunInResidentMode(); + bool RunInResidentMode(bool sendRegistration); bool RunDebugTask(AZStd::string&& debugFile, bool runCreateJobs, bool runProcessJob); bool RunOneShotTask(const AZStd::string& task); @@ -120,9 +122,6 @@ protected: void ProcessJob(const AssetBuilderSDK::ProcessJobFunction& job, const AssetBuilderSDK::ProcessJobRequest& request, AssetBuilderSDK::ProcessJobResponse& outResponse); - //! Handles a builder registration request - bool HandleRegisterBuilder(const AZStd::string& inputFilePath, const AZStd::string& outputFilePath) const; - //! If needed looks at collected data and updates the result code from the job accordingly. void UpdateResultCode(const AssetBuilderSDK::ProcessJobRequest& request, AssetBuilderSDK::ProcessJobResponse& response) const; @@ -141,7 +140,7 @@ protected: //! Currently loading builder AssetBuilder::ExternalModuleAssetBuilderInfo* m_currentAssetBuilder = nullptr; - + //! Thread for running a job, so we don't block the network thread while doing work AZStd::thread_desc m_jobThreadDesc; AZStd::thread m_jobThread; @@ -153,7 +152,7 @@ protected: AZStd::binary_semaphore m_mainEvent; //! Use to signal a new job is ready to be processed AZStd::binary_semaphore m_jobEvent; - + //! Lock for m_queuedJob AZStd::mutex m_jobMutex; diff --git a/Code/Tools/AssetProcessor/AssetBuilder/AssetBuilderStatic.cpp b/Code/Tools/AssetProcessor/AssetBuilder/AssetBuilderStatic.cpp new file mode 100644 index 0000000000..6a67fbe425 --- /dev/null +++ b/Code/Tools/AssetProcessor/AssetBuilder/AssetBuilderStatic.cpp @@ -0,0 +1,182 @@ +/* + * Copyright (c) Contributors to the Open 3D Engine Project. + * For complete copyright and license terms please see the LICENSE at the root of this distribution. + * + * SPDX-License-Identifier: Apache-2.0 OR MIT + * + */ + +#include +#include + +namespace AssetBuilder +{ + void Reflect(AZ::ReflectContext* context) + { + BuilderRegistrationRequest::Reflect(context); + + BuilderHelloRequest::Reflect(context); + BuilderHelloResponse::Reflect(context); + CreateJobsNetRequest::Reflect(context); + CreateJobsNetResponse::Reflect(context); + ProcessJobNetRequest::Reflect(context); + ProcessJobNetResponse::Reflect(context); + } + + void InitializeSerializationContext() + { + AZ::SerializeContext* serializeContext = nullptr; + + AZ::ComponentApplicationBus::BroadcastResult(serializeContext, &AZ::ComponentApplicationBus::Events::GetSerializeContext); + AZ_Assert(serializeContext, "Unable to retrieve serialize context."); + + Reflect(serializeContext); + } + + void BuilderHelloRequest::Reflect(AZ::ReflectContext* context) + { + auto serialize = azrtti_cast(context); + if (serialize) + { + serialize->Class()->Version(1)->Field("UUID", &BuilderHelloRequest::m_uuid); + } + } + + unsigned int BuilderHelloRequest::MessageType() + { + static unsigned int messageType = AZ_CRC("AssetBuilderSDK::BuilderHelloRequest", 0x213a7248); + + return messageType; + } + + unsigned int BuilderHelloRequest::GetMessageType() const + { + return MessageType(); + } + + void BuilderHelloResponse::Reflect(AZ::ReflectContext* context) + { + auto serialize = azrtti_cast(context); + if (serialize) + { + serialize->Class() + ->Version(1) + ->Field("Accepted", &BuilderHelloResponse::m_accepted) + ->Field("UUID", &BuilderHelloResponse::m_uuid); + } + } + + unsigned int BuilderHelloResponse::GetMessageType() const + { + return BuilderHelloRequest::MessageType(); + } + + ////////////////////////////////////////////////////////////////////////// + + void CreateJobsNetRequest::Reflect(AZ::ReflectContext* context) + { + auto serialize = azrtti_cast(context); + if (serialize) + { + serialize->Class()->Version(1)->Field("Request", &CreateJobsNetRequest::m_request); + } + } + + unsigned int CreateJobsNetRequest::MessageType() + { + static unsigned int messageType = AZ_CRC("AssetBuilderSDK::CreateJobsNetRequest", 0xc48209c0); + + return messageType; + } + + unsigned int CreateJobsNetRequest::GetMessageType() const + { + return MessageType(); + } + + void CreateJobsNetResponse::Reflect(AZ::ReflectContext* context) + { + auto serialize = azrtti_cast(context); + if (serialize) + { + serialize->Class()->Version(1)->Field("Response", &CreateJobsNetResponse::m_response); + } + } + + unsigned int CreateJobsNetResponse::GetMessageType() const + { + return CreateJobsNetRequest::MessageType(); + } + + void ProcessJobNetRequest::Reflect(AZ::ReflectContext* context) + { + auto serialize = azrtti_cast(context); + if (serialize) + { + serialize->Class()->Version(1)->Field("Request", &ProcessJobNetRequest::m_request); + } + } + + unsigned int ProcessJobNetRequest::MessageType() + { + static unsigned int messageType = AZ_CRC("AssetBuilderSDK::ProcessJobNetRequest", 0x479f340f); + + return messageType; + } + + unsigned int ProcessJobNetRequest::GetMessageType() const + { + return MessageType(); + } + + void ProcessJobNetResponse::Reflect(AZ::ReflectContext* context) + { + auto serialize = azrtti_cast(context); + if (serialize) + { + serialize->Class()->Version(1)->Field("Response", &ProcessJobNetResponse::m_response); + } + } + + unsigned int ProcessJobNetResponse::GetMessageType() const + { + return ProcessJobNetRequest::MessageType(); + } + + //--------------------------------------------------------------------- + void BuilderRegistration::Reflect(AZ::ReflectContext* context) + { + auto serialize = azrtti_cast(context); + if (serialize) + { + serialize->Class() + ->Version(1) + ->Field("Name", &BuilderRegistration::m_name) + ->Field("Patterns", &BuilderRegistration::m_patterns) + ->Field("BusId", &BuilderRegistration::m_busId) + ->Field("Version", &BuilderRegistration::m_version) + ->Field("AnalysisFingerprint", &BuilderRegistration::m_analysisFingerprint) + ->Field("Flags", &BuilderRegistration::m_flags) + ->Field("FlagsByJobKey", &BuilderRegistration::m_flagsByJobKey) + ->Field("ProductsToKeepOnFailure", &BuilderRegistration::m_productsToKeepOnFailure); + } + } + + void BuilderRegistrationRequest::Reflect(AZ::ReflectContext* context) + { + BuilderRegistration::Reflect(context); + + auto serialize = azrtti_cast(context); + if (serialize) + { + serialize->Class()->Version(1)->Field( + "Builders", &BuilderRegistrationRequest::m_builders); + } + } + + unsigned int BuilderRegistrationRequest::GetMessageType() const + { + return BuilderRegistrationRequest::MessageType; + } + +} // namespace AssetBuilder diff --git a/Code/Tools/AssetProcessor/AssetBuilder/AssetBuilderStatic.h b/Code/Tools/AssetProcessor/AssetBuilder/AssetBuilderStatic.h new file mode 100644 index 0000000000..41b6531116 --- /dev/null +++ b/Code/Tools/AssetProcessor/AssetBuilder/AssetBuilderStatic.h @@ -0,0 +1,140 @@ +/* + * Copyright (c) Contributors to the Open 3D Engine Project. + * For complete copyright and license terms please see the LICENSE at the root of this distribution. + * + * SPDX-License-Identifier: Apache-2.0 OR MIT + * + */ + +#pragma once + +#include +#include + +namespace AssetBuilder +{ + void Reflect(AZ::ReflectContext* context); + + void InitializeSerializationContext(); + + //! BuilderHelloRequest is sent by an AssetBuilder that is attempting to connect to the AssetProcessor to register itself as a worker + class BuilderHelloRequest : public AzFramework::AssetSystem::BaseAssetProcessorMessage + { + public: + AZ_CLASS_ALLOCATOR(BuilderHelloRequest, AZ::OSAllocator, 0); + AZ_RTTI(BuilderHelloRequest, "{5fab5962-a1d8-42a5-bf7a-fb1a8c5a9588}", BaseAssetProcessorMessage); + + static void Reflect(AZ::ReflectContext* context); + static unsigned int MessageType(); + + unsigned int GetMessageType() const override; + + //! Unique ID assigned to this builder to identify it + AZ::Uuid m_uuid = AZ::Uuid::CreateNull(); + }; + + //! BuilderHelloResponse contains the AssetProcessor's response to a builder connection attempt, indicating if it is accepted and the ID + //! that it was assigned + class BuilderHelloResponse : public AzFramework::AssetSystem::BaseAssetProcessorMessage + { + public: + AZ_CLASS_ALLOCATOR(BuilderHelloResponse, AZ::OSAllocator, 0); + AZ_RTTI(BuilderHelloResponse, "{5f3d7c11-6639-4c6f-980a-32be546903c2}", BaseAssetProcessorMessage); + + static void Reflect(AZ::ReflectContext* context); + + unsigned int GetMessageType() const override; + + //! Indicates if the builder was accepted by the AP + bool m_accepted = false; + + //! Unique ID assigned to the builder. If the builder isn't a local process, this is the ID assigned by the AP + AZ::Uuid m_uuid = AZ::Uuid::CreateNull(); + }; + + class CreateJobsNetRequest : public AzFramework::AssetSystem::BaseAssetProcessorMessage + { + public: + AZ_CLASS_ALLOCATOR(CreateJobsNetRequest, AZ::OSAllocator, 0); + AZ_RTTI(CreateJobsNetRequest, "{97fa717d-3a09-4d21-95c6-b2eafd773f1c}", BaseAssetProcessorMessage); + + static void Reflect(AZ::ReflectContext* context); + static unsigned int MessageType(); + + unsigned int GetMessageType() const override; + + AssetBuilderSDK::CreateJobsRequest m_request; + }; + + class CreateJobsNetResponse : public AzFramework::AssetSystem::BaseAssetProcessorMessage + { + public: + AZ_CLASS_ALLOCATOR(CreateJobsNetResponse, AZ::OSAllocator, 0); + AZ_RTTI(CreateJobsNetResponse, "{b2c7c2d3-b60e-4b27-b699-43e0ba991c33}", BaseAssetProcessorMessage); + + static void Reflect(AZ::ReflectContext* context); + + unsigned int GetMessageType() const override; + + AssetBuilderSDK::CreateJobsResponse m_response; + }; + + class ProcessJobNetRequest : public AzFramework::AssetSystem::BaseAssetProcessorMessage + { + public: + AZ_CLASS_ALLOCATOR(ProcessJobNetRequest, AZ::OSAllocator, 0); + AZ_RTTI(ProcessJobNetRequest, "{05288de1-020b-48db-b9de-715f17284efa}", BaseAssetProcessorMessage); + + static void Reflect(AZ::ReflectContext* context); + static unsigned int MessageType(); + + unsigned int GetMessageType() const override; + + AssetBuilderSDK::ProcessJobRequest m_request; + }; + + class ProcessJobNetResponse : public AzFramework::AssetSystem::BaseAssetProcessorMessage + { + public: + AZ_CLASS_ALLOCATOR(ProcessJobNetResponse, AZ::OSAllocator, 0); + AZ_RTTI(ProcessJobNetResponse, "{26ddf882-246c-4cfb-912f-9b8e389df4f6}", BaseAssetProcessorMessage); + + static void Reflect(AZ::ReflectContext* context); + + unsigned int GetMessageType() const override; + + AssetBuilderSDK::ProcessJobResponse m_response; + }; + + ////////////////////////////////////////////////////////////////////////// + struct BuilderRegistration + { + AZ_CLASS_ALLOCATOR(BuilderRegistration, AZ::OSAllocator, 0); + AZ_TYPE_INFO(BuilderRegistration, "{36E785C3-5046-4568-870A-336C8249E453}"); + + static void Reflect(AZ::ReflectContext* context); + + AZStd::string m_name; + AZStd::vector m_patterns; + AZ::Uuid m_busId; + int m_version = 0; + AZStd::string m_analysisFingerprint; + AZ::u8 m_flags = 0; + AZStd::unordered_map m_flagsByJobKey; + AZStd::unordered_map> m_productsToKeepOnFailure; + }; + + class BuilderRegistrationRequest : public AzFramework::AssetSystem::BaseAssetProcessorMessage + { + public: + AZ_CLASS_ALLOCATOR(BuilderRegistrationRequest, AZ::OSAllocator, 0); + AZ_RTTI(BuilderRegistrationRequest, "{FA9CF2D5-C847-47F3-979D-6C3AE061715C}", BaseAssetProcessorMessage); + static void Reflect(AZ::ReflectContext* context); + static constexpr unsigned int MessageType = AZ_CRC_CE("AssetSystem::BuilderRegistrationRequest"); + + BuilderRegistrationRequest() = default; + unsigned int GetMessageType() const override; + + AZStd::vector m_builders; + }; +} // namespace AssetBuilder diff --git a/Code/Tools/AssetProcessor/AssetBuilder/CMakeLists.txt b/Code/Tools/AssetProcessor/AssetBuilder/CMakeLists.txt index 6baa99d43b..f23e749169 100644 --- a/Code/Tools/AssetProcessor/AssetBuilder/CMakeLists.txt +++ b/Code/Tools/AssetProcessor/AssetBuilder/CMakeLists.txt @@ -6,6 +6,22 @@ # # +ly_add_target( + NAME AssetBuilder.Static STATIC + NAMESPACE AZ + FILES_CMAKE + asset_builder_static_files.cmake + INCLUDE_DIRECTORIES + PUBLIC + . + BUILD_DEPENDENCIES + PUBLIC + AZ::AzCore + AZ::AzFramework + AZ::AzToolsFramework + AZ::AssetBuilderSDK +) + ly_add_target( NAME AssetBuilder EXECUTABLE NAMESPACE AZ @@ -17,6 +33,7 @@ ly_add_target( . BUILD_DEPENDENCIES PRIVATE + AssetBuilder.Static 3rdParty::Qt::Core 3rdParty::Qt::Gui 3rdParty::Qt::Network diff --git a/Code/Tools/AssetProcessor/AssetBuilder/asset_builder_static_files.cmake b/Code/Tools/AssetProcessor/AssetBuilder/asset_builder_static_files.cmake new file mode 100644 index 0000000000..48de742704 --- /dev/null +++ b/Code/Tools/AssetProcessor/AssetBuilder/asset_builder_static_files.cmake @@ -0,0 +1,12 @@ +# +# Copyright (c) Contributors to the Open 3D Engine Project. +# For complete copyright and license terms please see the LICENSE at the root of this distribution. +# +# SPDX-License-Identifier: Apache-2.0 OR MIT +# +# + +set(FILES + AssetBuilderStatic.h + AssetBuilderStatic.cpp +) diff --git a/Code/Tools/AssetProcessor/AssetBuilderSDK/AssetBuilderSDK/AssetBuilderSDK.cpp b/Code/Tools/AssetProcessor/AssetBuilderSDK/AssetBuilderSDK/AssetBuilderSDK.cpp index b5323dd419..43126dd673 100644 --- a/Code/Tools/AssetProcessor/AssetBuilderSDK/AssetBuilderSDK/AssetBuilderSDK.cpp +++ b/Code/Tools/AssetProcessor/AssetBuilderSDK/AssetBuilderSDK/AssetBuilderSDK.cpp @@ -1172,19 +1172,10 @@ namespace AssetBuilderSDK JobProduct::Reflect(context); AssetBuilderDesc::Reflect(context); - RegisterBuilderRequest::Reflect(context); - RegisterBuilderResponse::Reflect(context); CreateJobsRequest::Reflect(context); CreateJobsResponse::Reflect(context); ProcessJobRequest::Reflect(context); ProcessJobResponse::Reflect(context); - - BuilderHelloRequest::Reflect(context); - BuilderHelloResponse::Reflect(context); - CreateJobsNetRequest::Reflect(context); - CreateJobsNetResponse::Reflect(context); - ProcessJobNetRequest::Reflect(context); - ProcessJobNetResponse::Reflect(context); } void InitializeSerializationContext() @@ -1263,24 +1254,6 @@ namespace AssetBuilderSDK } } - void RegisterBuilderRequest::Reflect(AZ::ReflectContext* context) - { - if (AZ::SerializeContext* serializeContext = azrtti_cast(context)) - { - serializeContext->Class()-> - Version(1)-> - Field("FilePath", &RegisterBuilderRequest::m_filePath); - } - - if (AZ::BehaviorContext* behaviorContext = azrtti_cast(context)) - { - behaviorContext->Class("RegisterBuilderRequest") - ->Attribute(AZ::Script::Attributes::Scope, AZ::Script::Attributes::ScopeFlags::Automation) - ->Attribute(AZ::Script::Attributes::Module, "asset.builder") - ->Property("filePath", BehaviorValueProperty(&RegisterBuilderRequest::m_filePath)); - } - } - void AssetBuilderDesc::Reflect(AZ::ReflectContext* context) { if (AZ::SerializeContext* serializeContext = azrtti_cast(context)) @@ -1313,25 +1286,6 @@ namespace AssetBuilderSDK } } - void RegisterBuilderResponse::Reflect(AZ::ReflectContext* context) - { - if (AZ::SerializeContext* serializeContext = azrtti_cast(context)) - { - serializeContext->Class() - ->Version(1) - ->Field("Asset Builder Desc List", &RegisterBuilderResponse::m_assetBuilderDescList); - } - - if (AZ::BehaviorContext* behaviorContext = azrtti_cast(context)) - { - behaviorContext->Class("RegisterBuilderResponse") - ->Attribute(AZ::Script::Attributes::Scope, AZ::Script::Attributes::ScopeFlags::Automation) - ->Attribute(AZ::Script::Attributes::Module, "asset.builder") - ->Constructor() - ->Property("assetBuilderDescList", BehaviorValueProperty(&RegisterBuilderResponse::m_assetBuilderDescList)); - } - } - bool CreateJobsResponse::Succeeded() const { return m_result == CreateJobsResultCode::Success; @@ -1362,128 +1316,6 @@ namespace AssetBuilderSDK } } - void BuilderHelloRequest::Reflect(AZ::ReflectContext* context) - { - auto serialize = azrtti_cast(context); - if (serialize) - { - serialize->Class() - ->Version(1) - ->Field("UUID", &BuilderHelloRequest::m_uuid); - } - } - - unsigned int BuilderHelloRequest::MessageType() - { - static unsigned int messageType = AZ_CRC("AssetBuilderSDK::BuilderHelloRequest", 0x213a7248); - - return messageType; - } - - unsigned int BuilderHelloRequest::GetMessageType() const - { - return MessageType(); - } - - void BuilderHelloResponse::Reflect(AZ::ReflectContext* context) - { - auto serialize = azrtti_cast(context); - if (serialize) - { - serialize->Class() - ->Version(1) - ->Field("Accepted", &BuilderHelloResponse::m_accepted) - ->Field("UUID", &BuilderHelloResponse::m_uuid); - } - } - - unsigned int BuilderHelloResponse::GetMessageType() const - { - return BuilderHelloRequest::MessageType(); - } - - ////////////////////////////////////////////////////////////////////////// - - void CreateJobsNetRequest::Reflect(AZ::ReflectContext* context) - { - auto serialize = azrtti_cast(context); - if (serialize) - { - serialize->Class() - ->Version(1) - ->Field("Request", &CreateJobsNetRequest::m_request); - } - } - - unsigned int CreateJobsNetRequest::MessageType() - { - static unsigned int messageType = AZ_CRC("AssetBuilderSDK::CreateJobsNetRequest", 0xc48209c0); - - return messageType; - } - - unsigned int CreateJobsNetRequest::GetMessageType() const - { - return MessageType(); - } - - void CreateJobsNetResponse::Reflect(AZ::ReflectContext* context) - { - auto serialize = azrtti_cast(context); - if (serialize) - { - serialize->Class() - ->Version(1) - ->Field("Response", &CreateJobsNetResponse::m_response); - } - } - - unsigned int CreateJobsNetResponse::GetMessageType() const - { - return CreateJobsNetRequest::MessageType(); - } - - - - void ProcessJobNetRequest::Reflect(AZ::ReflectContext* context) - { - auto serialize = azrtti_cast(context); - if (serialize) - { - serialize->Class() - ->Version(1) - ->Field("Request", &ProcessJobNetRequest::m_request); - } - } - - unsigned int ProcessJobNetRequest::MessageType() - { - static unsigned int messageType = AZ_CRC("AssetBuilderSDK::ProcessJobNetRequest", 0x479f340f); - - return messageType; - } - - unsigned int ProcessJobNetRequest::GetMessageType() const - { - return MessageType(); - } - - void ProcessJobNetResponse::Reflect(AZ::ReflectContext* context) - { - auto serialize = azrtti_cast(context); - if (serialize) - { - serialize->Class() - ->Version(1) - ->Field("Response", &ProcessJobNetResponse::m_response); - } - } - - unsigned int ProcessJobNetResponse::GetMessageType() const - { - return ProcessJobNetRequest::MessageType(); - } - JobDependency::JobDependency(const AZStd::string& jobKey, const AZStd::string& platformIdentifier, const JobDependencyType& type, const SourceFileDependency& sourceFile) : m_jobKey(jobKey) , m_platformIdentifier(platformIdentifier) diff --git a/Code/Tools/AssetProcessor/AssetBuilderSDK/AssetBuilderSDK/AssetBuilderSDK.h b/Code/Tools/AssetProcessor/AssetBuilderSDK/AssetBuilderSDK/AssetBuilderSDK.h index f977e15be9..65bf9cc7d4 100644 --- a/Code/Tools/AssetProcessor/AssetBuilderSDK/AssetBuilderSDK/AssetBuilderSDK.h +++ b/Code/Tools/AssetProcessor/AssetBuilderSDK/AssetBuilderSDK/AssetBuilderSDK.h @@ -464,35 +464,6 @@ namespace AssetBuilderSDK AZStd::string m_platformIdentifier; }; - //! RegisterBuilderRequest contains input data that will be sent by the AssetProcessor to the builder during the startup registration phase - struct RegisterBuilderRequest - { - AZ_CLASS_ALLOCATOR(RegisterBuilderRequest, AZ::SystemAllocator, 0); - AZ_TYPE_INFO(RegisterBuilderRequest, "{7C6C5198-4766-42B8-9A1E-48479CE2F5EA}"); - - AZStd::string m_filePath; - - RegisterBuilderRequest() {} - - explicit RegisterBuilderRequest(const AZStd::string& filePath) - : m_filePath(filePath) - { - } - - static void Reflect(AZ::ReflectContext* context); - }; - - //! INTERNAL USE ONLY - RegisterBuilderResponse contains registration data that will be sent by the builder to the AssetProcessor in response to RegisterBuilderRequest - struct RegisterBuilderResponse - { - AZ_CLASS_ALLOCATOR(RegisterBuilderResponse, AZ::SystemAllocator, 0); - AZ_TYPE_INFO(RegisterBuilderResponse, "{0AE5583F-C763-410E-BA7F-78BD90546C01}"); - - AZStd::vector m_assetBuilderDescList; - - static void Reflect(AZ::ReflectContext* context); - }; - /** * This tells you about a platform in your CreateJobsRequest or your ProcessJobRequest */ @@ -756,99 +727,9 @@ namespace AssetBuilderSDK static void Reflect(AZ::ReflectContext* context); }; - //! BuilderHelloRequest is sent by an AssetBuilder that is attempting to connect to the AssetProcessor to register itself as a worker - class BuilderHelloRequest : public AzFramework::AssetSystem::BaseAssetProcessorMessage - { - public: + - AZ_CLASS_ALLOCATOR(BuilderHelloRequest, AZ::OSAllocator, 0); - AZ_RTTI(BuilderHelloRequest, "{5fab5962-a1d8-42a5-bf7a-fb1a8c5a9588}", BaseAssetProcessorMessage); - - static void Reflect(AZ::ReflectContext* context); - static unsigned int MessageType(); - - unsigned int GetMessageType() const override; - - //! Unique ID assigned to this builder to identify it - AZ::Uuid m_uuid = AZ::Uuid::CreateNull(); - }; - - //! BuilderHelloResponse contains the AssetProcessor's response to a builder connection attempt, indicating if it is accepted and the ID that it was assigned - class BuilderHelloResponse : public AzFramework::AssetSystem::BaseAssetProcessorMessage - { - public: - - AZ_CLASS_ALLOCATOR(BuilderHelloResponse, AZ::OSAllocator, 0); - AZ_RTTI(BuilderHelloResponse, "{5f3d7c11-6639-4c6f-980a-32be546903c2}", BaseAssetProcessorMessage); - - static void Reflect(AZ::ReflectContext* context); - - unsigned int GetMessageType() const override; - - //! Indicates if the builder was accepted by the AP - bool m_accepted = false; - - //! Unique ID assigned to the builder. If the builder isn't a local process, this is the ID assigned by the AP - AZ::Uuid m_uuid = AZ::Uuid::CreateNull(); - }; - - class CreateJobsNetRequest : public AzFramework::AssetSystem::BaseAssetProcessorMessage - { - public: - - AZ_CLASS_ALLOCATOR(CreateJobsNetRequest, AZ::OSAllocator, 0); - AZ_RTTI(CreateJobsNetRequest, "{97fa717d-3a09-4d21-95c6-b2eafd773f1c}", BaseAssetProcessorMessage); - - static void Reflect(AZ::ReflectContext* context); - static unsigned int MessageType(); - - unsigned int GetMessageType() const override; - - CreateJobsRequest m_request; - }; - - class CreateJobsNetResponse : public AzFramework::AssetSystem::BaseAssetProcessorMessage - { - public: - - AZ_CLASS_ALLOCATOR(CreateJobsNetResponse, AZ::OSAllocator, 0); - AZ_RTTI(CreateJobsNetResponse, "{b2c7c2d3-b60e-4b27-b699-43e0ba991c33}", BaseAssetProcessorMessage); - - static void Reflect(AZ::ReflectContext* context); - - unsigned int GetMessageType() const override; - - CreateJobsResponse m_response; - }; - - class ProcessJobNetRequest : public AzFramework::AssetSystem::BaseAssetProcessorMessage - { - public: - - AZ_CLASS_ALLOCATOR(ProcessJobNetRequest, AZ::OSAllocator, 0); - AZ_RTTI(ProcessJobNetRequest, "{05288de1-020b-48db-b9de-715f17284efa}", BaseAssetProcessorMessage); - - static void Reflect(AZ::ReflectContext* context); - static unsigned int MessageType(); - - unsigned int GetMessageType() const override; - - ProcessJobRequest m_request; - }; - - class ProcessJobNetResponse : public AzFramework::AssetSystem::BaseAssetProcessorMessage - { - public: - - AZ_CLASS_ALLOCATOR(ProcessJobNetResponse, AZ::OSAllocator, 0); - AZ_RTTI(ProcessJobNetResponse, "{26ddf882-246c-4cfb-912f-9b8e389df4f6}", BaseAssetProcessorMessage); - - static void Reflect(AZ::ReflectContext* context); - - unsigned int GetMessageType() const override; - - ProcessJobResponse m_response; - }; + //! JobCancelListener can be used by builders in their processJob method to listen for job cancellation request. //! The address of this listener is the jobid which can be found in the process job request. @@ -935,44 +816,3 @@ namespace AZ AZ_TYPE_INFO_SPECIALIZE(AssetBuilderSDK::ProductPathDependencyType, "{EF77742B-9627-4072-B431-396AA7183C80}"); AZ_TYPE_INFO_SPECIALIZE(AssetBuilderSDK::SourceFileDependency::SourceFileDependencyType, "{BE9C8805-DB17-4500-944A-EB33FD0BE347}"); } - -//! This macro should be used by every AssetBuilder to register itself, -//! AssetProcessor uses these exported function to identify whether a dll is an Asset Builder or not -//! If you want something highly custom you can do these entry points yourself instead of using the macro. -#define REGISTER_ASSETBUILDER \ - extern void BuilderOnInit(); \ - extern void BuilderDestroy(); \ - extern void BuilderRegisterDescriptors(); \ - extern void BuilderAddComponents(AZ::Entity * entity); \ - extern "C" \ - { \ - AZ_DLL_EXPORT int IsAssetBuilder() \ - { \ - return 0; \ - } \ - \ - AZ_DLL_EXPORT void InitializeModule(AZ::EnvironmentInstance sharedEnvironment) \ - { \ - AZ::Environment::Attach(sharedEnvironment); \ - BuilderOnInit(); \ - } \ - \ - AZ_DLL_EXPORT void UninitializeModule() \ - { \ - BuilderDestroy(); \ - AZ::Environment::Detach(); \ - } \ - \ - AZ_DLL_EXPORT void ModuleRegisterDescriptors() \ - { \ - BuilderRegisterDescriptors(); \ - } \ - \ - AZ_DLL_EXPORT void ModuleAddComponents(AZ::Entity * entity) \ - { \ - BuilderAddComponents(entity); \ - } \ - } -// confusion-reducing note: above end-brace is part of the macro, not a namespace - - diff --git a/Code/Tools/AssetProcessor/CMakeLists.txt b/Code/Tools/AssetProcessor/CMakeLists.txt index 98c5f8e5e8..a47ced43be 100644 --- a/Code/Tools/AssetProcessor/CMakeLists.txt +++ b/Code/Tools/AssetProcessor/CMakeLists.txt @@ -42,6 +42,7 @@ ly_add_target( AZ::AzQtComponents AZ::AzToolsFramework AZ::AssetBuilderSDK + AZ::AssetBuilder.Static ${additional_dependencies} RUNTIME_DEPENDENCIES AZ::AssetBuilder diff --git a/Code/Tools/AssetProcessor/native/assetprocessor.h b/Code/Tools/AssetProcessor/native/assetprocessor.h index 1c13eca200..83eb0464a5 100644 --- a/Code/Tools/AssetProcessor/native/assetprocessor.h +++ b/Code/Tools/AssetProcessor/native/assetprocessor.h @@ -85,7 +85,7 @@ namespace AssetProcessor enum AssetCatalogStatus { - RequiresSaving, + RequiresSaving, UpToDate }; @@ -213,11 +213,11 @@ namespace AssetProcessor bool m_critical = false; int m_priority = -1; - // indicates whether we need to check the server first for the outputs of this job + // indicates whether we need to check the server first for the outputs of this job // before we start processing locally bool m_checkServer = false; - - // Indicates whether this job needs to be processed irrespective of whether its fingerprint got modified or not. + + // Indicates whether this job needs to be processed irrespective of whether its fingerprint got modified or not. bool m_autoProcessJob = false; AssetBuilderSDK::AssetBuilderDesc m_assetBuilderDesc; @@ -251,9 +251,9 @@ namespace AssetProcessor JobDetails() = default; }; - - //! JobDesc struct is used for identifying jobs that need to be processed again - //! because of job dependency declared on them by other jobs + + //! JobDesc struct is used for identifying jobs that need to be processed again + //! because of job dependency declared on them by other jobs struct JobDesc { AZStd::string m_databaseSourceName; @@ -283,7 +283,7 @@ namespace AssetProcessor } }; - //! JobIndentifier is an internal structure that store all the data that can uniquely identify a job + //! JobIndentifier is an internal structure that store all the data that can uniquely identify a job struct JobIndentifier { JobDesc m_jobDesc; diff --git a/Code/Tools/AssetProcessor/native/ui/MainWindow.cpp b/Code/Tools/AssetProcessor/native/ui/MainWindow.cpp index c3674e6431..0837384d95 100644 --- a/Code/Tools/AssetProcessor/native/ui/MainWindow.cpp +++ b/Code/Tools/AssetProcessor/native/ui/MainWindow.cpp @@ -165,7 +165,7 @@ void MainWindow::Activate() ui->connectionTreeView->header()->resizeSection(ConnectionManager::PortColumn, 60); ui->connectionTreeView->header()->resizeSection(ConnectionManager::PlatformColumn, 60); ui->connectionTreeView->header()->resizeSection(ConnectionManager::AutoConnectColumn, 60); - + ui->connectionTreeView->header()->setStretchLastSection(false); connect(ui->connectionTreeView->selectionModel(), &QItemSelectionModel::selectionChanged, this, &MainWindow::OnConnectionSelectionChanged); @@ -189,12 +189,12 @@ void MainWindow::Activate() ui->allowListAllowedListConnectionsListView->setModel(&m_allowedListAddresses); connect(ui->allowedListRejectedConnectionsListView, &QListView::clicked, this, &MainWindow::OnRejectedConnectionsListViewClicked); ui->allowedListRejectedConnectionsListView->setModel(&m_rejectedAddresses); - + connect(ui->allowedListEnableCheckBox, &QCheckBox::toggled, this, &MainWindow::OnAllowedListCheckBoxToggled); - + connect(ui->allowedListAddHostNameToolButton, &QToolButton::clicked, this, &MainWindow::OnAddHostNameAllowedListButtonClicked); connect(ui->allowedListAddIPToolButton, &QPushButton::clicked, this, &MainWindow::OnAddIPAllowedListButtonClicked); - + connect(ui->allowedListToAllowedListToolButton, &QPushButton::clicked, this, &MainWindow::OnToAllowedListButtonClicked); connect(ui->allowedListToRejectedListToolButton, &QToolButton::clicked, this, &MainWindow::OnToRejectedListButtonClicked); @@ -204,7 +204,7 @@ void MainWindow::Activate() QRegExpValidator* hostNameValidator = new QRegExpValidator(validHostName, this); ui->allowedListAddHostNameLineEdit->setValidator(hostNameValidator); - + QRegExpValidator* ipValidator = new QRegExpValidator(validIP, this); ui->allowedListAddIPLineEdit->setValidator(ipValidator); @@ -235,7 +235,7 @@ void MainWindow::Activate() m_logSortFilterProxy->setSourceModel(m_logsModel); m_logSortFilterProxy->setFilterKeyColumn(AzToolsFramework::Logging::LogTableModel::ColumnMessage); m_logSortFilterProxy->setFilterCaseSensitivity(Qt::CaseInsensitive); - + ui->jobLogTableView->setModel(m_logSortFilterProxy); ui->jobLogTableView->setItemDelegate(new AzToolsFramework::Logging::LogTableItemDelegate(ui->jobLogTableView)); ui->jobLogTableView->setExpandOnSelection(); @@ -400,7 +400,7 @@ void MainWindow::Activate() bool zeroAnalysisModeFromSettings = settings.value("EnableZeroAnalysis", QVariant(true)).toBool(); settings.endGroup(); - QObject::connect(ui->modtimeSkippingCheckBox, &QCheckBox::stateChanged, this, + QObject::connect(ui->modtimeSkippingCheckBox, &QCheckBox::stateChanged, this, [this](int newCheckState) { bool newOption = newCheckState == Qt::Checked ? true : false; @@ -543,7 +543,7 @@ void MainWindow::OnAddConnection(bool /*checked*/) m_guiApplicationManager->GetConnectionManager()->addUserConnection(); } -void MainWindow::OnAllowedListConnectionsListViewClicked() +void MainWindow::OnAllowedListConnectionsListViewClicked() { ui->allowedListRejectedConnectionsListView->clearSelection(); } @@ -553,7 +553,7 @@ void MainWindow::OnRejectedConnectionsListViewClicked() ui->allowListAllowedListConnectionsListView->clearSelection(); } -void MainWindow::OnAllowedListCheckBoxToggled() +void MainWindow::OnAllowedListCheckBoxToggled() { if (!ui->allowedListEnableCheckBox->isChecked()) { @@ -588,7 +588,7 @@ void MainWindow::OnAllowedListCheckBoxToggled() ui->allowedListToAllowedListToolButton->setEnabled(true); ui->allowedListToRejectedListToolButton->setEnabled(true); } - + m_guiApplicationManager->GetConnectionManager()->AllowedListingEnabled(ui->allowedListEnableCheckBox->isChecked()); } @@ -858,7 +858,7 @@ void MainWindow::OnAssetProcessorStatusChanged(const AssetProcessor::AssetProces text = tr("Working, analyzing jobs remaining %1, processing jobs remaining %2...").arg(m_createJobCount).arg(m_processJobsCount); ui->timerContainerWidget->setVisible(false); ui->productAssetDetailsPanel->SetScanQueueEnabled(false); - + IntervalAssetTabFilterRefresh(); } else @@ -877,7 +877,7 @@ void MainWindow::OnAssetProcessorStatusChanged(const AssetProcessor::AssetProces break; case AssetProcessorStatus::Processing_Jobs: CheckStartProcessTimers(); - m_processJobsCount = entry.m_count; + m_processJobsCount = entry.m_count; if (m_processJobsCount + m_createJobCount > 0) { @@ -983,7 +983,7 @@ void MainWindow::ApplyConfig() ui->jobLogTableView->header()->resizeSection(AzToolsFramework::Logging::LogTableModel::ColumnType, m_config.logTypeColumnWidth); } -MainWindow::LogSortFilterProxy::LogSortFilterProxy(QObject* parentOjbect) : QSortFilterProxyModel(parentOjbect) +MainWindow::LogSortFilterProxy::LogSortFilterProxy(QObject* parentOjbect) : QSortFilterProxyModel(parentOjbect) { } @@ -1302,7 +1302,7 @@ void MainWindow::ShowJobViewContextMenu(const QPoint& pos) ui->sourceAssetDetailsPanel->GoToSource(item->m_elementId.GetInputAssetName().toUtf8().constData()); }); - QString productMenuTitle(tr("View product asset...")); + QString productMenuTitle(tr("View product asset...")); if (item->m_jobState != AzToolsFramework::AssetSystem::JobStatus::Completed) { QString disabledActionTooltip(tr("Only completed jobs are available in the Assets tab.")); @@ -1610,7 +1610,7 @@ void MainWindow::ShowProductAssetContextMenu(const QPoint& pos) { AzQtComponents::ShowFileOnDesktop(pathToProduct.GetValue()); } - + }); QString fileOrFolder(cachedAsset->getChildCount() > 0 ? tr("folder") : tr("file")); diff --git a/Code/Tools/AssetProcessor/native/utilities/ApplicationManager.cpp b/Code/Tools/AssetProcessor/native/utilities/ApplicationManager.cpp index 7a14f158d6..57884b304f 100644 --- a/Code/Tools/AssetProcessor/native/utilities/ApplicationManager.cpp +++ b/Code/Tools/AssetProcessor/native/utilities/ApplicationManager.cpp @@ -231,44 +231,6 @@ bool ApplicationManager::InitiatedShutdown() const return m_duringShutdown; } -void ApplicationManager::GetExternalBuilderFileList(QStringList& externalBuilderModules) -{ - externalBuilderModules.clear(); - - static const char* builder_folder_name = "Builders"; - - // LY_ASSET_BUILDERS is defined by the CMakeLists.txt. The asset builders add themselves to a variable that - // is populated to allow selective building of those asset builder targets. - // This allows left over Asset builders in the output directory to not be loaded by the AssetProcessor -#if !defined(LY_ASSET_BUILDERS) - #error LY_ASSET_BUILDERS was not defined for ApplicationManager.cpp -#endif - - QDir builderDir = QDir::toNativeSeparators(QString(this->m_frameworkApp.GetExecutableFolder())); - builderDir.cd(QString(builder_folder_name)); - if (builderDir.exists()) - { - AZStd::vector tokens; - AZ::StringFunc::Tokenize(AZStd::string_view(LY_ASSET_BUILDERS), tokens, ','); - AZStd::string builderLibrary; - for (const AZStd::string& token : tokens) - { - QString assetBuilderPath(token.c_str()); - if (builderDir.exists(assetBuilderPath)) - { - externalBuilderModules.push_back(builderDir.absoluteFilePath(assetBuilderPath)); - } - } - } - - if (externalBuilderModules.empty()) - { - AZ_TracePrintf(AssetProcessor::ConsoleChannel, "AssetProcessor was unable to locate any external builders\n"); - } -} - - - QDir ApplicationManager::GetSystemRoot() const { return m_systemRoot; @@ -459,15 +421,6 @@ void ApplicationManager::PopulateApplicationDependencies() m_filesOfInterest.push_back(dir.absoluteFilePath(pathWithPlatformExtension)); } - // Get the external builder modules to add to the files of interest - QStringList builderModuleFileList; - GetExternalBuilderFileList(builderModuleFileList); - for (const QString& builderModuleFile : builderModuleFileList) - { - m_filesOfInterest.push_back(builderModuleFile); - } - - QDir assetRoot; AssetUtilities::ComputeAssetRoot(assetRoot); diff --git a/Code/Tools/AssetProcessor/native/utilities/ApplicationManager.h b/Code/Tools/AssetProcessor/native/utilities/ApplicationManager.h index 91cf2185b7..56d65d6784 100644 --- a/Code/Tools/AssetProcessor/native/utilities/ApplicationManager.h +++ b/Code/Tools/AssetProcessor/native/utilities/ApplicationManager.h @@ -139,7 +139,7 @@ protected: void RegisterObjectForQuit(QObject* source, bool insertInFront = false); bool NeedRestart() const; void addRunningThread(AssetProcessor::ThreadWorker* thread); - + template void RegisterInternalBuilder(const QString& builderName); @@ -151,9 +151,6 @@ protected: bool m_duringStartup = true; AssetProcessorAZApplication m_frameworkApp; QCoreApplication* m_qApp = nullptr; - - //! Get the list of external builder files for this asset processor - void GetExternalBuilderFileList(QStringList& externalBuilderModules); virtual void Reflect() = 0; virtual const char* GetLogBaseName() = 0; diff --git a/Code/Tools/AssetProcessor/native/utilities/ApplicationManagerBase.cpp b/Code/Tools/AssetProcessor/native/utilities/ApplicationManagerBase.cpp index 8837e9dc3f..04036602db 100644 --- a/Code/Tools/AssetProcessor/native/utilities/ApplicationManagerBase.cpp +++ b/Code/Tools/AssetProcessor/native/utilities/ApplicationManagerBase.cpp @@ -25,6 +25,7 @@ #include #include #include +#include #include @@ -32,9 +33,6 @@ #include -//! Amount of time to wait between checking the status of the AssetBuilder process -static const int s_MaximumSleepTimeMS = 10; - //! CreateJobs will wait up to 2 minutes before timing out //! This shouldn't need to be so high but very large slices can take a while to process currently //! This should be reduced down to something more reasonable after slice jobs are sped up @@ -64,6 +62,7 @@ ApplicationManagerBase::~ApplicationManagerBase() AZ::Debug::TraceMessageBus::Handler::BusDisconnect(); AssetProcessor::AssetBuilderRegistrationBus::Handler::BusDisconnect(); AssetBuilderSDK::AssetBuilderBus::Handler::BusDisconnect(); + AssetProcessor::AssetBuilderInfoBus::Handler::BusDisconnect(); if (m_settingsRegistryBuilder) { @@ -192,7 +191,7 @@ void ApplicationManagerBase::InitAssetProcessorManager() { m_assetProcessorManager->SetEnableModtimeSkippingFeature(true); } - + if (commandLine->HasSwitch(Command_enableQueryLogging.m_switch)) { m_assetProcessorManager->SetQueryLogging(true); @@ -206,7 +205,7 @@ void ApplicationManagerBase::InitAssetProcessorManager() { m_dependencyScanPattern = commandLine->GetSwitchValue(Command_dsp.m_switch, 0).c_str(); } - + m_fileDependencyScanPattern = "*"; if (commandLine->HasSwitch(Command_fileDependencyScanPattern.m_switch)) @@ -327,7 +326,7 @@ void ApplicationManagerBase::InitAssetCatalog() AssetProcessor::AssetCatalog* catalog = new AssetCatalog(assetCatalogHelper, m_platformConfiguration); // Using a direct connection so we know the catalog has been updated before continuing on with code might depend on the asset being in the catalog - connect(m_assetProcessorManager, &AssetProcessorManager::AssetMessage, catalog, &AssetCatalog::OnAssetMessage, Qt::DirectConnection); + connect(m_assetProcessorManager, &AssetProcessorManager::AssetMessage, catalog, &AssetCatalog::OnAssetMessage, Qt::DirectConnection); connect(m_assetProcessorManager, &AssetProcessorManager::SourceQueued, catalog, &AssetCatalog::OnSourceQueued); connect(m_assetProcessorManager, &AssetProcessorManager::SourceFinished, catalog, &AssetCatalog::OnSourceFinished); connect(m_assetProcessorManager, &AssetProcessorManager::PathDependencyResolved, catalog, &AssetCatalog::OnDependencyResolved); @@ -379,12 +378,12 @@ void ApplicationManagerBase::InitAssetScanner() QObject::connect(m_assetScanner, &AssetScanner::FilesFound, [this](QSet files) { m_fileStateCache->AddInfoSet(files); }); QObject::connect(m_assetScanner, &AssetScanner::FoldersFound, [this](QSet files) { m_fileStateCache->AddInfoSet(files); }); QObject::connect(m_assetScanner, &AssetScanner::ExcludedFound, [this](QSet files) { m_fileStateCache->AddInfoSet(files); }); - + // file table QObject::connect(m_assetScanner, &AssetScanner::AssetScanningStatusChanged, m_fileProcessor.get(), &FileProcessor::OnAssetScannerStatusChange); QObject::connect(m_assetScanner, &AssetScanner::FilesFound, m_fileProcessor.get(), &FileProcessor::AssessFilesFromScanner); QObject::connect(m_assetScanner, &AssetScanner::FoldersFound, m_fileProcessor.get(), &FileProcessor::AssessFoldersFromScanner); - + } void ApplicationManagerBase::DestroyAssetScanner() @@ -591,6 +590,51 @@ void ApplicationManagerBase::InitConnectionManager() }, AZStd::placeholders::_1, AZStd::placeholders::_2, AZStd::placeholders::_3, AZStd::placeholders::_4) ); + m_connectionManager->RegisterService( + AssetBuilder::BuilderRegistrationRequest::MessageType, + [this](unsigned int /*connId*/, unsigned int /*type*/, unsigned int /*serial*/, QByteArray payload, QString) + { + AssetBuilder::BuilderRegistrationRequest registrationRequest; + + if (m_builderRegistrationComplete) + { + return; + } + + m_builderRegistrationComplete = true; + + if (AssetProcessor::UnpackMessage(payload, registrationRequest)) + { + for (const auto& builder : registrationRequest.m_builders) + { + AssetBuilderSDK::AssetBuilderDesc desc; + desc.m_name = builder.m_name; + desc.m_patterns = builder.m_patterns; + desc.m_version = builder.m_version; + desc.m_analysisFingerprint = builder.m_analysisFingerprint; + desc.m_flags = builder.m_flags; + desc.m_busId = builder.m_busId; + desc.m_flagsByJobKey = builder.m_flagsByJobKey; + desc.m_productsToKeepOnFailure = builder.m_productsToKeepOnFailure; + + // Builders registered this way are always external builders + desc.m_builderType = AssetBuilderSDK::AssetBuilderDesc::AssetBuilderType::External; + + RegisterBuilderInformation(desc); + } + + QTimer::singleShot( + 0, this, + [this]() + { + if (!PostActivate()) + { + QuitRequested(); + } + }); + } + }); + //You can get Asset Processor Current State using AzFramework::AssetSystem::RequestAssetProcessorStatus; auto GetState = [this](unsigned int connId, unsigned int, unsigned int serial, QByteArray payload, QString) @@ -633,11 +677,11 @@ void ApplicationManagerBase::InitConnectionManager() AssetProcessorPlatformStatusRequest requestMessage; if (AssetProcessor::UnpackMessage(payload, requestMessage)) { - AzToolsFramework::AssetSystemRequestBus::BroadcastResult(responseMessage.m_isPlatformEnabled, + AzToolsFramework::AssetSystemRequestBus::BroadcastResult(responseMessage.m_isPlatformEnabled, &AzToolsFramework::AssetSystemRequestBus::Events::IsAssetPlatformEnabled, requestMessage.m_platform.c_str()); } - AssetProcessor::ConnectionBus::Event(connId, + AssetProcessor::ConnectionBus::Event(connId, &AssetProcessor::ConnectionBus::Events::SendResponse, serial, responseMessage); }); @@ -653,11 +697,11 @@ void ApplicationManagerBase::InitConnectionManager() if (AssetProcessor::UnpackMessage(payload, requestMessage)) { const char* platformIdentifier = requestMessage.m_platform.c_str(); - responseMessage.m_numberOfPendingJobs = + responseMessage.m_numberOfPendingJobs = GetRCController()->NumberOfPendingJobsPerPlatform(platformIdentifier); } - AssetProcessor::ConnectionBus::Event(connId, + AssetProcessor::ConnectionBus::Event(connId, &AssetProcessor::ConnectionBus::Events::SendResponse, serial, responseMessage); }); } @@ -696,7 +740,7 @@ void ApplicationManagerBase::InitAssetRequestHandler(AssetProcessor::AssetReques QObject::connect(GetAssetProcessorManager(), &AssetProcessorManager::SendAssetExistsResponse, m_assetRequestHandler, &AssetRequestHandler::OnRequestAssetExistsResponse); QObject::connect(GetAssetProcessorManager(), &AssetProcessorManager::FenceFileDetected, m_assetRequestHandler, &AssetRequestHandler::OnFenceFileDetected); - + // connect the Asset Request Handler to RC: QObject::connect(m_assetRequestHandler, &AssetRequestHandler::RequestCompileGroup, GetRCController(), &RCController::OnRequestCompileGroup); QObject::connect(m_assetRequestHandler, &AssetRequestHandler::RequestEscalateAssetBySearchTerm, GetRCController(), &RCController::OnEscalateJobsBySearchTerm); @@ -840,14 +884,6 @@ bool ApplicationManagerBase::Run() return false; } - bool startedSuccessfully = true; - - if (!PostActivate()) - { - QuitRequested(); - startedSuccessfully = false; - } - AZ_Printf(AssetProcessor::ConsoleChannel, "Asset Processor Batch Processing Started.\n"); AZ_Printf(AssetProcessor::ConsoleChannel, "-----------------------------------------\n"); QElapsedTimer allAssetsProcessingTimer; @@ -867,7 +903,7 @@ bool ApplicationManagerBase::Run() RemoveOldTempFolders(); Destroy(); - return (startedSuccessfully && FailedAssetsCount() == 0); + return FailedAssetsCount() == 0; } void ApplicationManagerBase::HandleFileRelocation() const @@ -899,7 +935,7 @@ void ApplicationManagerBase::HandleFileRelocation() const while(!m_sourceControlReady) { // We need to wait for source control to be ready before continuing - + if (printCounter % 10 == 0) { AZ_TracePrintf(AssetProcessor::ConsoleChannel, "Waiting for Source Control connection\n"); @@ -1129,7 +1165,7 @@ void ApplicationManagerBase::CheckForIdle() TryScanProductDependencies(); TryHandleFileRelocation(); - + // since we are shutting down, we save the registry and then we quit. AZ_Printf(AssetProcessor::ConsoleChannel, "No assets remain in the build queue. Saving the catalog, and then shutting down.\n"); // stop accepting any further idle messages, as we will shut down - don't want this function to repeat! @@ -1173,7 +1209,7 @@ void ApplicationManagerBase::InitBuilderManager() { m_builderManager->ConnectionLost(connId); }); - + } void ApplicationManagerBase::ShutdownBuilderManager() @@ -1207,7 +1243,7 @@ void ApplicationManagerBase::ShutDownAssetDatabase() AzToolsFramework::AssetDatabase::AssetDatabaseRequests::Bus::Handler::BusDisconnect(); } -void ApplicationManagerBase::InitFileProcessor() +void ApplicationManagerBase::InitFileProcessor() { AssetProcessor::ThreadController* fileProcessorHelper = new AssetProcessor::ThreadController(); @@ -1298,22 +1334,13 @@ bool ApplicationManagerBase::Activate() } InitBuilderConfiguration(); - - m_isCurrentlyLoadingGems = true; - if (!ActivateModules()) - { - // ActivateModules reports any errors it encounters. - m_isCurrentlyLoadingGems = false; - return false; - } - - m_isCurrentlyLoadingGems = false; PopulateApplicationDependencies(); InitAssetProcessorManager(); AssetBuilderSDK::InitializeSerializationContext(); AssetBuilderSDK::InitializeBehaviorContext(); - + AssetBuilder::InitializeSerializationContext(); + InitFileStateCache(); InitFileProcessor(); @@ -1341,7 +1368,7 @@ bool ApplicationManagerBase::Activate() RegisterObjectForQuit(m_rcController); m_connectionsToRemoveOnShutdown << QObject::connect( - m_assetProcessorManager, &AssetProcessor::AssetProcessorManager::AssetProcessorManagerIdleState, + m_assetProcessorManager, &AssetProcessor::AssetProcessorManager::AssetProcessorManagerIdleState, this, [this](bool state) { if (state) @@ -1362,7 +1389,7 @@ bool ApplicationManagerBase::Activate() }); m_connectionsToRemoveOnShutdown << QObject::connect( - this, &ApplicationManagerBase::CheckAssetProcessorManagerIdleState, + this, &ApplicationManagerBase::CheckAssetProcessorManagerIdleState, m_assetProcessorManager, &AssetProcessor::AssetProcessorManager::CheckAssetProcessorIdleState); MakeActivationConnections(); @@ -1376,6 +1403,22 @@ bool ApplicationManagerBase::Activate() return false; } } + + AssetProcessor::AssetProcessorStatusEntry entry(AssetProcessor::AssetProcessorStatus::Initializing_Builders, 0, QString()); + Q_EMIT AssetProcessorStatusChanged(entry); + + AZStd::thread_desc desc; + desc.m_name = "Builder Component Registration"; + AZStd::thread builderRegistrationThread( + desc, + []() + { + AssetProcessor::BuilderRef builder; + AssetProcessor::BuilderManagerBus::BroadcastResult(builder, &AssetProcessor::BuilderManagerBus::Events::GetBuilder, true); + }); + + builderRegistrationThread.detach(); + return true; } @@ -1384,11 +1427,6 @@ bool ApplicationManagerBase::PostActivate() m_connectionManager->LoadConnections(); InitializeInternalBuilders(); - if (!InitializeExternalBuilders()) - { - AZ_Error("AssetProcessor", false, "AssetProcessor is closing. Failed to initialize and load all the external builders. Please ensure that Builders_Temp directory is not read-only. Please see log for more information.\n"); - return false; - } Q_EMIT OnBuildersRegistered(); @@ -1401,7 +1439,7 @@ bool ApplicationManagerBase::PostActivate() AZ::SystemTickBus::Broadcast(&AZ::SystemTickEvents::OnSystemTick); }); - // now that everything is up and running, we start scanning. Before this, we don't want file events to start percolating through the + // now that everything is up and running, we start scanning. Before this, we don't want file events to start percolating through the // asset system. GetAssetScanner()->StartScan(); @@ -1425,124 +1463,20 @@ bool ApplicationManagerBase::InitializeInternalBuilders() return result; } -bool ApplicationManagerBase::InitializeExternalBuilders() -{ - AssetProcessor::AssetProcessorStatusEntry entry(AssetProcessor::AssetProcessorStatus::Initializing_Builders); - Q_EMIT AssetProcessorStatusChanged(entry); - QCoreApplication::processEvents(QEventLoop::AllEvents); - - - // Get the list of external build modules (full paths) - QStringList fileList; - GetExternalBuilderFileList(fileList); - - for (const QString& filePath : fileList) - { - if (QLibrary::isLibrary(filePath)) - { - AssetProcessor::ExternalModuleAssetBuilderInfo* externalAssetBuilderInfo = new AssetProcessor::ExternalModuleAssetBuilderInfo(filePath); - AssetProcessor::AssetBuilderType assetBuilderType = externalAssetBuilderInfo->Load(); - AZ_TracePrintf(AssetProcessor::ConsoleChannel, "AssetProcessor is loading library %s\n", filePath.toUtf8().data()); - if (assetBuilderType == AssetProcessor::AssetBuilderType::None) - { - AZ_Warning(AssetProcessor::DebugChannel, false, "Non-builder DLL was found in Builders directory %s, skipping. \n", filePath.toUtf8().data()); - delete externalAssetBuilderInfo; - continue; - } - - if (assetBuilderType == AssetProcessor::AssetBuilderType::Invalid) - { - AZ_Warning(AssetProcessor::DebugChannel, false, "AssetProcessor was not able to load the library: %s\n", filePath.toUtf8().data()); - delete externalAssetBuilderInfo; - return false; - } - - AZ_TracePrintf(AssetProcessor::ConsoleChannel, "Initializing and registering builder %s\n", externalAssetBuilderInfo->GetName().toUtf8().data()); - - m_currentExternalAssetBuilder = externalAssetBuilderInfo; - - externalAssetBuilderInfo->Initialize(); - - m_currentExternalAssetBuilder = nullptr; - - m_externalAssetBuilders.push_back(externalAssetBuilderInfo); - } - } - - // Also init external builders which may be inside of Gems - AzToolsFramework::ToolsApplicationRequestBus::Broadcast( - &AzToolsFramework::ToolsApplicationRequests::CreateAndAddEntityFromComponentTags, - AZStd::vector({ AssetBuilderSDK::ComponentTags::AssetBuilder }), "AssetBuilders Entity"); - - return true; -} - -bool ApplicationManagerBase::WaitForBuilderExit(AzFramework::ProcessWatcher* processWatcher, AssetBuilderSDK::JobCancelListener* jobCancelListener, AZ::u32 processTimeoutLimitInSeconds) -{ - AZ::u32 exitCode = 0; - bool finishedOK = false; - QElapsedTimer ticker; - ProcessCommunicatorTracePrinter tracer(processWatcher->GetCommunicator(), "AssetBuilder"); - - ticker.start(); - - while (!finishedOK) - { - AZStd::this_thread::sleep_for(AZStd::chrono::milliseconds(s_MaximumSleepTimeMS)); - - tracer.Pump(); - - if (ticker.elapsed() > processTimeoutLimitInSeconds * 1000 || (jobCancelListener && jobCancelListener->IsCancelled())) - { - break; - } - - if (!processWatcher->IsProcessRunning(&exitCode)) - { - finishedOK = true; // we either cant wait for it, or it finished. - break; - } - } - - tracer.Pump(); // empty whats left if possible. - - if (processWatcher->IsProcessRunning(&exitCode)) - { - processWatcher->TerminateProcess(1); - } - - if (exitCode != 0) - { - AZ_Error(AssetProcessor::ConsoleChannel, false, "AssetBuilder exited with error code %d", exitCode); - return false; - } - else if (jobCancelListener && jobCancelListener->IsCancelled()) - { - AZ_TracePrintf(AssetProcessor::DebugChannel, "AssetBuilder was terminated. There was a request to cancel the job.\n"); - return false; - } - else if (!finishedOK) - { - AZ_Error(AssetProcessor::ConsoleChannel, false, "AssetBuilder failed to terminate within %d seconds", processTimeoutLimitInSeconds); - return false; - } - - return true; -} - void ApplicationManagerBase::RegisterBuilderInformation(const AssetBuilderSDK::AssetBuilderDesc& builderDesc) { - // Create Job Function validation - AZ_Error(AssetProcessor::ConsoleChannel, - builderDesc.m_createJobFunction, - "Create Job Function (m_createJobFunction) for %s builder is empty.\n", - builderDesc.m_name.c_str()); + if (!builderDesc.IsExternalBuilder()) + { + // Create Job Function validation + AZ_Error( + AssetProcessor::ConsoleChannel, builderDesc.m_createJobFunction, + "Create Job Function (m_createJobFunction) for %s builder is empty.\n", builderDesc.m_name.c_str()); - // Process Job Function validation - AZ_Error(AssetProcessor::ConsoleChannel, - builderDesc.m_processJobFunction, - "Process Job Function (m_processJobFunction) for %s builder is empty.\n", - builderDesc.m_name.c_str()); + // Process Job Function validation + AZ_Error( + AssetProcessor::ConsoleChannel, builderDesc.m_processJobFunction, + "Process Job Function (m_processJobFunction) for %s builder is empty.\n", builderDesc.m_name.c_str()); + } // Bus ID validation AZ_Error(AssetProcessor::ConsoleChannel, @@ -1550,67 +1484,66 @@ void ApplicationManagerBase::RegisterBuilderInformation(const AssetBuilderSDK::A "Bus ID for %s builder is empty.\n", builderDesc.m_name.c_str()); - // This is an external builder registering, we will want to track its builder desc since it can register multiple ones - AZStd::string builderFilePath; - if (m_currentExternalAssetBuilder) - { - m_currentExternalAssetBuilder->RegisterBuilderDesc(builderDesc.m_busId); - builderFilePath = m_currentExternalAssetBuilder->GetModuleFullPath().toUtf8().data(); - } - AssetBuilderSDK::AssetBuilderDesc modifiedBuilderDesc = builderDesc; // Allow for overrides defined in a BuilderConfig.ini file to update our code defined default values AssetProcessor::BuilderConfigurationRequestBus::Broadcast(&AssetProcessor::BuilderConfigurationRequests::UpdateBuilderDescriptor, builderDesc.m_name, modifiedBuilderDesc); if (builderDesc.IsExternalBuilder()) { - // We're going to override the createJob function so we can run it externally in AssetBuilder, rather than having it run inside the AP - modifiedBuilderDesc.m_createJobFunction = [builderFilePath](const AssetBuilderSDK::CreateJobsRequest& request, AssetBuilderSDK::CreateJobsResponse& response) + // We're going to override the createJob function so we can run it externally in AssetBuilder, rather than having it run + // inside the AP + modifiedBuilderDesc.m_createJobFunction = + [](const AssetBuilderSDK::CreateJobsRequest& request, AssetBuilderSDK::CreateJobsResponse& response) + { + AssetProcessor::BuilderRef builderRef; + AssetProcessor::BuilderManagerBus::BroadcastResult(builderRef, &AssetProcessor::BuilderManagerBusTraits::GetBuilder, false); + + if (builderRef) { - AssetProcessor::BuilderRef builderRef; - AssetProcessor::BuilderManagerBus::BroadcastResult(builderRef, &AssetProcessor::BuilderManagerBusTraits::GetBuilder); + int retryCount = 0; + AssetProcessor::BuilderRunJobOutcome result; - if (builderRef) + do { - int retryCount = 0; - AssetProcessor::BuilderRunJobOutcome result; - - do - { - retryCount++; - result = builderRef->RunJob(request, response, s_MaximumCreateJobsTimeSeconds, "create", builderFilePath, nullptr); - } while (result == AssetProcessor::BuilderRunJobOutcome::LostConnection && retryCount <= AssetProcessor::RetriesForJobNetworkError); - } - else - { - AZ_Error("AssetProcessor", false, "Failed to retrieve a valid builder to process job"); - } - }; + retryCount++; + result = builderRef->RunJob( + request, response, s_MaximumCreateJobsTimeSeconds, "create", "", nullptr); + } while (result == AssetProcessor::BuilderRunJobOutcome::LostConnection && + retryCount <= AssetProcessor::RetriesForJobNetworkError); + } + else + { + AZ_Error("AssetProcessor", false, "Failed to retrieve a valid builder to process job"); + } + }; // Also override the processJob function to run externally - modifiedBuilderDesc.m_processJobFunction = [builderFilePath](const AssetBuilderSDK::ProcessJobRequest& request, AssetBuilderSDK::ProcessJobResponse& response) + modifiedBuilderDesc.m_processJobFunction = + [](const AssetBuilderSDK::ProcessJobRequest& request, AssetBuilderSDK::ProcessJobResponse& response) + { + AssetBuilderSDK::JobCancelListener jobCancelListener(request.m_jobId); + + AssetProcessor::BuilderRef builderRef; + AssetProcessor::BuilderManagerBus::BroadcastResult(builderRef, &AssetProcessor::BuilderManagerBusTraits::GetBuilder, false); + + if (builderRef) { - AssetBuilderSDK::JobCancelListener jobCancelListener(request.m_jobId); + int retryCount = 0; + AssetProcessor::BuilderRunJobOutcome result; - AssetProcessor::BuilderRef builderRef; - AssetProcessor::BuilderManagerBus::BroadcastResult(builderRef, &AssetProcessor::BuilderManagerBusTraits::GetBuilder); - - if (builderRef) + do { - int retryCount = 0; - AssetProcessor::BuilderRunJobOutcome result; - - do - { - retryCount++; - result = builderRef->RunJob(request, response, s_MaximumProcessJobsTimeSeconds, "process", builderFilePath, &jobCancelListener, request.m_tempDirPath); - } while (result == AssetProcessor::BuilderRunJobOutcome::LostConnection && retryCount <= AssetProcessor::RetriesForJobNetworkError); - } - else - { - AZ_Error("AssetProcessor", false, "Failed to retrieve a valid builder to process job"); - } - }; + retryCount++; + result = builderRef->RunJob( + request, response, s_MaximumProcessJobsTimeSeconds, "process", "", &jobCancelListener, request.m_tempDirPath); + } while (result == AssetProcessor::BuilderRunJobOutcome::LostConnection && + retryCount <= AssetProcessor::RetriesForJobNetworkError); + } + else + { + AZ_Error("AssetProcessor", false, "Failed to retrieve a valid builder to process job"); + } + }; } if (m_builderDescMap.find(modifiedBuilderDesc.m_busId) != m_builderDescMap.end()) @@ -1768,7 +1701,7 @@ bool ApplicationManagerBase::CheckSufficientDiskSpace(const QString& savePath, q [[maybe_unused]] bool result = AzToolsFramework::ToolsFileUtils::GetFreeDiskSpace(savePath, bytesFree); AZ_Assert(result, "Unable to determine the amount of free space on drive containing path (%s).", savePath.toUtf8().constData()); - + if (bytesFree < requiredSpace + s_ReservedDiskSpaceInBytes) { if (shutdownIfInsufficient) @@ -1806,8 +1739,8 @@ void ApplicationManagerBase::RemoveOldTempFolders() return; } - // We will remove old temp folders if either their modified time is older than the cutoff time or - // if the total number of temp folders have exceeded the maximum number of temp folders. + // We will remove old temp folders if either their modified time is older than the cutoff time or + // if the total number of temp folders have exceeded the maximum number of temp folders. QFileInfoList entries = root.entryInfoList(QDir::Dirs | QDir::NoDotAndDotDot, QDir::Time); // sorting by modification time int folderCount = 0; bool removeFolder = false; @@ -1821,9 +1754,9 @@ void ApplicationManagerBase::RemoveOldTempFolders() // Since we are sorting the folders list from latest to oldest, we will either be in a state where we have to delete all the remaining folders or not // because either we have reached the folder limit or reached the cutoff date limit. - removeFolder = removeFolder || (folderCount++ >= s_MaximumTempFolders) || + removeFolder = removeFolder || (folderCount++ >= s_MaximumTempFolders) || (entry.lastModified() < cutoffTime); - + if (removeFolder) { QDir dir(entry.absoluteFilePath()); @@ -1837,8 +1770,6 @@ void ApplicationManagerBase::ConnectivityStateChanged(const AzToolsFramework::So Q_EMIT SourceControlReady(); } - - void ApplicationManagerBase::OnAssetProcessorManagerIdleState(bool isIdle) { // these can come in during shutdown. diff --git a/Code/Tools/AssetProcessor/native/utilities/ApplicationManagerBase.h b/Code/Tools/AssetProcessor/native/utilities/ApplicationManagerBase.h index 886880df3a..dbc4841599 100644 --- a/Code/Tools/AssetProcessor/native/utilities/ApplicationManagerBase.h +++ b/Code/Tools/AssetProcessor/native/utilities/ApplicationManagerBase.h @@ -149,7 +149,6 @@ protected: void CreateQtApplication() override; bool InitializeInternalBuilders(); - bool InitializeExternalBuilders(); void InitBuilderManager(); void ShutdownBuilderManager(); bool InitAssetDatabase(); @@ -173,8 +172,6 @@ protected: AssetProcessor::AssetCatalog* GetAssetCatalog() const { return m_assetCatalog; } - static bool WaitForBuilderExit(AzFramework::ProcessWatcher* processWatcher, AssetBuilderSDK::JobCancelListener* jobCancelListener, AZ::u32 processTimeoutLimitInSeconds); - ApplicationServer* m_applicationServer = nullptr; ConnectionManager* m_connectionManager = nullptr; @@ -218,6 +215,8 @@ protected: AZStd::shared_ptr m_internalBuilder; AZStd::shared_ptr m_settingsRegistryBuilder; + bool m_builderRegistrationComplete = false; + // Builder description map based on the builder id AZStd::unordered_map m_builderDescMap; @@ -231,7 +230,7 @@ protected: AZStd::list m_externalAssetBuilders; AssetProcessor::ExternalModuleAssetBuilderInfo* m_currentExternalAssetBuilder = nullptr; - + QAtomicInt m_connectionsAwaitingAssetCatalogSave = 0; int m_remainingAPMJobs = 0; bool m_assetProcessorManagerIsReady = false; diff --git a/Code/Tools/AssetProcessor/native/utilities/BuilderManager.cpp b/Code/Tools/AssetProcessor/native/utilities/BuilderManager.cpp index ddaa5aa777..75c84630f5 100644 --- a/Code/Tools/AssetProcessor/native/utilities/BuilderManager.cpp +++ b/Code/Tools/AssetProcessor/native/utilities/BuilderManager.cpp @@ -16,6 +16,7 @@ #include #include #include +#include namespace AssetProcessor { @@ -138,7 +139,7 @@ namespace AssetProcessor } } - bool Builder::Start() + bool Builder::Start(bool doRegistration) { // Get the current BinXXX folder based on the current running AP QString applicationDir = QCoreApplication::instance()->applicationDirPath(); @@ -155,7 +156,7 @@ namespace AssetProcessor return false; } - const AZStd::vector params = BuildParams("resident", buildersFolder.c_str(), UuidString(), "", ""); + const AZStd::vector params = BuildParams("resident", buildersFolder.c_str(), UuidString(), "", "", doRegistration); m_processWatcher = LaunchProcess(fullExePathString.c_str(), params); @@ -179,7 +180,7 @@ namespace AssetProcessor return !m_processWatcher || (m_processWatcher && m_processWatcher->IsProcessRunning(exitCode)); } - AZStd::vector Builder::BuildParams(const char* task, const char* moduleFilePath, const AZStd::string& builderGuid, const AZStd::string& jobDescriptionFile, const AZStd::string& jobResponseFile) const + AZStd::vector Builder::BuildParams(const char* task, const char* moduleFilePath, const AZStd::string& builderGuid, const AZStd::string& jobDescriptionFile, const AZStd::string& jobResponseFile, bool doRegistration) const { QDir projectCacheRoot; AssetUtilities::ComputeProjectCacheRoot(projectCacheRoot); @@ -200,6 +201,11 @@ namespace AssetProcessor params.emplace_back(AZStd::string::format(R"(-engine-path="%s")", enginePath.c_str())); params.emplace_back(AZStd::string::format("-port=%d", portNumber)); + if(doRegistration) + { + params.emplace_back("--register"); + } + if (moduleFilePath && moduleFilePath[0]) { params.emplace_back(AZStd::string::format(R"(-module="%s")", moduleFilePath)); @@ -232,7 +238,7 @@ namespace AssetProcessor { AzFramework::ProcessLauncher::ProcessLaunchInfo processLaunchInfo; processLaunchInfo.m_processExecutableString = fullExePath; - + AZStd::vector commandLineArray{ fullExePath }; commandLineArray.insert(commandLineArray.end(), params.begin(), params.end()); processLaunchInfo.m_commandlineParameters = AZStd::move(commandLineArray); @@ -350,17 +356,19 @@ namespace AssetProcessor BuilderManager::BuilderManager(ConnectionManager* connectionManager) { using namespace AZStd::placeholders; - connectionManager->RegisterService(AssetBuilderSDK::BuilderHelloRequest::MessageType(), AZStd::bind(&BuilderManager::IncomingBuilderPing, this, _1, _2, _3, _4, _5)); + connectionManager->RegisterService(AssetBuilder::BuilderHelloRequest::MessageType(), AZStd::bind(&BuilderManager::IncomingBuilderPing, this, _1, _2, _3, _4, _5)); // Setup a background thread to pump the idle builders so they don't get blocked trying to output to stdout/err - m_pollingThread = AZStd::thread([this]() + AZStd::thread_desc desc; + desc.m_name = "BuilderManager Idle Pump"; + m_pollingThread = AZStd::thread(desc, [this]() + { + while (!m_quitListener.WasQuitRequested()) { - while (!m_quitListener.WasQuitRequested()) - { - PumpIdleBuilders(); - AZStd::this_thread::sleep_for(AZStd::chrono::milliseconds(s_IdleBuilderPumpingDelayMS)); - } - }); + PumpIdleBuilders(); + AZStd::this_thread::sleep_for(AZStd::chrono::milliseconds(s_IdleBuilderPumpingDelayMS)); + } + }); m_quitListener.BusConnect(); BusConnect(); @@ -399,8 +407,8 @@ namespace AssetProcessor void BuilderManager::IncomingBuilderPing(AZ::u32 connId, AZ::u32 /*type*/, AZ::u32 serial, QByteArray payload, QString platform) { - AssetBuilderSDK::BuilderHelloRequest requestPing; - AssetBuilderSDK::BuilderHelloResponse responsePing; + AssetBuilder::BuilderHelloRequest requestPing; + AssetBuilder::BuilderHelloResponse responsePing; if (!AZ::Utils::LoadObjectFromBufferInPlace(payload.data(), payload.length(), requestPing)) { @@ -476,7 +484,7 @@ namespace AssetProcessor return builder; } - BuilderRef BuilderManager::GetBuilder() + BuilderRef BuilderManager::GetBuilder(bool doRegistration) { AZStd::shared_ptr newBuilder; BuilderRef builderRef; @@ -484,27 +492,30 @@ namespace AssetProcessor { AZStd::unique_lock lock(m_buildersMutex); - for (auto itr = m_builders.begin(); itr != m_builders.end(); ) + if (!doRegistration) { - auto& builder = itr->second; - - if (!builder->m_busy) + for (auto itr = m_builders.begin(); itr != m_builders.end();) { - builder->PumpCommunicator(); + auto& builder = itr->second; - if (builder->IsValid()) + if (!builder->m_busy) { - return BuilderRef(builder); + builder->PumpCommunicator(); + + if (builder->IsValid()) + { + return BuilderRef(builder); + } + else + { + itr = m_builders.erase(itr); + } } else { - itr = m_builders.erase(itr); + ++itr; } } - else - { - ++itr; - } } AZ_TracePrintf("BuilderManager", "Starting new builder for job request\n"); @@ -516,7 +527,7 @@ namespace AssetProcessor builderRef = BuilderRef(newBuilder); } - if (!newBuilder->Start()) + if (!newBuilder->Start(doRegistration)) { AZ_Error("BuilderManager", false, "Builder failed to start"); diff --git a/Code/Tools/AssetProcessor/native/utilities/BuilderManager.h b/Code/Tools/AssetProcessor/native/utilities/BuilderManager.h index 1208345a39..64241b106e 100644 --- a/Code/Tools/AssetProcessor/native/utilities/BuilderManager.h +++ b/Code/Tools/AssetProcessor/native/utilities/BuilderManager.h @@ -39,7 +39,7 @@ namespace AssetProcessor virtual ~BuilderManagerBusTraits() = default; //! Returns a builder for doing work - virtual BuilderRef GetBuilder() = 0; + virtual BuilderRef GetBuilder(bool doRegistration) = 0; }; using BuilderManagerBus = AZ::EBus; @@ -98,12 +98,12 @@ namespace AssetProcessor private: //! Starts the builder process and waits for it to connect - bool Start(); + bool Start(bool doRegistration); //! Sets the connection id and signals that the builder has connected void SetConnection(AZ::u32 connId); - AZStd::vector BuildParams(const char* task, const char* moduleFilePath, const AZStd::string& builderGuid, const AZStd::string& jobDescriptionFile, const AZStd::string& jobResponseFile) const; + AZStd::vector BuildParams(const char* task, const char* moduleFilePath, const AZStd::string& builderGuid, const AZStd::string& jobDescriptionFile, const AZStd::string& jobResponseFile, bool doRegistration) const; AZStd::unique_ptr LaunchProcess(const char* fullExePath, const AZStd::vector& params) const; //! Waits for the builder exe to send the job response and pumps stdout/err @@ -169,7 +169,7 @@ namespace AssetProcessor void ConnectionLost(AZ::u32 connId); //BuilderManagerBus - BuilderRef GetBuilder() override; + BuilderRef GetBuilder(bool doRegistration) override; private: diff --git a/Code/Tools/AssetProcessor/native/utilities/BuilderManager.inl b/Code/Tools/AssetProcessor/native/utilities/BuilderManager.inl index 9893f32f6a..039680dfae 100644 --- a/Code/Tools/AssetProcessor/native/utilities/BuilderManager.inl +++ b/Code/Tools/AssetProcessor/native/utilities/BuilderManager.inl @@ -50,7 +50,7 @@ namespace AssetProcessor if (!netResponse.m_response.Succeeded() || s_createRequestFileForSuccessfulJob) { - // we write the request out to disk for failure or debugging + // we write the request out to disk for failure or debugging if (!DebugWriteRequestFile(tempFolderPath.c_str(), request, task, modulePath)) { return BuilderRunJobOutcome::FailedToWriteDebugRequest; @@ -83,7 +83,7 @@ namespace AssetProcessor return false; } - auto params = BuildParams(task.c_str(), modulePath.c_str(), "", jobRequestFile, jobResponseFile); + auto params = BuildParams(task.c_str(), modulePath.c_str(), "", jobRequestFile, jobResponseFile, false); AZStd::string paramString; AZ::StringFunc::Join(paramString, params.begin(), params.end(), " "); diff --git a/Code/Tools/AssetProcessor/native/utilities/GUIApplicationManager.cpp b/Code/Tools/AssetProcessor/native/utilities/GUIApplicationManager.cpp index 7652b67db0..13a6b0699a 100644 --- a/Code/Tools/AssetProcessor/native/utilities/GUIApplicationManager.cpp +++ b/Code/Tools/AssetProcessor/native/utilities/GUIApplicationManager.cpp @@ -318,16 +318,8 @@ bool GUIApplicationManager::Run() qApp->setQuitOnLastWindowClosed(false); - QTimer::singleShot(0, this, [this]() - { - if (!PostActivate()) - { - QuitRequested(); - m_startedSuccessfully = false; - } - }); - m_duringStartup = false; + m_startedSuccessfully = true; int resultCode = qApp->exec(); // this blocks until the last window is closed. @@ -483,6 +475,7 @@ bool GUIApplicationManager::PostActivate() { if (!ApplicationManagerBase::PostActivate()) { + m_startedSuccessfully = false; return false; }