birkeh/o3de
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Asset Processor: Remove gem loading from AP (#6488)

Browse Source 
* AssetBuilder sends builder registration network message to AP

Signed-off-by: amzn-mike <80125227+amzn-mike@users.noreply.github.com>

* Add AP activating status message

Signed-off-by: amzn-mike <80125227+amzn-mike@users.noreply.github.com>

* First builder handles registration.

Fixed deadlock caused by AP and AssetBuilder waiting on each other when registering by moving AP builder start code to a thread

Signed-off-by: amzn-mike <80125227+amzn-mike@users.noreply.github.com>

* Clean up external builder registration

Signed-off-by: amzn-mike <80125227+amzn-mike@users.noreply.github.com>

* Add thread description for builder manager idle thread

Signed-off-by: amzn-mike <80125227+amzn-mike@users.noreply.github.com>

* Remove gem loading

Signed-off-by: amzn-mike <80125227+amzn-mike@users.noreply.github.com>

* Clean up builder registration and remove unused functions

Signed-off-by: amzn-mike <80125227+amzn-mike@users.noreply.github.com>

* Remove PostActivate call from batch application since it will be called after builders are registered

Signed-off-by: amzn-mike <80125227+amzn-mike@users.noreply.github.com>

* Removal external builder dependency scanning since we no longer support builder dlls

Signed-off-by: amzn-mike <80125227+amzn-mike@users.noreply.github.com>

* Fix missing bus disconnect

Signed-off-by: amzn-mike <80125227+amzn-mike@users.noreply.github.com>

* Remove unused variable

Signed-off-by: amzn-mike <80125227+amzn-mike@users.noreply.github.com>

* Moved AP-AssetBuilder specific types into AssetBuilder.Static library.  Also removed some unused/old code

Signed-off-by: amzn-mike <80125227+amzn-mike@users.noreply.github.com>
This commit is contained in:
amzn-mike
2022-01-04 14:34:56 -06:00
committed by GitHub
parent 052e282208
commit 8ee384f436
20 changed files with 659 additions and 727 deletions
Download Patch File Download Diff File Expand all files Collapse all files
Code/Tools/AssetProcessor/AssetBuilder/AssetBuilderApplication.cpp
+2
View File
@@ -37,6 +37,7 @@
#include <AssetBuilderInfo.h>
#include <AzCore/Interface/Interface.h>
#include <Entity/EntityUtilityComponent.h>
#include <AssetBuilderStatic.h>
namespace AssetBuilder
{
@@ -165,6 +166,7 @@ void AssetBuilderApplication::StartCommon(AZ::Entity* systemEntity)
AssetBuilderSDK::InitializeSerializationContext();
AssetBuilderSDK::InitializeBehaviorContext();
AssetBuilder::InitializeSerializationContext();
// the asset builder app never writes source files, only assets, so there is no need to do any kind of asset upgrading
AZ::Data::AssetManager::Instance().SetAssetInfoUpgradingEnabled(false);
Code/Tools/AssetProcessor/AssetBuilder/AssetBuilderComponent.cpp
+78 -55
View File
@@ -34,6 +34,7 @@
#include <AzCore/Interface/Interface.h>
#include <AzFramework/Asset/AssetSystemComponent.h>
#include <ToolsComponents/ToolsAssetCatalogComponent.h>
#include <AssetBuilderStatic.h>
// Command-line parameter options:
static const char* const s_paramHelp = "help"; // Print help information.
@@ -51,10 +52,10 @@ static const char* const s_paramDebugCreate = "debug_create"; // Debug mode for
static const char* const s_paramDebugProcess = "debug_process"; // Debug mode for the process job of the specified file.
static const char* const s_paramPlatformTags = "tags"; // Additional list of tags to add platform tag list.
static const char* const s_paramPlatform = "platform"; // Platform to use
static const char* const s_paramRegisterBuilders = "register"; // Indicates the AP is starting up and requesting a list of registered builders
// Task modes:
static const char* const s_taskResident = "resident"; // stays up and running indefinitely, accepting jobs via network connection
static const char* const s_taskRegisterBuilder = "register"; // outputs all the builder descriptors
static const char* const s_taskCreateJob = "create"; // runs a builders createJobs function
static const char* const s_taskProcessJob = "process"; // runs processJob function
static const char* const s_taskDebug = "debug"; // runs a one shot job in a fake environment for a specified file.
@@ -204,6 +205,40 @@ void AssetBuilderComponent::Reflect(AZ::ReflectContext* context)
}
}
bool AssetBuilderComponent::DoHelloPing()
{
using namespace AssetBuilder;
BuilderHelloRequest request;
BuilderHelloResponse response;
AZStd::string id;
if (!GetParameter(s_paramId, id))
{
return false;
}
request.m_uuid = AZ::Uuid::CreateString(id.c_str());
AZ_TracePrintf(
"AssetBuilderComponent", "RunInResidentMode: Pinging asset processor with the builder UUID %s\n",
request.m_uuid.ToString<AZStd::string>().c_str());
bool result = AzFramework::AssetSystem::SendRequest(request, response);
AZ_Error("AssetBuilder", result, "Failed to send hello request to Asset Processor");
// This error is only shown if we successfully got a response AND the response explicitly indicates the AP rejected the builder
AZ_Error("AssetBuilder", !result || response.m_accepted, "Asset Processor rejected connection request");
if (result)
{
AZ_TracePrintf("AssetBuilder", "Builder ID: %s\n", response.m_uuid.ToString<AZStd::string>().c_str());
}
return result;
}
bool AssetBuilderComponent::Run()
{
AZ_TracePrintf("AssetBuilderComponent", "Run: Parsing command line.\n");
@@ -217,8 +252,8 @@ bool AssetBuilderComponent::Run()
}
AZStd::string task;
AZStd::string debugFile;
if (GetParameter(s_paramDebug, debugFile, false))
{
task = s_taskDebug;
@@ -256,11 +291,13 @@ bool AssetBuilderComponent::Run()
AZ_TracePrintf("AssetBuilderComponent", "Run: Connecting back to Asset Processor...\n");
bool connectedToAssetProcessor = ConnectToAssetProcessor();
//AP connection is required to access the asset catalog
AZ_Error("AssetBuilder", connectedToAssetProcessor, "Failed to establish a network connection to the AssetProcessor. Use -help for options.");;
AZ_Error("AssetBuilder", connectedToAssetProcessor, "Failed to establish a network connection to the AssetProcessor. Use -help for options.");
bool registerBuilders = commandLine->GetNumSwitchValues(s_paramRegisterBuilders) > 0;
IBuilderApplication* builderApplication = AZ::Interface<IBuilderApplication>::Get();
if(!builderApplication)
if (!builderApplication)
{
AZ_Error("AssetBuilder", false, "Failed to retreive IBuilderApplication interface");
return false;
@@ -274,7 +311,7 @@ bool AssetBuilderComponent::Run()
{
if (task == s_taskResident)
{
result = RunInResidentMode();
result = RunInResidentMode(registerBuilders);
}
else if (task == s_taskDebug)
{
@@ -370,43 +407,46 @@ bool AssetBuilderComponent::ConnectToAssetProcessor()
//////////////////////////////////////////////////////////////////////////
bool AssetBuilderComponent::RunInResidentMode()
bool AssetBuilderComponent::SendRegisteredBuildersToAp()
{
using namespace AssetBuilderSDK;
AssetBuilder::BuilderRegistrationRequest registrationRequest;
for (const auto& [uuid, desc] : m_assetBuilderDescMap)
{
AssetBuilder::BuilderRegistration registration;
registration.m_name = desc->m_name;
registration.m_analysisFingerprint = desc->m_analysisFingerprint;
registration.m_flags = desc->m_flags;
registration.m_flagsByJobKey = desc->m_flagsByJobKey;
registration.m_version = desc->m_version;
registration.m_busId = desc->m_busId;
registration.m_patterns = desc->m_patterns;
registration.m_productsToKeepOnFailure = desc->m_productsToKeepOnFailure;
registrationRequest.m_builders.push_back(AZStd::move(registration));
}
bool result = SendRequest(registrationRequest);
AZ_Error("AssetBuilder", result, "Failed to send builder registration request to Asset Processor");
return result;
}
bool AssetBuilderComponent::RunInResidentMode(bool sendRegistration)
{
using namespace AssetBuilder;
using namespace AZStd::placeholders;
AZ_TracePrintf("AssetBuilderComponent", "RunInResidentMode: Starting resident mode (waiting for commands to arrive)\n");
AZStd::string port, id, builderFolder;
if (!GetParameter(s_paramId, id)
|| !GetParameter(s_paramModule, builderFolder))
{
return false;
}
if (!LoadBuilders(builderFolder))
{
return false;
}
AzFramework::SocketConnection::GetInstance()->AddMessageHandler(CreateJobsNetRequest::MessageType(), AZStd::bind(&AssetBuilderComponent::CreateJobsResidentHandler, this, _1, _2, _3, _4));
AzFramework::SocketConnection::GetInstance()->AddMessageHandler(ProcessJobNetRequest::MessageType(), AZStd::bind(&AssetBuilderComponent::ProcessJobResidentHandler, this, _1, _2, _3, _4));
BuilderHelloRequest request;
BuilderHelloResponse response;
bool result = DoHelloPing() && ((sendRegistration && SendRegisteredBuildersToAp()) || !sendRegistration);
request.m_uuid = AZ::Uuid::CreateString(id.c_str());
AZ_TracePrintf("AssetBuilderComponent", "RunInResidentMode: Pinging asset processor with the builder UUID %s\n", request.m_uuid.ToString<AZStd::string>().c_str());
bool result = AzFramework::AssetSystem::SendRequest(request, response);
AZ_Error("AssetBuilder", result, "Failed to send hello request to Asset Processor");
// This error is only shown if we successfully got a response AND the response explicitly indicates the AP rejected the builder
AZ_Error("AssetBuilder", !result || response.m_accepted, "Asset Processor rejected connection request");
if (result && response.m_accepted)
if (result)
{
m_running = true;
@@ -415,7 +455,6 @@ bool AssetBuilderComponent::RunInResidentMode()
AzFramework::EngineConnectionEvents::Bus::Handler::BusConnect(); // Listen for disconnects
AZ_TracePrintf("AssetBuilder", "Builder ID: %s\n", response.m_uuid.ToString<AZStd::string>().c_str());
AZ_TracePrintf("AssetBuilder", "Resident mode ready\n");
m_mainEvent.acquire();
AZ_TracePrintf("AssetBuilder", "Shutting down\n");
@@ -736,11 +775,7 @@ bool AssetBuilderComponent::RunOneShotTask(const AZStd::string& task)
AZ::StringFunc::Path::Normalize(inputFilePath);
AZ::StringFunc::Path::Normalize(outputFilePath);
if (task == s_taskRegisterBuilder)
{
return HandleRegisterBuilder(inputFilePath, outputFilePath);
}
else if (task == s_taskCreateJob)
if (task == s_taskCreateJob)
{
auto func = [this](const AssetBuilderSDK::CreateJobsRequest& request, AssetBuilderSDK::CreateJobsResponse& response)
{
@@ -896,7 +931,7 @@ void AssetBuilderComponent::JobThread()
{
case JobType::Create:
{
using namespace AssetBuilderSDK;
using namespace AssetBuilder;
auto* netRequest = azrtti_cast<CreateJobsNetRequest*>(job->m_netRequest.get());
auto* netResponse = azrtti_cast<CreateJobsNetResponse*>(job->m_netResponse.get());
@@ -922,7 +957,7 @@ void AssetBuilderComponent::JobThread()
}
case JobType::Process:
{
using namespace AssetBuilderSDK;
using namespace AssetBuilder;
AZ_TracePrintf("AssetBuilder", "Running processJob task\n");
@@ -981,14 +1016,14 @@ void AssetBuilderComponent::JobThread()
void AssetBuilderComponent::CreateJobsResidentHandler(AZ::u32 /*typeId*/, AZ::u32 serial, const void* data, AZ::u32 dataLength)
{
using namespace AssetBuilderSDK;
using namespace AssetBuilder;
ResidentJobHandler<CreateJobsNetRequest, CreateJobsNetResponse>(serial, data, dataLength, JobType::Create);
}
void AssetBuilderComponent::ProcessJobResidentHandler(AZ::u32 /*typeId*/, AZ::u32 serial, const void* data, AZ::u32 dataLength)
{
using namespace AssetBuilderSDK;
using namespace AssetBuilder;
ResidentJobHandler<ProcessJobNetRequest, ProcessJobNetResponse>(serial, data, dataLength, JobType::Process);
}
@@ -1018,18 +1053,6 @@ bool AssetBuilderComponent::HandleTask(const AZStd::string& inputFilePath, const
return true;
}
bool AssetBuilderComponent::HandleRegisterBuilder(const AZStd::string& /*inputFilePath*/, const AZStd::string& outputFilePath) const
{
AssetBuilderSDK::RegisterBuilderResponse response;
for (const auto& pair : m_assetBuilderDescMap)
{
response.m_assetBuilderDescList.push_back(*pair.second);
}
return AZ::Utils::SaveObjectToFile(outputFilePath, AZ::DataStream::ST_XML, &response);
}
void AssetBuilderComponent::UpdateResultCode(const AssetBuilderSDK::ProcessJobRequest& request, AssetBuilderSDK::ProcessJobResponse& response) const
{
if (request.m_jobDescription.m_failOnError)
Code/Tools/AssetProcessor/AssetBuilder/AssetBuilderComponent.h
+6 -7
View File
@@ -46,6 +46,7 @@ class AssetBuilderComponent
public:
AZ_COMPONENT(AssetBuilderComponent, "{04332899-5d73-4d41-86b7-b1017d349673}")
static void Reflect(AZ::ReflectContext* context);
bool DoHelloPing();
AssetBuilderComponent() = default;
~AssetBuilderComponent() override = default;
@@ -64,7 +65,7 @@ public:
void RegisterBuilderInformation(const AssetBuilderSDK::AssetBuilderDesc& builderDesc) override;
void RegisterComponentDescriptor(AZ::ComponentDescriptor* descriptor) override;
//EngineConnectionEvents Handler
void Disconnected(AzFramework::SocketConnection* connection) override;
@@ -98,12 +99,13 @@ protected:
static const char* GetLibraryExtension();
bool ConnectToAssetProcessor();
bool SendRegisteredBuildersToAp();
bool LoadBuilders(const AZStd::string& builderFolder);
bool LoadBuilder(const AZStd::string& filePath);
void UnloadBuilders();
//! Hooks up net job request handling and keeps the AssetBuilder running indefinitely
bool RunInResidentMode();
bool RunInResidentMode(bool sendRegistration);
bool RunDebugTask(AZStd::string&& debugFile, bool runCreateJobs, bool runProcessJob);
bool RunOneShotTask(const AZStd::string& task);
@@ -120,9 +122,6 @@ protected:
void ProcessJob(const AssetBuilderSDK::ProcessJobFunction& job, const AssetBuilderSDK::ProcessJobRequest& request, AssetBuilderSDK::ProcessJobResponse& outResponse);
//! Handles a builder registration request
bool HandleRegisterBuilder(const AZStd::string& inputFilePath, const AZStd::string& outputFilePath) const;
//! If needed looks at collected data and updates the result code from the job accordingly.
void UpdateResultCode(const AssetBuilderSDK::ProcessJobRequest& request, AssetBuilderSDK::ProcessJobResponse& response) const;
@@ -141,7 +140,7 @@ protected:
//! Currently loading builder
AssetBuilder::ExternalModuleAssetBuilderInfo* m_currentAssetBuilder = nullptr;
//! Thread for running a job, so we don't block the network thread while doing work
AZStd::thread_desc m_jobThreadDesc;
AZStd::thread m_jobThread;
@@ -153,7 +152,7 @@ protected:
AZStd::binary_semaphore m_mainEvent;
//! Use to signal a new job is ready to be processed
AZStd::binary_semaphore m_jobEvent;
//! Lock for m_queuedJob
AZStd::mutex m_jobMutex;
Code/Tools/AssetProcessor/AssetBuilder/AssetBuilderStatic.cpp
+182
View File
@@ -0,0 +1,182 @@
/*
* Copyright (c) Contributors to the Open 3D Engine Project.
* For complete copyright and license terms please see the LICENSE at the root of this distribution.
*
* SPDX-License-Identifier: Apache-2.0 OR MIT
*
*/
#include <AssetBuilderStatic.h>
#include <AzCore/Component/ComponentApplicationBus.h>
namespace AssetBuilder
{
void Reflect(AZ::ReflectContext* context)
{
BuilderRegistrationRequest::Reflect(context);
BuilderHelloRequest::Reflect(context);
BuilderHelloResponse::Reflect(context);
CreateJobsNetRequest::Reflect(context);
CreateJobsNetResponse::Reflect(context);
ProcessJobNetRequest::Reflect(context);
ProcessJobNetResponse::Reflect(context);
}
void InitializeSerializationContext()
{
AZ::SerializeContext* serializeContext = nullptr;
AZ::ComponentApplicationBus::BroadcastResult(serializeContext, &AZ::ComponentApplicationBus::Events::GetSerializeContext);
AZ_Assert(serializeContext, "Unable to retrieve serialize context.");
Reflect(serializeContext);
}
void BuilderHelloRequest::Reflect(AZ::ReflectContext* context)
{
auto serialize = azrtti_cast<AZ::SerializeContext*>(context);
if (serialize)
{
serialize->Class<BuilderHelloRequest>()->Version(1)->Field("UUID", &BuilderHelloRequest::m_uuid);
}
}
unsigned int BuilderHelloRequest::MessageType()
{
static unsigned int messageType = AZ_CRC("AssetBuilderSDK::BuilderHelloRequest", 0x213a7248);
return messageType;
}
unsigned int BuilderHelloRequest::GetMessageType() const
{
return MessageType();
}
void BuilderHelloResponse::Reflect(AZ::ReflectContext* context)
{
auto serialize = azrtti_cast<AZ::SerializeContext*>(context);
if (serialize)
{
serialize->Class<BuilderHelloResponse>()
->Version(1)
->Field("Accepted", &BuilderHelloResponse::m_accepted)
->Field("UUID", &BuilderHelloResponse::m_uuid);
}
}
unsigned int BuilderHelloResponse::GetMessageType() const
{
return BuilderHelloRequest::MessageType();
}
//////////////////////////////////////////////////////////////////////////
void CreateJobsNetRequest::Reflect(AZ::ReflectContext* context)
{
auto serialize = azrtti_cast<AZ::SerializeContext*>(context);
if (serialize)
{
serialize->Class<CreateJobsNetRequest>()->Version(1)->Field("Request", &CreateJobsNetRequest::m_request);
}
}
unsigned int CreateJobsNetRequest::MessageType()
{
static unsigned int messageType = AZ_CRC("AssetBuilderSDK::CreateJobsNetRequest", 0xc48209c0);
return messageType;
}
unsigned int CreateJobsNetRequest::GetMessageType() const
{
return MessageType();
}
void CreateJobsNetResponse::Reflect(AZ::ReflectContext* context)
{
auto serialize = azrtti_cast<AZ::SerializeContext*>(context);
if (serialize)
{
serialize->Class<CreateJobsNetResponse>()->Version(1)->Field("Response", &CreateJobsNetResponse::m_response);
}
}
unsigned int CreateJobsNetResponse::GetMessageType() const
{
return CreateJobsNetRequest::MessageType();
}
void ProcessJobNetRequest::Reflect(AZ::ReflectContext* context)
{
auto serialize = azrtti_cast<AZ::SerializeContext*>(context);
if (serialize)
{
serialize->Class<ProcessJobNetRequest>()->Version(1)->Field("Request", &ProcessJobNetRequest::m_request);
}
}
unsigned int ProcessJobNetRequest::MessageType()
{
static unsigned int messageType = AZ_CRC("AssetBuilderSDK::ProcessJobNetRequest", 0x479f340f);
return messageType;
}
unsigned int ProcessJobNetRequest::GetMessageType() const
{
return MessageType();
}
void ProcessJobNetResponse::Reflect(AZ::ReflectContext* context)
{
auto serialize = azrtti_cast<AZ::SerializeContext*>(context);
if (serialize)
{
serialize->Class<ProcessJobNetResponse>()->Version(1)->Field("Response", &ProcessJobNetResponse::m_response);
}
}
unsigned int ProcessJobNetResponse::GetMessageType() const
{
return ProcessJobNetRequest::MessageType();
}
//---------------------------------------------------------------------
void BuilderRegistration::Reflect(AZ::ReflectContext* context)
{
auto serialize = azrtti_cast<AZ::SerializeContext*>(context);
if (serialize)
{
serialize->Class<BuilderRegistration>()
->Version(1)
->Field("Name", &BuilderRegistration::m_name)
->Field("Patterns", &BuilderRegistration::m_patterns)
->Field("BusId", &BuilderRegistration::m_busId)
->Field("Version", &BuilderRegistration::m_version)
->Field("AnalysisFingerprint", &BuilderRegistration::m_analysisFingerprint)
->Field("Flags", &BuilderRegistration::m_flags)
->Field("FlagsByJobKey", &BuilderRegistration::m_flagsByJobKey)
->Field("ProductsToKeepOnFailure", &BuilderRegistration::m_productsToKeepOnFailure);
}
}
void BuilderRegistrationRequest::Reflect(AZ::ReflectContext* context)
{
BuilderRegistration::Reflect(context);
auto serialize = azrtti_cast<AZ::SerializeContext*>(context);
if (serialize)
{
serialize->Class<BuilderRegistrationRequest, BaseAssetProcessorMessage>()->Version(1)->Field(
"Builders", &BuilderRegistrationRequest::m_builders);
}
}
unsigned int BuilderRegistrationRequest::GetMessageType() const
{
return BuilderRegistrationRequest::MessageType;
}
} // namespace AssetBuilder
Code/Tools/AssetProcessor/AssetBuilder/AssetBuilderStatic.h
+140
View File
@@ -0,0 +1,140 @@
/*
* Copyright (c) Contributors to the Open 3D Engine Project.
* For complete copyright and license terms please see the LICENSE at the root of this distribution.
*
* SPDX-License-Identifier: Apache-2.0 OR MIT
*
*/
#pragma once
#include <AzFramework/Asset/AssetProcessorMessages.h>
#include <AssetBuilderSDK/AssetBuilderSDK.h>
namespace AssetBuilder
{
void Reflect(AZ::ReflectContext* context);
void InitializeSerializationContext();
//! BuilderHelloRequest is sent by an AssetBuilder that is attempting to connect to the AssetProcessor to register itself as a worker
class BuilderHelloRequest : public AzFramework::AssetSystem::BaseAssetProcessorMessage
{
public:
AZ_CLASS_ALLOCATOR(BuilderHelloRequest, AZ::OSAllocator, 0);
AZ_RTTI(BuilderHelloRequest, "{5fab5962-a1d8-42a5-bf7a-fb1a8c5a9588}", BaseAssetProcessorMessage);
static void Reflect(AZ::ReflectContext* context);
static unsigned int MessageType();
unsigned int GetMessageType() const override;
//! Unique ID assigned to this builder to identify it
AZ::Uuid m_uuid = AZ::Uuid::CreateNull();
};
//! BuilderHelloResponse contains the AssetProcessor's response to a builder connection attempt, indicating if it is accepted and the ID
//! that it was assigned
class BuilderHelloResponse : public AzFramework::AssetSystem::BaseAssetProcessorMessage
{
public:
AZ_CLASS_ALLOCATOR(BuilderHelloResponse, AZ::OSAllocator, 0);
AZ_RTTI(BuilderHelloResponse, "{5f3d7c11-6639-4c6f-980a-32be546903c2}", BaseAssetProcessorMessage);
static void Reflect(AZ::ReflectContext* context);
unsigned int GetMessageType() const override;
//! Indicates if the builder was accepted by the AP
bool m_accepted = false;
//! Unique ID assigned to the builder. If the builder isn't a local process, this is the ID assigned by the AP
AZ::Uuid m_uuid = AZ::Uuid::CreateNull();
};
class CreateJobsNetRequest : public AzFramework::AssetSystem::BaseAssetProcessorMessage
{
public:
AZ_CLASS_ALLOCATOR(CreateJobsNetRequest, AZ::OSAllocator, 0);
AZ_RTTI(CreateJobsNetRequest, "{97fa717d-3a09-4d21-95c6-b2eafd773f1c}", BaseAssetProcessorMessage);
static void Reflect(AZ::ReflectContext* context);
static unsigned int MessageType();
unsigned int GetMessageType() const override;
AssetBuilderSDK::CreateJobsRequest m_request;
};
class CreateJobsNetResponse : public AzFramework::AssetSystem::BaseAssetProcessorMessage
{
public:
AZ_CLASS_ALLOCATOR(CreateJobsNetResponse, AZ::OSAllocator, 0);
AZ_RTTI(CreateJobsNetResponse, "{b2c7c2d3-b60e-4b27-b699-43e0ba991c33}", BaseAssetProcessorMessage);
static void Reflect(AZ::ReflectContext* context);
unsigned int GetMessageType() const override;
AssetBuilderSDK::CreateJobsResponse m_response;
};
class ProcessJobNetRequest : public AzFramework::AssetSystem::BaseAssetProcessorMessage
{
public:
AZ_CLASS_ALLOCATOR(ProcessJobNetRequest, AZ::OSAllocator, 0);
AZ_RTTI(ProcessJobNetRequest, "{05288de1-020b-48db-b9de-715f17284efa}", BaseAssetProcessorMessage);
static void Reflect(AZ::ReflectContext* context);
static unsigned int MessageType();
unsigned int GetMessageType() const override;
AssetBuilderSDK::ProcessJobRequest m_request;
};
class ProcessJobNetResponse : public AzFramework::AssetSystem::BaseAssetProcessorMessage
{
public:
AZ_CLASS_ALLOCATOR(ProcessJobNetResponse, AZ::OSAllocator, 0);
AZ_RTTI(ProcessJobNetResponse, "{26ddf882-246c-4cfb-912f-9b8e389df4f6}", BaseAssetProcessorMessage);
static void Reflect(AZ::ReflectContext* context);
unsigned int GetMessageType() const override;
AssetBuilderSDK::ProcessJobResponse m_response;
};
//////////////////////////////////////////////////////////////////////////
struct BuilderRegistration
{
AZ_CLASS_ALLOCATOR(BuilderRegistration, AZ::OSAllocator, 0);
AZ_TYPE_INFO(BuilderRegistration, "{36E785C3-5046-4568-870A-336C8249E453}");
static void Reflect(AZ::ReflectContext* context);
AZStd::string m_name;
AZStd::vector<AssetBuilderSDK::AssetBuilderPattern> m_patterns;
AZ::Uuid m_busId;
int m_version = 0;
AZStd::string m_analysisFingerprint;
AZ::u8 m_flags = 0;
AZStd::unordered_map<AZStd::string, AZ::u8> m_flagsByJobKey;
AZStd::unordered_map<AZStd::string, AZStd::unordered_set<AZ::u32>> m_productsToKeepOnFailure;
};
class BuilderRegistrationRequest : public AzFramework::AssetSystem::BaseAssetProcessorMessage
{
public:
AZ_CLASS_ALLOCATOR(BuilderRegistrationRequest, AZ::OSAllocator, 0);
AZ_RTTI(BuilderRegistrationRequest, "{FA9CF2D5-C847-47F3-979D-6C3AE061715C}", BaseAssetProcessorMessage);
static void Reflect(AZ::ReflectContext* context);
static constexpr unsigned int MessageType = AZ_CRC_CE("AssetSystem::BuilderRegistrationRequest");
BuilderRegistrationRequest() = default;
unsigned int GetMessageType() const override;
AZStd::vector<BuilderRegistration> m_builders;
};
} // namespace AssetBuilder
Code/Tools/AssetProcessor/AssetBuilder/CMakeLists.txt
+17
View File
@@ -6,6 +6,22 @@
#
#
ly_add_target(
NAME AssetBuilder.Static STATIC
NAMESPACE AZ
FILES_CMAKE
asset_builder_static_files.cmake
INCLUDE_DIRECTORIES
PUBLIC
.
BUILD_DEPENDENCIES
PUBLIC
AZ::AzCore
AZ::AzFramework
AZ::AzToolsFramework
AZ::AssetBuilderSDK
)
ly_add_target(
NAME AssetBuilder EXECUTABLE
NAMESPACE AZ
@@ -17,6 +33,7 @@ ly_add_target(
.
BUILD_DEPENDENCIES
PRIVATE
AssetBuilder.Static
3rdParty::Qt::Core
3rdParty::Qt::Gui
3rdParty::Qt::Network
Code/Tools/AssetProcessor/AssetBuilder/asset_builder_static_files.cmake
+12
View File
@@ -0,0 +1,12 @@
#
# Copyright (c) Contributors to the Open 3D Engine Project.
# For complete copyright and license terms please see the LICENSE at the root of this distribution.
#
# SPDX-License-Identifier: Apache-2.0 OR MIT
#
#
set(FILES
AssetBuilderStatic.h
AssetBuilderStatic.cpp
)
Code/Tools/AssetProcessor/AssetBuilderSDK/AssetBuilderSDK/AssetBuilderSDK.cpp
-168
View File
@@ -1172,19 +1172,10 @@ namespace AssetBuilderSDK
JobProduct::Reflect(context);
AssetBuilderDesc::Reflect(context);
RegisterBuilderRequest::Reflect(context);
RegisterBuilderResponse::Reflect(context);
CreateJobsRequest::Reflect(context);
CreateJobsResponse::Reflect(context);
ProcessJobRequest::Reflect(context);
ProcessJobResponse::Reflect(context);
BuilderHelloRequest::Reflect(context);
BuilderHelloResponse::Reflect(context);
CreateJobsNetRequest::Reflect(context);
CreateJobsNetResponse::Reflect(context);
ProcessJobNetRequest::Reflect(context);
ProcessJobNetResponse::Reflect(context);
}
void InitializeSerializationContext()
@@ -1263,24 +1254,6 @@ namespace AssetBuilderSDK
}
}
void RegisterBuilderRequest::Reflect(AZ::ReflectContext* context)
{
if (AZ::SerializeContext* serializeContext = azrtti_cast<AZ::SerializeContext*>(context))
{
serializeContext->Class<RegisterBuilderRequest>()->
Version(1)->
Field("FilePath", &RegisterBuilderRequest::m_filePath);
}
if (AZ::BehaviorContext* behaviorContext = azrtti_cast<AZ::BehaviorContext*>(context))
{
behaviorContext->Class<RegisterBuilderRequest>("RegisterBuilderRequest")
->Attribute(AZ::Script::Attributes::Scope, AZ::Script::Attributes::ScopeFlags::Automation)
->Attribute(AZ::Script::Attributes::Module, "asset.builder")
->Property("filePath", BehaviorValueProperty(&RegisterBuilderRequest::m_filePath));
}
}
void AssetBuilderDesc::Reflect(AZ::ReflectContext* context)
{
if (AZ::SerializeContext* serializeContext = azrtti_cast<AZ::SerializeContext*>(context))
@@ -1313,25 +1286,6 @@ namespace AssetBuilderSDK
}
}
void RegisterBuilderResponse::Reflect(AZ::ReflectContext* context)
{
if (AZ::SerializeContext* serializeContext = azrtti_cast<AZ::SerializeContext*>(context))
{
serializeContext->Class<RegisterBuilderResponse>()
->Version(1)
->Field("Asset Builder Desc List", &RegisterBuilderResponse::m_assetBuilderDescList);
}
if (AZ::BehaviorContext* behaviorContext = azrtti_cast<AZ::BehaviorContext*>(context))
{
behaviorContext->Class<RegisterBuilderResponse>("RegisterBuilderResponse")
->Attribute(AZ::Script::Attributes::Scope, AZ::Script::Attributes::ScopeFlags::Automation)
->Attribute(AZ::Script::Attributes::Module, "asset.builder")
->Constructor()
->Property("assetBuilderDescList", BehaviorValueProperty(&RegisterBuilderResponse::m_assetBuilderDescList));
}
}
bool CreateJobsResponse::Succeeded() const
{
return m_result == CreateJobsResultCode::Success;
@@ -1362,128 +1316,6 @@ namespace AssetBuilderSDK
}
}
void BuilderHelloRequest::Reflect(AZ::ReflectContext* context)
{
auto serialize = azrtti_cast<AZ::SerializeContext*>(context);
if (serialize)
{
serialize->Class<BuilderHelloRequest>()
->Version(1)
->Field("UUID", &BuilderHelloRequest::m_uuid);
}
}
unsigned int BuilderHelloRequest::MessageType()
{
static unsigned int messageType = AZ_CRC("AssetBuilderSDK::BuilderHelloRequest", 0x213a7248);
return messageType;
}
unsigned int BuilderHelloRequest::GetMessageType() const
{
return MessageType();
}
void BuilderHelloResponse::Reflect(AZ::ReflectContext* context)
{
auto serialize = azrtti_cast<AZ::SerializeContext*>(context);
if (serialize)
{
serialize->Class<BuilderHelloResponse>()
->Version(1)
->Field("Accepted", &BuilderHelloResponse::m_accepted)
->Field("UUID", &BuilderHelloResponse::m_uuid);
}
}
unsigned int BuilderHelloResponse::GetMessageType() const
{
return BuilderHelloRequest::MessageType();
}
//////////////////////////////////////////////////////////////////////////
void CreateJobsNetRequest::Reflect(AZ::ReflectContext* context)
{
auto serialize = azrtti_cast<AZ::SerializeContext*>(context);
if (serialize)
{
serialize->Class<CreateJobsNetRequest>()
->Version(1)
->Field("Request", &CreateJobsNetRequest::m_request);
}
}
unsigned int CreateJobsNetRequest::MessageType()
{
static unsigned int messageType = AZ_CRC("AssetBuilderSDK::CreateJobsNetRequest", 0xc48209c0);
return messageType;
}
unsigned int CreateJobsNetRequest::GetMessageType() const
{
return MessageType();
}
void CreateJobsNetResponse::Reflect(AZ::ReflectContext* context)
{
auto serialize = azrtti_cast<AZ::SerializeContext*>(context);
if (serialize)
{
serialize->Class<CreateJobsNetResponse>()
->Version(1)
->Field("Response", &CreateJobsNetResponse::m_response);
}
}
unsigned int CreateJobsNetResponse::GetMessageType() const
{
return CreateJobsNetRequest::MessageType();
}
void ProcessJobNetRequest::Reflect(AZ::ReflectContext* context)
{
auto serialize = azrtti_cast<AZ::SerializeContext*>(context);
if (serialize)
{
serialize->Class<ProcessJobNetRequest>()
->Version(1)
->Field("Request", &ProcessJobNetRequest::m_request);
}
}
unsigned int ProcessJobNetRequest::MessageType()
{
static unsigned int messageType = AZ_CRC("AssetBuilderSDK::ProcessJobNetRequest", 0x479f340f);
return messageType;
}
unsigned int ProcessJobNetRequest::GetMessageType() const
{
return MessageType();
}
void ProcessJobNetResponse::Reflect(AZ::ReflectContext* context)
{
auto serialize = azrtti_cast<AZ::SerializeContext*>(context);
if (serialize)
{
serialize->Class<ProcessJobNetResponse>()
->Version(1)
->Field("Response", &ProcessJobNetResponse::m_response);
}
}
unsigned int ProcessJobNetResponse::GetMessageType() const
{
return ProcessJobNetRequest::MessageType();
}
JobDependency::JobDependency(const AZStd::string& jobKey, const AZStd::string& platformIdentifier, const JobDependencyType& type, const SourceFileDependency& sourceFile)
: m_jobKey(jobKey)
, m_platformIdentifier(platformIdentifier)
Code/Tools/AssetProcessor/AssetBuilderSDK/AssetBuilderSDK/AssetBuilderSDK.h
+2 -162
View File
@@ -464,35 +464,6 @@ namespace AssetBuilderSDK
AZStd::string m_platformIdentifier;
};
//! RegisterBuilderRequest contains input data that will be sent by the AssetProcessor to the builder during the startup registration phase
struct RegisterBuilderRequest
{
AZ_CLASS_ALLOCATOR(RegisterBuilderRequest, AZ::SystemAllocator, 0);
AZ_TYPE_INFO(RegisterBuilderRequest, "{7C6C5198-4766-42B8-9A1E-48479CE2F5EA}");
AZStd::string m_filePath;
RegisterBuilderRequest() {}
explicit RegisterBuilderRequest(const AZStd::string& filePath)
: m_filePath(filePath)
{
}
static void Reflect(AZ::ReflectContext* context);
};
//! INTERNAL USE ONLY - RegisterBuilderResponse contains registration data that will be sent by the builder to the AssetProcessor in response to RegisterBuilderRequest
struct RegisterBuilderResponse
{
AZ_CLASS_ALLOCATOR(RegisterBuilderResponse, AZ::SystemAllocator, 0);
AZ_TYPE_INFO(RegisterBuilderResponse, "{0AE5583F-C763-410E-BA7F-78BD90546C01}");
AZStd::vector<AssetBuilderDesc> m_assetBuilderDescList;
static void Reflect(AZ::ReflectContext* context);
};
/**
* This tells you about a platform in your CreateJobsRequest or your ProcessJobRequest
*/
@@ -756,99 +727,9 @@ namespace AssetBuilderSDK
static void Reflect(AZ::ReflectContext* context);
};
//! BuilderHelloRequest is sent by an AssetBuilder that is attempting to connect to the AssetProcessor to register itself as a worker
class BuilderHelloRequest : public AzFramework::AssetSystem::BaseAssetProcessorMessage
{
public:
AZ_CLASS_ALLOCATOR(BuilderHelloRequest, AZ::OSAllocator, 0);
AZ_RTTI(BuilderHelloRequest, "{5fab5962-a1d8-42a5-bf7a-fb1a8c5a9588}", BaseAssetProcessorMessage);
static void Reflect(AZ::ReflectContext* context);
static unsigned int MessageType();
unsigned int GetMessageType() const override;
//! Unique ID assigned to this builder to identify it
AZ::Uuid m_uuid = AZ::Uuid::CreateNull();
};
//! BuilderHelloResponse contains the AssetProcessor's response to a builder connection attempt, indicating if it is accepted and the ID that it was assigned
class BuilderHelloResponse : public AzFramework::AssetSystem::BaseAssetProcessorMessage
{
public:
AZ_CLASS_ALLOCATOR(BuilderHelloResponse, AZ::OSAllocator, 0);
AZ_RTTI(BuilderHelloResponse, "{5f3d7c11-6639-4c6f-980a-32be546903c2}", BaseAssetProcessorMessage);
static void Reflect(AZ::ReflectContext* context);
unsigned int GetMessageType() const override;
//! Indicates if the builder was accepted by the AP
bool m_accepted = false;
//! Unique ID assigned to the builder. If the builder isn't a local process, this is the ID assigned by the AP
AZ::Uuid m_uuid = AZ::Uuid::CreateNull();
};
class CreateJobsNetRequest : public AzFramework::AssetSystem::BaseAssetProcessorMessage
{
public:
AZ_CLASS_ALLOCATOR(CreateJobsNetRequest, AZ::OSAllocator, 0);
AZ_RTTI(CreateJobsNetRequest, "{97fa717d-3a09-4d21-95c6-b2eafd773f1c}", BaseAssetProcessorMessage);
static void Reflect(AZ::ReflectContext* context);
static unsigned int MessageType();
unsigned int GetMessageType() const override;
CreateJobsRequest m_request;
};
class CreateJobsNetResponse : public AzFramework::AssetSystem::BaseAssetProcessorMessage
{
public:
AZ_CLASS_ALLOCATOR(CreateJobsNetResponse, AZ::OSAllocator, 0);
AZ_RTTI(CreateJobsNetResponse, "{b2c7c2d3-b60e-4b27-b699-43e0ba991c33}", BaseAssetProcessorMessage);
static void Reflect(AZ::ReflectContext* context);
unsigned int GetMessageType() const override;
CreateJobsResponse m_response;
};
class ProcessJobNetRequest : public AzFramework::AssetSystem::BaseAssetProcessorMessage
{
public:
AZ_CLASS_ALLOCATOR(ProcessJobNetRequest, AZ::OSAllocator, 0);
AZ_RTTI(ProcessJobNetRequest, "{05288de1-020b-48db-b9de-715f17284efa}", BaseAssetProcessorMessage);
static void Reflect(AZ::ReflectContext* context);
static unsigned int MessageType();
unsigned int GetMessageType() const override;
ProcessJobRequest m_request;
};
class ProcessJobNetResponse : public AzFramework::AssetSystem::BaseAssetProcessorMessage
{
public:
AZ_CLASS_ALLOCATOR(ProcessJobNetResponse, AZ::OSAllocator, 0);
AZ_RTTI(ProcessJobNetResponse, "{26ddf882-246c-4cfb-912f-9b8e389df4f6}", BaseAssetProcessorMessage);
static void Reflect(AZ::ReflectContext* context);
unsigned int GetMessageType() const override;
ProcessJobResponse m_response;
};
//! JobCancelListener can be used by builders in their processJob method to listen for job cancellation request.
//! The address of this listener is the jobid which can be found in the process job request.
@@ -935,44 +816,3 @@ namespace AZ
AZ_TYPE_INFO_SPECIALIZE(AssetBuilderSDK::ProductPathDependencyType, "{EF77742B-9627-4072-B431-396AA7183C80}");
AZ_TYPE_INFO_SPECIALIZE(AssetBuilderSDK::SourceFileDependency::SourceFileDependencyType, "{BE9C8805-DB17-4500-944A-EB33FD0BE347}");
}
//! This macro should be used by every AssetBuilder to register itself,
//! AssetProcessor uses these exported function to identify whether a dll is an Asset Builder or not
//! If you want something highly custom you can do these entry points yourself instead of using the macro.
#define REGISTER_ASSETBUILDER \
extern void BuilderOnInit(); \
extern void BuilderDestroy(); \
extern void BuilderRegisterDescriptors(); \
extern void BuilderAddComponents(AZ::Entity * entity); \
extern "C" \
{ \
AZ_DLL_EXPORT int IsAssetBuilder() \
{ \
return 0; \
} \
\
AZ_DLL_EXPORT void InitializeModule(AZ::EnvironmentInstance sharedEnvironment) \
{ \
AZ::Environment::Attach(sharedEnvironment); \
BuilderOnInit(); \
} \
\
AZ_DLL_EXPORT void UninitializeModule() \
{ \
BuilderDestroy(); \
AZ::Environment::Detach(); \
} \
\
AZ_DLL_EXPORT void ModuleRegisterDescriptors() \
{ \
BuilderRegisterDescriptors(); \
} \
\
AZ_DLL_EXPORT void ModuleAddComponents(AZ::Entity * entity) \
{ \
BuilderAddComponents(entity); \
} \
}
// confusion-reducing note: above end-brace is part of the macro, not a namespace
Code/Tools/AssetProcessor/CMakeLists.txt
+1
View File
@@ -42,6 +42,7 @@ ly_add_target(
AZ::AzQtComponents
AZ::AzToolsFramework
AZ::AssetBuilderSDK
AZ::AssetBuilder.Static
${additional_dependencies}
RUNTIME_DEPENDENCIES
AZ::AssetBuilder
Code/Tools/AssetProcessor/native/assetprocessor.h
+8 -8
View File
@@ -85,7 +85,7 @@ namespace AssetProcessor
enum AssetCatalogStatus
{
RequiresSaving,
RequiresSaving,
UpToDate
};
@@ -213,11 +213,11 @@ namespace AssetProcessor
bool m_critical = false;
int m_priority = -1;
// indicates whether we need to check the server first for the outputs of this job
// indicates whether we need to check the server first for the outputs of this job
// before we start processing locally
bool m_checkServer = false;
// Indicates whether this job needs to be processed irrespective of whether its fingerprint got modified or not.
// Indicates whether this job needs to be processed irrespective of whether its fingerprint got modified or not.
bool m_autoProcessJob = false;
AssetBuilderSDK::AssetBuilderDesc m_assetBuilderDesc;
@@ -251,9 +251,9 @@ namespace AssetProcessor
JobDetails() = default;
};
//! JobDesc struct is used for identifying jobs that need to be processed again
//! because of job dependency declared on them by other jobs
//! JobDesc struct is used for identifying jobs that need to be processed again
//! because of job dependency declared on them by other jobs
struct JobDesc
{
AZStd::string m_databaseSourceName;
@@ -283,7 +283,7 @@ namespace AssetProcessor
}
};
//! JobIndentifier is an internal structure that store all the data that can uniquely identify a job
//! JobIndentifier is an internal structure that store all the data that can uniquely identify a job
struct JobIndentifier
{
JobDesc m_jobDesc;
Code/Tools/AssetProcessor/native/ui/MainWindow.cpp
+15 -15
View File
@@ -165,7 +165,7 @@ void MainWindow::Activate()
ui->connectionTreeView->header()->resizeSection(ConnectionManager::PortColumn, 60);
ui->connectionTreeView->header()->resizeSection(ConnectionManager::PlatformColumn, 60);
ui->connectionTreeView->header()->resizeSection(ConnectionManager::AutoConnectColumn, 60);
ui->connectionTreeView->header()->setStretchLastSection(false);
connect(ui->connectionTreeView->selectionModel(), &QItemSelectionModel::selectionChanged, this, &MainWindow::OnConnectionSelectionChanged);
@@ -189,12 +189,12 @@ void MainWindow::Activate()
ui->allowListAllowedListConnectionsListView->setModel(&m_allowedListAddresses);
connect(ui->allowedListRejectedConnectionsListView, &QListView::clicked, this, &MainWindow::OnRejectedConnectionsListViewClicked);
ui->allowedListRejectedConnectionsListView->setModel(&m_rejectedAddresses);
connect(ui->allowedListEnableCheckBox, &QCheckBox::toggled, this, &MainWindow::OnAllowedListCheckBoxToggled);
connect(ui->allowedListAddHostNameToolButton, &QToolButton::clicked, this, &MainWindow::OnAddHostNameAllowedListButtonClicked);
connect(ui->allowedListAddIPToolButton, &QPushButton::clicked, this, &MainWindow::OnAddIPAllowedListButtonClicked);
connect(ui->allowedListToAllowedListToolButton, &QPushButton::clicked, this, &MainWindow::OnToAllowedListButtonClicked);
connect(ui->allowedListToRejectedListToolButton, &QToolButton::clicked, this, &MainWindow::OnToRejectedListButtonClicked);
@@ -204,7 +204,7 @@ void MainWindow::Activate()
QRegExpValidator* hostNameValidator = new QRegExpValidator(validHostName, this);
ui->allowedListAddHostNameLineEdit->setValidator(hostNameValidator);
QRegExpValidator* ipValidator = new QRegExpValidator(validIP, this);
ui->allowedListAddIPLineEdit->setValidator(ipValidator);
@@ -235,7 +235,7 @@ void MainWindow::Activate()
m_logSortFilterProxy->setSourceModel(m_logsModel);
m_logSortFilterProxy->setFilterKeyColumn(AzToolsFramework::Logging::LogTableModel::ColumnMessage);
m_logSortFilterProxy->setFilterCaseSensitivity(Qt::CaseInsensitive);
ui->jobLogTableView->setModel(m_logSortFilterProxy);
ui->jobLogTableView->setItemDelegate(new AzToolsFramework::Logging::LogTableItemDelegate(ui->jobLogTableView));
ui->jobLogTableView->setExpandOnSelection();
@@ -400,7 +400,7 @@ void MainWindow::Activate()
bool zeroAnalysisModeFromSettings = settings.value("EnableZeroAnalysis", QVariant(true)).toBool();
settings.endGroup();
QObject::connect(ui->modtimeSkippingCheckBox, &QCheckBox::stateChanged, this,
QObject::connect(ui->modtimeSkippingCheckBox, &QCheckBox::stateChanged, this,
[this](int newCheckState)
{
bool newOption = newCheckState == Qt::Checked ? true : false;
@@ -543,7 +543,7 @@ void MainWindow::OnAddConnection(bool /*checked*/)
m_guiApplicationManager->GetConnectionManager()->addUserConnection();
}
void MainWindow::OnAllowedListConnectionsListViewClicked()
void MainWindow::OnAllowedListConnectionsListViewClicked()
{
ui->allowedListRejectedConnectionsListView->clearSelection();
}
@@ -553,7 +553,7 @@ void MainWindow::OnRejectedConnectionsListViewClicked()
ui->allowListAllowedListConnectionsListView->clearSelection();
}
void MainWindow::OnAllowedListCheckBoxToggled()
void MainWindow::OnAllowedListCheckBoxToggled()
{
if (!ui->allowedListEnableCheckBox->isChecked())
{
@@ -588,7 +588,7 @@ void MainWindow::OnAllowedListCheckBoxToggled()
ui->allowedListToAllowedListToolButton->setEnabled(true);
ui->allowedListToRejectedListToolButton->setEnabled(true);
}
m_guiApplicationManager->GetConnectionManager()->AllowedListingEnabled(ui->allowedListEnableCheckBox->isChecked());
}
@@ -858,7 +858,7 @@ void MainWindow::OnAssetProcessorStatusChanged(const AssetProcessor::AssetProces
text = tr("Working, analyzing jobs remaining %1, processing jobs remaining %2...").arg(m_createJobCount).arg(m_processJobsCount);
ui->timerContainerWidget->setVisible(false);
ui->productAssetDetailsPanel->SetScanQueueEnabled(false);
IntervalAssetTabFilterRefresh();
}
else
@@ -877,7 +877,7 @@ void MainWindow::OnAssetProcessorStatusChanged(const AssetProcessor::AssetProces
break;
case AssetProcessorStatus::Processing_Jobs:
CheckStartProcessTimers();
m_processJobsCount = entry.m_count;
m_processJobsCount = entry.m_count;
if (m_processJobsCount + m_createJobCount > 0)
{
@@ -983,7 +983,7 @@ void MainWindow::ApplyConfig()
ui->jobLogTableView->header()->resizeSection(AzToolsFramework::Logging::LogTableModel::ColumnType, m_config.logTypeColumnWidth);
}
MainWindow::LogSortFilterProxy::LogSortFilterProxy(QObject* parentOjbect) : QSortFilterProxyModel(parentOjbect)
MainWindow::LogSortFilterProxy::LogSortFilterProxy(QObject* parentOjbect) : QSortFilterProxyModel(parentOjbect)
{
}
@@ -1302,7 +1302,7 @@ void MainWindow::ShowJobViewContextMenu(const QPoint& pos)
ui->sourceAssetDetailsPanel->GoToSource(item->m_elementId.GetInputAssetName().toUtf8().constData());
});
QString productMenuTitle(tr("View product asset..."));
QString productMenuTitle(tr("View product asset..."));
if (item->m_jobState != AzToolsFramework::AssetSystem::JobStatus::Completed)
{
QString disabledActionTooltip(tr("Only completed jobs are available in the Assets tab."));
@@ -1610,7 +1610,7 @@ void MainWindow::ShowProductAssetContextMenu(const QPoint& pos)
{
AzQtComponents::ShowFileOnDesktop(pathToProduct.GetValue());
}
});
QString fileOrFolder(cachedAsset->getChildCount() > 0 ? tr("folder") : tr("file"));
Code/Tools/AssetProcessor/native/utilities/ApplicationManager.cpp
-47
View File
@@ -231,44 +231,6 @@ bool ApplicationManager::InitiatedShutdown() const
return m_duringShutdown;
}
void ApplicationManager::GetExternalBuilderFileList(QStringList& externalBuilderModules)
{
externalBuilderModules.clear();
static const char* builder_folder_name = "Builders";
// LY_ASSET_BUILDERS is defined by the CMakeLists.txt. The asset builders add themselves to a variable that
// is populated to allow selective building of those asset builder targets.
// This allows left over Asset builders in the output directory to not be loaded by the AssetProcessor
#if !defined(LY_ASSET_BUILDERS)
#error LY_ASSET_BUILDERS was not defined for ApplicationManager.cpp
#endif
QDir builderDir = QDir::toNativeSeparators(QString(this->m_frameworkApp.GetExecutableFolder()));
builderDir.cd(QString(builder_folder_name));
if (builderDir.exists())
{
AZStd::vector<AZStd::string> tokens;
AZ::StringFunc::Tokenize(AZStd::string_view(LY_ASSET_BUILDERS), tokens, ',');
AZStd::string builderLibrary;
for (const AZStd::string& token : tokens)
{
QString assetBuilderPath(token.c_str());
if (builderDir.exists(assetBuilderPath))
{
externalBuilderModules.push_back(builderDir.absoluteFilePath(assetBuilderPath));
}
}
}
if (externalBuilderModules.empty())
{
AZ_TracePrintf(AssetProcessor::ConsoleChannel, "AssetProcessor was unable to locate any external builders\n");
}
}
QDir ApplicationManager::GetSystemRoot() const
{
return m_systemRoot;
@@ -459,15 +421,6 @@ void ApplicationManager::PopulateApplicationDependencies()
m_filesOfInterest.push_back(dir.absoluteFilePath(pathWithPlatformExtension));
}
// Get the external builder modules to add to the files of interest
QStringList builderModuleFileList;
GetExternalBuilderFileList(builderModuleFileList);
for (const QString& builderModuleFile : builderModuleFileList)
{
m_filesOfInterest.push_back(builderModuleFile);
}
QDir assetRoot;
AssetUtilities::ComputeAssetRoot(assetRoot);
Code/Tools/AssetProcessor/native/utilities/ApplicationManager.h
+1 -4
View File
@@ -139,7 +139,7 @@ protected:
void RegisterObjectForQuit(QObject* source, bool insertInFront = false);
bool NeedRestart() const;
void addRunningThread(AssetProcessor::ThreadWorker* thread);
template<class BuilderClass>
void RegisterInternalBuilder(const QString& builderName);
@@ -151,9 +151,6 @@ protected:
bool m_duringStartup = true;
AssetProcessorAZApplication m_frameworkApp;
QCoreApplication* m_qApp = nullptr;
//! Get the list of external builder files for this asset processor
void GetExternalBuilderFileList(QStringList& externalBuilderModules);
virtual void Reflect() = 0;
virtual const char* GetLogBaseName() = 0;
Code/Tools/AssetProcessor/native/utilities/ApplicationManagerBase.cpp
+145 -214
View File
@@ -25,6 +25,7 @@
#include <native/InternalBuilders/SettingsRegistryBuilder.h>
#include <AzToolsFramework/Application/Ticker.h>
#include <AzToolsFramework/ToolsFileUtils/ToolsFileUtils.h>
#include <AssetBuilder/AssetBuilderStatic.h>
#include <iostream>
@@ -32,9 +33,6 @@
#include <QElapsedTimer>
//! Amount of time to wait between checking the status of the AssetBuilder process
static const int s_MaximumSleepTimeMS = 10;
//! CreateJobs will wait up to 2 minutes before timing out
//! This shouldn't need to be so high but very large slices can take a while to process currently
//! This should be reduced down to something more reasonable after slice jobs are sped up
@@ -64,6 +62,7 @@ ApplicationManagerBase::~ApplicationManagerBase()
AZ::Debug::TraceMessageBus::Handler::BusDisconnect();
AssetProcessor::AssetBuilderRegistrationBus::Handler::BusDisconnect();
AssetBuilderSDK::AssetBuilderBus::Handler::BusDisconnect();
AssetProcessor::AssetBuilderInfoBus::Handler::BusDisconnect();
if (m_settingsRegistryBuilder)
{
@@ -192,7 +191,7 @@ void ApplicationManagerBase::InitAssetProcessorManager()
{
m_assetProcessorManager->SetEnableModtimeSkippingFeature(true);
}
if (commandLine->HasSwitch(Command_enableQueryLogging.m_switch))
{
m_assetProcessorManager->SetQueryLogging(true);
@@ -206,7 +205,7 @@ void ApplicationManagerBase::InitAssetProcessorManager()
{
m_dependencyScanPattern = commandLine->GetSwitchValue(Command_dsp.m_switch, 0).c_str();
}
m_fileDependencyScanPattern = "*";
if (commandLine->HasSwitch(Command_fileDependencyScanPattern.m_switch))
@@ -327,7 +326,7 @@ void ApplicationManagerBase::InitAssetCatalog()
AssetProcessor::AssetCatalog* catalog = new AssetCatalog(assetCatalogHelper, m_platformConfiguration);
// Using a direct connection so we know the catalog has been updated before continuing on with code might depend on the asset being in the catalog
connect(m_assetProcessorManager, &AssetProcessorManager::AssetMessage, catalog, &AssetCatalog::OnAssetMessage, Qt::DirectConnection);
connect(m_assetProcessorManager, &AssetProcessorManager::AssetMessage, catalog, &AssetCatalog::OnAssetMessage, Qt::DirectConnection);
connect(m_assetProcessorManager, &AssetProcessorManager::SourceQueued, catalog, &AssetCatalog::OnSourceQueued);
connect(m_assetProcessorManager, &AssetProcessorManager::SourceFinished, catalog, &AssetCatalog::OnSourceFinished);
connect(m_assetProcessorManager, &AssetProcessorManager::PathDependencyResolved, catalog, &AssetCatalog::OnDependencyResolved);
@@ -379,12 +378,12 @@ void ApplicationManagerBase::InitAssetScanner()
QObject::connect(m_assetScanner, &AssetScanner::FilesFound, [this](QSet<AssetFileInfo> files) { m_fileStateCache->AddInfoSet(files); });
QObject::connect(m_assetScanner, &AssetScanner::FoldersFound, [this](QSet<AssetFileInfo> files) { m_fileStateCache->AddInfoSet(files); });
QObject::connect(m_assetScanner, &AssetScanner::ExcludedFound, [this](QSet<AssetFileInfo> files) { m_fileStateCache->AddInfoSet(files); });
// file table
QObject::connect(m_assetScanner, &AssetScanner::AssetScanningStatusChanged, m_fileProcessor.get(), &FileProcessor::OnAssetScannerStatusChange);
QObject::connect(m_assetScanner, &AssetScanner::FilesFound, m_fileProcessor.get(), &FileProcessor::AssessFilesFromScanner);
QObject::connect(m_assetScanner, &AssetScanner::FoldersFound, m_fileProcessor.get(), &FileProcessor::AssessFoldersFromScanner);
}
void ApplicationManagerBase::DestroyAssetScanner()
@@ -591,6 +590,51 @@ void ApplicationManagerBase::InitConnectionManager()
}, AZStd::placeholders::_1, AZStd::placeholders::_2, AZStd::placeholders::_3, AZStd::placeholders::_4)
);
m_connectionManager->RegisterService(
AssetBuilder::BuilderRegistrationRequest::MessageType,
[this](unsigned int /*connId*/, unsigned int /*type*/, unsigned int /*serial*/, QByteArray payload, QString)
{
AssetBuilder::BuilderRegistrationRequest registrationRequest;
if (m_builderRegistrationComplete)
{
return;
}
m_builderRegistrationComplete = true;
if (AssetProcessor::UnpackMessage(payload, registrationRequest))
{
for (const auto& builder : registrationRequest.m_builders)
{
AssetBuilderSDK::AssetBuilderDesc desc;
desc.m_name = builder.m_name;
desc.m_patterns = builder.m_patterns;
desc.m_version = builder.m_version;
desc.m_analysisFingerprint = builder.m_analysisFingerprint;
desc.m_flags = builder.m_flags;
desc.m_busId = builder.m_busId;
desc.m_flagsByJobKey = builder.m_flagsByJobKey;
desc.m_productsToKeepOnFailure = builder.m_productsToKeepOnFailure;
// Builders registered this way are always external builders
desc.m_builderType = AssetBuilderSDK::AssetBuilderDesc::AssetBuilderType::External;
RegisterBuilderInformation(desc);
}
QTimer::singleShot(
0, this,
[this]()
{
if (!PostActivate())
{
QuitRequested();
}
});
}
});
//You can get Asset Processor Current State
using AzFramework::AssetSystem::RequestAssetProcessorStatus;
auto GetState = [this](unsigned int connId, unsigned int, unsigned int serial, QByteArray payload, QString)
@@ -633,11 +677,11 @@ void ApplicationManagerBase::InitConnectionManager()
AssetProcessorPlatformStatusRequest requestMessage;
if (AssetProcessor::UnpackMessage(payload, requestMessage))
{
AzToolsFramework::AssetSystemRequestBus::BroadcastResult(responseMessage.m_isPlatformEnabled,
AzToolsFramework::AssetSystemRequestBus::BroadcastResult(responseMessage.m_isPlatformEnabled,
&AzToolsFramework::AssetSystemRequestBus::Events::IsAssetPlatformEnabled, requestMessage.m_platform.c_str());
}
AssetProcessor::ConnectionBus::Event(connId,
AssetProcessor::ConnectionBus::Event(connId,
&AssetProcessor::ConnectionBus::Events::SendResponse, serial, responseMessage);
});
@@ -653,11 +697,11 @@ void ApplicationManagerBase::InitConnectionManager()
if (AssetProcessor::UnpackMessage(payload, requestMessage))
{
const char* platformIdentifier = requestMessage.m_platform.c_str();
responseMessage.m_numberOfPendingJobs =
responseMessage.m_numberOfPendingJobs =
GetRCController()->NumberOfPendingJobsPerPlatform(platformIdentifier);
}
AssetProcessor::ConnectionBus::Event(connId,
AssetProcessor::ConnectionBus::Event(connId,
&AssetProcessor::ConnectionBus::Events::SendResponse, serial, responseMessage);
});
}
@@ -696,7 +740,7 @@ void ApplicationManagerBase::InitAssetRequestHandler(AssetProcessor::AssetReques
QObject::connect(GetAssetProcessorManager(), &AssetProcessorManager::SendAssetExistsResponse, m_assetRequestHandler, &AssetRequestHandler::OnRequestAssetExistsResponse);
QObject::connect(GetAssetProcessorManager(), &AssetProcessorManager::FenceFileDetected, m_assetRequestHandler, &AssetRequestHandler::OnFenceFileDetected);
// connect the Asset Request Handler to RC:
QObject::connect(m_assetRequestHandler, &AssetRequestHandler::RequestCompileGroup, GetRCController(), &RCController::OnRequestCompileGroup);
QObject::connect(m_assetRequestHandler, &AssetRequestHandler::RequestEscalateAssetBySearchTerm, GetRCController(), &RCController::OnEscalateJobsBySearchTerm);
@@ -840,14 +884,6 @@ bool ApplicationManagerBase::Run()
return false;
}
bool startedSuccessfully = true;
if (!PostActivate())
{
QuitRequested();
startedSuccessfully = false;
}
AZ_Printf(AssetProcessor::ConsoleChannel, "Asset Processor Batch Processing Started.\n");
AZ_Printf(AssetProcessor::ConsoleChannel, "-----------------------------------------\n");
QElapsedTimer allAssetsProcessingTimer;
@@ -867,7 +903,7 @@ bool ApplicationManagerBase::Run()
RemoveOldTempFolders();
Destroy();
return (startedSuccessfully && FailedAssetsCount() == 0);
return FailedAssetsCount() == 0;
}
void ApplicationManagerBase::HandleFileRelocation() const
@@ -899,7 +935,7 @@ void ApplicationManagerBase::HandleFileRelocation() const
while(!m_sourceControlReady)
{
// We need to wait for source control to be ready before continuing
if (printCounter % 10 == 0)
{
AZ_TracePrintf(AssetProcessor::ConsoleChannel, "Waiting for Source Control connection\n");
@@ -1129,7 +1165,7 @@ void ApplicationManagerBase::CheckForIdle()
TryScanProductDependencies();
TryHandleFileRelocation();
// since we are shutting down, we save the registry and then we quit.
AZ_Printf(AssetProcessor::ConsoleChannel, "No assets remain in the build queue. Saving the catalog, and then shutting down.\n");
// stop accepting any further idle messages, as we will shut down - don't want this function to repeat!
@@ -1173,7 +1209,7 @@ void ApplicationManagerBase::InitBuilderManager()
{
m_builderManager->ConnectionLost(connId);
});
}
void ApplicationManagerBase::ShutdownBuilderManager()
@@ -1207,7 +1243,7 @@ void ApplicationManagerBase::ShutDownAssetDatabase()
AzToolsFramework::AssetDatabase::AssetDatabaseRequests::Bus::Handler::BusDisconnect();
}
void ApplicationManagerBase::InitFileProcessor()
void ApplicationManagerBase::InitFileProcessor()
{
AssetProcessor::ThreadController<AssetProcessor::FileProcessor>* fileProcessorHelper = new AssetProcessor::ThreadController<AssetProcessor::FileProcessor>();
@@ -1298,22 +1334,13 @@ bool ApplicationManagerBase::Activate()
}
InitBuilderConfiguration();
m_isCurrentlyLoadingGems = true;
if (!ActivateModules())
{
// ActivateModules reports any errors it encounters.
m_isCurrentlyLoadingGems = false;
return false;
}
m_isCurrentlyLoadingGems = false;
PopulateApplicationDependencies();
InitAssetProcessorManager();
AssetBuilderSDK::InitializeSerializationContext();
AssetBuilderSDK::InitializeBehaviorContext();
AssetBuilder::InitializeSerializationContext();
InitFileStateCache();
InitFileProcessor();
@@ -1341,7 +1368,7 @@ bool ApplicationManagerBase::Activate()
RegisterObjectForQuit(m_rcController);
m_connectionsToRemoveOnShutdown << QObject::connect(
m_assetProcessorManager, &AssetProcessor::AssetProcessorManager::AssetProcessorManagerIdleState,
m_assetProcessorManager, &AssetProcessor::AssetProcessorManager::AssetProcessorManagerIdleState,
this, [this](bool state)
{
if (state)
@@ -1362,7 +1389,7 @@ bool ApplicationManagerBase::Activate()
});
m_connectionsToRemoveOnShutdown << QObject::connect(
this, &ApplicationManagerBase::CheckAssetProcessorManagerIdleState,
this, &ApplicationManagerBase::CheckAssetProcessorManagerIdleState,
m_assetProcessorManager, &AssetProcessor::AssetProcessorManager::CheckAssetProcessorIdleState);
MakeActivationConnections();
@@ -1376,6 +1403,22 @@ bool ApplicationManagerBase::Activate()
return false;
}
}
AssetProcessor::AssetProcessorStatusEntry entry(AssetProcessor::AssetProcessorStatus::Initializing_Builders, 0, QString());
Q_EMIT AssetProcessorStatusChanged(entry);
AZStd::thread_desc desc;
desc.m_name = "Builder Component Registration";
AZStd::thread builderRegistrationThread(
desc,
[]()
{
AssetProcessor::BuilderRef builder;
AssetProcessor::BuilderManagerBus::BroadcastResult(builder, &AssetProcessor::BuilderManagerBus::Events::GetBuilder, true);
});
builderRegistrationThread.detach();
return true;
}
@@ -1384,11 +1427,6 @@ bool ApplicationManagerBase::PostActivate()
m_connectionManager->LoadConnections();
InitializeInternalBuilders();
if (!InitializeExternalBuilders())
{
AZ_Error("AssetProcessor", false, "AssetProcessor is closing. Failed to initialize and load all the external builders. Please ensure that Builders_Temp directory is not read-only. Please see log for more information.\n");
return false;
}
Q_EMIT OnBuildersRegistered();
@@ -1401,7 +1439,7 @@ bool ApplicationManagerBase::PostActivate()
AZ::SystemTickBus::Broadcast(&AZ::SystemTickEvents::OnSystemTick);
});
// now that everything is up and running, we start scanning. Before this, we don't want file events to start percolating through the
// now that everything is up and running, we start scanning. Before this, we don't want file events to start percolating through the
// asset system.
GetAssetScanner()->StartScan();
@@ -1425,124 +1463,20 @@ bool ApplicationManagerBase::InitializeInternalBuilders()
return result;
}
bool ApplicationManagerBase::InitializeExternalBuilders()
{
AssetProcessor::AssetProcessorStatusEntry entry(AssetProcessor::AssetProcessorStatus::Initializing_Builders);
Q_EMIT AssetProcessorStatusChanged(entry);
QCoreApplication::processEvents(QEventLoop::AllEvents);
// Get the list of external build modules (full paths)
QStringList fileList;
GetExternalBuilderFileList(fileList);
for (const QString& filePath : fileList)
{
if (QLibrary::isLibrary(filePath))
{
AssetProcessor::ExternalModuleAssetBuilderInfo* externalAssetBuilderInfo = new AssetProcessor::ExternalModuleAssetBuilderInfo(filePath);
AssetProcessor::AssetBuilderType assetBuilderType = externalAssetBuilderInfo->Load();
AZ_TracePrintf(AssetProcessor::ConsoleChannel, "AssetProcessor is loading library %s\n", filePath.toUtf8().data());
if (assetBuilderType == AssetProcessor::AssetBuilderType::None)
{
AZ_Warning(AssetProcessor::DebugChannel, false, "Non-builder DLL was found in Builders directory %s, skipping. \n", filePath.toUtf8().data());
delete externalAssetBuilderInfo;
continue;
}
if (assetBuilderType == AssetProcessor::AssetBuilderType::Invalid)
{
AZ_Warning(AssetProcessor::DebugChannel, false, "AssetProcessor was not able to load the library: %s\n", filePath.toUtf8().data());
delete externalAssetBuilderInfo;
return false;
}
AZ_TracePrintf(AssetProcessor::ConsoleChannel, "Initializing and registering builder %s\n", externalAssetBuilderInfo->GetName().toUtf8().data());
m_currentExternalAssetBuilder = externalAssetBuilderInfo;
externalAssetBuilderInfo->Initialize();
m_currentExternalAssetBuilder = nullptr;
m_externalAssetBuilders.push_back(externalAssetBuilderInfo);
}
}
// Also init external builders which may be inside of Gems
AzToolsFramework::ToolsApplicationRequestBus::Broadcast(
&AzToolsFramework::ToolsApplicationRequests::CreateAndAddEntityFromComponentTags,
AZStd::vector<AZ::Crc32>({ AssetBuilderSDK::ComponentTags::AssetBuilder }), "AssetBuilders Entity");
return true;
}
bool ApplicationManagerBase::WaitForBuilderExit(AzFramework::ProcessWatcher* processWatcher, AssetBuilderSDK::JobCancelListener* jobCancelListener, AZ::u32 processTimeoutLimitInSeconds)
{
AZ::u32 exitCode = 0;
bool finishedOK = false;
QElapsedTimer ticker;
ProcessCommunicatorTracePrinter tracer(processWatcher->GetCommunicator(), "AssetBuilder");
ticker.start();
while (!finishedOK)
{
AZStd::this_thread::sleep_for(AZStd::chrono::milliseconds(s_MaximumSleepTimeMS));
tracer.Pump();
if (ticker.elapsed() > processTimeoutLimitInSeconds * 1000 || (jobCancelListener && jobCancelListener->IsCancelled()))
{
break;
}
if (!processWatcher->IsProcessRunning(&exitCode))
{
finishedOK = true; // we either cant wait for it, or it finished.
break;
}
}
tracer.Pump(); // empty whats left if possible.
if (processWatcher->IsProcessRunning(&exitCode))
{
processWatcher->TerminateProcess(1);
}
if (exitCode != 0)
{
AZ_Error(AssetProcessor::ConsoleChannel, false, "AssetBuilder exited with error code %d", exitCode);
return false;
}
else if (jobCancelListener && jobCancelListener->IsCancelled())
{
AZ_TracePrintf(AssetProcessor::DebugChannel, "AssetBuilder was terminated. There was a request to cancel the job.\n");
return false;
}
else if (!finishedOK)
{
AZ_Error(AssetProcessor::ConsoleChannel, false, "AssetBuilder failed to terminate within %d seconds", processTimeoutLimitInSeconds);
return false;
}
return true;
}
void ApplicationManagerBase::RegisterBuilderInformation(const AssetBuilderSDK::AssetBuilderDesc& builderDesc)
{
// Create Job Function validation
AZ_Error(AssetProcessor::ConsoleChannel,
builderDesc.m_createJobFunction,
"Create Job Function (m_createJobFunction) for %s builder is empty.\n",
builderDesc.m_name.c_str());
if (!builderDesc.IsExternalBuilder())
{
// Create Job Function validation
AZ_Error(
AssetProcessor::ConsoleChannel, builderDesc.m_createJobFunction,
"Create Job Function (m_createJobFunction) for %s builder is empty.\n", builderDesc.m_name.c_str());
// Process Job Function validation
AZ_Error(AssetProcessor::ConsoleChannel,
builderDesc.m_processJobFunction,
"Process Job Function (m_processJobFunction) for %s builder is empty.\n",
builderDesc.m_name.c_str());
// Process Job Function validation
AZ_Error(
AssetProcessor::ConsoleChannel, builderDesc.m_processJobFunction,
"Process Job Function (m_processJobFunction) for %s builder is empty.\n", builderDesc.m_name.c_str());
}
// Bus ID validation
AZ_Error(AssetProcessor::ConsoleChannel,
@@ -1550,67 +1484,66 @@ void ApplicationManagerBase::RegisterBuilderInformation(const AssetBuilderSDK::A
"Bus ID for %s builder is empty.\n",
builderDesc.m_name.c_str());
// This is an external builder registering, we will want to track its builder desc since it can register multiple ones
AZStd::string builderFilePath;
if (m_currentExternalAssetBuilder)
{
m_currentExternalAssetBuilder->RegisterBuilderDesc(builderDesc.m_busId);
builderFilePath = m_currentExternalAssetBuilder->GetModuleFullPath().toUtf8().data();
}
AssetBuilderSDK::AssetBuilderDesc modifiedBuilderDesc = builderDesc;
// Allow for overrides defined in a BuilderConfig.ini file to update our code defined default values
AssetProcessor::BuilderConfigurationRequestBus::Broadcast(&AssetProcessor::BuilderConfigurationRequests::UpdateBuilderDescriptor, builderDesc.m_name, modifiedBuilderDesc);
if (builderDesc.IsExternalBuilder())
{
// We're going to override the createJob function so we can run it externally in AssetBuilder, rather than having it run inside the AP
modifiedBuilderDesc.m_createJobFunction = [builderFilePath](const AssetBuilderSDK::CreateJobsRequest& request, AssetBuilderSDK::CreateJobsResponse& response)
// We're going to override the createJob function so we can run it externally in AssetBuilder, rather than having it run
// inside the AP
modifiedBuilderDesc.m_createJobFunction =
[](const AssetBuilderSDK::CreateJobsRequest& request, AssetBuilderSDK::CreateJobsResponse& response)
{
AssetProcessor::BuilderRef builderRef;
AssetProcessor::BuilderManagerBus::BroadcastResult(builderRef, &AssetProcessor::BuilderManagerBusTraits::GetBuilder, false);
if (builderRef)
{
AssetProcessor::BuilderRef builderRef;
AssetProcessor::BuilderManagerBus::BroadcastResult(builderRef, &AssetProcessor::BuilderManagerBusTraits::GetBuilder);
int retryCount = 0;
AssetProcessor::BuilderRunJobOutcome result;
if (builderRef)
do
{
int retryCount = 0;
AssetProcessor::BuilderRunJobOutcome result;
do
{
retryCount++;
result = builderRef->RunJob<AssetBuilderSDK::CreateJobsNetRequest, AssetBuilderSDK::CreateJobsNetResponse>(request, response, s_MaximumCreateJobsTimeSeconds, "create", builderFilePath, nullptr);
} while (result == AssetProcessor::BuilderRunJobOutcome::LostConnection && retryCount <= AssetProcessor::RetriesForJobNetworkError);
}
else
{
AZ_Error("AssetProcessor", false, "Failed to retrieve a valid builder to process job");
}
};
retryCount++;
result = builderRef->RunJob<AssetBuilder::CreateJobsNetRequest, AssetBuilder::CreateJobsNetResponse>(
request, response, s_MaximumCreateJobsTimeSeconds, "create", "", nullptr);
} while (result == AssetProcessor::BuilderRunJobOutcome::LostConnection &&
retryCount <= AssetProcessor::RetriesForJobNetworkError);
}
else
{
AZ_Error("AssetProcessor", false, "Failed to retrieve a valid builder to process job");
}
};
// Also override the processJob function to run externally
modifiedBuilderDesc.m_processJobFunction = [builderFilePath](const AssetBuilderSDK::ProcessJobRequest& request, AssetBuilderSDK::ProcessJobResponse& response)
modifiedBuilderDesc.m_processJobFunction =
[](const AssetBuilderSDK::ProcessJobRequest& request, AssetBuilderSDK::ProcessJobResponse& response)
{
AssetBuilderSDK::JobCancelListener jobCancelListener(request.m_jobId);
AssetProcessor::BuilderRef builderRef;
AssetProcessor::BuilderManagerBus::BroadcastResult(builderRef, &AssetProcessor::BuilderManagerBusTraits::GetBuilder, false);
if (builderRef)
{
AssetBuilderSDK::JobCancelListener jobCancelListener(request.m_jobId);
int retryCount = 0;
AssetProcessor::BuilderRunJobOutcome result;
AssetProcessor::BuilderRef builderRef;
AssetProcessor::BuilderManagerBus::BroadcastResult(builderRef, &AssetProcessor::BuilderManagerBusTraits::GetBuilder);
if (builderRef)
do
{
int retryCount = 0;
AssetProcessor::BuilderRunJobOutcome result;
do
{
retryCount++;
result = builderRef->RunJob<AssetBuilderSDK::ProcessJobNetRequest, AssetBuilderSDK::ProcessJobNetResponse>(request, response, s_MaximumProcessJobsTimeSeconds, "process", builderFilePath, &jobCancelListener, request.m_tempDirPath);
} while (result == AssetProcessor::BuilderRunJobOutcome::LostConnection && retryCount <= AssetProcessor::RetriesForJobNetworkError);
}
else
{
AZ_Error("AssetProcessor", false, "Failed to retrieve a valid builder to process job");
}
};
retryCount++;
result = builderRef->RunJob<AssetBuilder::ProcessJobNetRequest, AssetBuilder::ProcessJobNetResponse>(
request, response, s_MaximumProcessJobsTimeSeconds, "process", "", &jobCancelListener, request.m_tempDirPath);
} while (result == AssetProcessor::BuilderRunJobOutcome::LostConnection &&
retryCount <= AssetProcessor::RetriesForJobNetworkError);
}
else
{
AZ_Error("AssetProcessor", false, "Failed to retrieve a valid builder to process job");
}
};
}
if (m_builderDescMap.find(modifiedBuilderDesc.m_busId) != m_builderDescMap.end())
@@ -1768,7 +1701,7 @@ bool ApplicationManagerBase::CheckSufficientDiskSpace(const QString& savePath, q
[[maybe_unused]] bool result = AzToolsFramework::ToolsFileUtils::GetFreeDiskSpace(savePath, bytesFree);
AZ_Assert(result, "Unable to determine the amount of free space on drive containing path (%s).", savePath.toUtf8().constData());
if (bytesFree < requiredSpace + s_ReservedDiskSpaceInBytes)
{
if (shutdownIfInsufficient)
@@ -1806,8 +1739,8 @@ void ApplicationManagerBase::RemoveOldTempFolders()
return;
}
// We will remove old temp folders if either their modified time is older than the cutoff time or
// if the total number of temp folders have exceeded the maximum number of temp folders.
// We will remove old temp folders if either their modified time is older than the cutoff time or
// if the total number of temp folders have exceeded the maximum number of temp folders.
QFileInfoList entries = root.entryInfoList(QDir::Dirs | QDir::NoDotAndDotDot, QDir::Time); // sorting by modification time
int folderCount = 0;
bool removeFolder = false;
@@ -1821,9 +1754,9 @@ void ApplicationManagerBase::RemoveOldTempFolders()
// Since we are sorting the folders list from latest to oldest, we will either be in a state where we have to delete all the remaining folders or not
// because either we have reached the folder limit or reached the cutoff date limit.
removeFolder = removeFolder || (folderCount++ >= s_MaximumTempFolders) ||
removeFolder = removeFolder || (folderCount++ >= s_MaximumTempFolders) ||
(entry.lastModified() < cutoffTime);
if (removeFolder)
{
QDir dir(entry.absoluteFilePath());
@@ -1837,8 +1770,6 @@ void ApplicationManagerBase::ConnectivityStateChanged(const AzToolsFramework::So
Q_EMIT SourceControlReady();
}
void ApplicationManagerBase::OnAssetProcessorManagerIdleState(bool isIdle)
{
// these can come in during shutdown.
Code/Tools/AssetProcessor/native/utilities/ApplicationManagerBase.h
+3 -4
View File
@@ -149,7 +149,6 @@ protected:
void CreateQtApplication() override;
bool InitializeInternalBuilders();
bool InitializeExternalBuilders();
void InitBuilderManager();
void ShutdownBuilderManager();
bool InitAssetDatabase();
@@ -173,8 +172,6 @@ protected:
AssetProcessor::AssetCatalog* GetAssetCatalog() const { return m_assetCatalog; }
static bool WaitForBuilderExit(AzFramework::ProcessWatcher* processWatcher, AssetBuilderSDK::JobCancelListener* jobCancelListener, AZ::u32 processTimeoutLimitInSeconds);
ApplicationServer* m_applicationServer = nullptr;
ConnectionManager* m_connectionManager = nullptr;
@@ -218,6 +215,8 @@ protected:
AZStd::shared_ptr<AssetProcessor::InternalRecognizerBasedBuilder> m_internalBuilder;
AZStd::shared_ptr<AssetProcessor::SettingsRegistryBuilder> m_settingsRegistryBuilder;
bool m_builderRegistrationComplete = false;
// Builder description map based on the builder id
AZStd::unordered_map<AZ::Uuid, AssetBuilderSDK::AssetBuilderDesc> m_builderDescMap;
@@ -231,7 +230,7 @@ protected:
AZStd::list<AssetProcessor::ExternalModuleAssetBuilderInfo*> m_externalAssetBuilders;
AssetProcessor::ExternalModuleAssetBuilderInfo* m_currentExternalAssetBuilder = nullptr;
QAtomicInt m_connectionsAwaitingAssetCatalogSave = 0;
int m_remainingAPMJobs = 0;
bool m_assetProcessorManagerIsReady = false;
Code/Tools/AssetProcessor/native/utilities/BuilderManager.cpp
+39 -28
View File
@@ -16,6 +16,7 @@
#include <native/utilities/AssetBuilderInfo.h>
#include <QCoreApplication>
#include <AzCore/Settings/SettingsRegistryMergeUtils.h>
#include <AssetBuilder/AssetBuilderStatic.h>
namespace AssetProcessor
{
@@ -138,7 +139,7 @@ namespace AssetProcessor
}
}
bool Builder::Start()
bool Builder::Start(bool doRegistration)
{
// Get the current BinXXX folder based on the current running AP
QString applicationDir = QCoreApplication::instance()->applicationDirPath();
@@ -155,7 +156,7 @@ namespace AssetProcessor
return false;
}
const AZStd::vector<AZStd::string> params = BuildParams("resident", buildersFolder.c_str(), UuidString(), "", "");
const AZStd::vector<AZStd::string> params = BuildParams("resident", buildersFolder.c_str(), UuidString(), "", "", doRegistration);
m_processWatcher = LaunchProcess(fullExePathString.c_str(), params);
@@ -179,7 +180,7 @@ namespace AssetProcessor
return !m_processWatcher || (m_processWatcher && m_processWatcher->IsProcessRunning(exitCode));
}
AZStd::vector<AZStd::string> Builder::BuildParams(const char* task, const char* moduleFilePath, const AZStd::string& builderGuid, const AZStd::string& jobDescriptionFile, const AZStd::string& jobResponseFile) const
AZStd::vector<AZStd::string> Builder::BuildParams(const char* task, const char* moduleFilePath, const AZStd::string& builderGuid, const AZStd::string& jobDescriptionFile, const AZStd::string& jobResponseFile, bool doRegistration) const
{
QDir projectCacheRoot;
AssetUtilities::ComputeProjectCacheRoot(projectCacheRoot);
@@ -200,6 +201,11 @@ namespace AssetProcessor
params.emplace_back(AZStd::string::format(R"(-engine-path="%s")", enginePath.c_str()));
params.emplace_back(AZStd::string::format("-port=%d", portNumber));
if(doRegistration)
{
params.emplace_back("--register");
}
if (moduleFilePath && moduleFilePath[0])
{
params.emplace_back(AZStd::string::format(R"(-module="%s")", moduleFilePath));
@@ -232,7 +238,7 @@ namespace AssetProcessor
{
AzFramework::ProcessLauncher::ProcessLaunchInfo processLaunchInfo;
processLaunchInfo.m_processExecutableString = fullExePath;
AZStd::vector<AZStd::string> commandLineArray{ fullExePath };
commandLineArray.insert(commandLineArray.end(), params.begin(), params.end());
processLaunchInfo.m_commandlineParameters = AZStd::move(commandLineArray);
@@ -350,17 +356,19 @@ namespace AssetProcessor
BuilderManager::BuilderManager(ConnectionManager* connectionManager)
{
using namespace AZStd::placeholders;
connectionManager->RegisterService(AssetBuilderSDK::BuilderHelloRequest::MessageType(), AZStd::bind(&BuilderManager::IncomingBuilderPing, this, _1, _2, _3, _4, _5));
connectionManager->RegisterService(AssetBuilder::BuilderHelloRequest::MessageType(), AZStd::bind(&BuilderManager::IncomingBuilderPing, this, _1, _2, _3, _4, _5));
// Setup a background thread to pump the idle builders so they don't get blocked trying to output to stdout/err
m_pollingThread = AZStd::thread([this]()
AZStd::thread_desc desc;
desc.m_name = "BuilderManager Idle Pump";
m_pollingThread = AZStd::thread(desc, [this]()
{
while (!m_quitListener.WasQuitRequested())
{
while (!m_quitListener.WasQuitRequested())
{
PumpIdleBuilders();
AZStd::this_thread::sleep_for(AZStd::chrono::milliseconds(s_IdleBuilderPumpingDelayMS));
}
});
PumpIdleBuilders();
AZStd::this_thread::sleep_for(AZStd::chrono::milliseconds(s_IdleBuilderPumpingDelayMS));
}
});
m_quitListener.BusConnect();
BusConnect();
@@ -399,8 +407,8 @@ namespace AssetProcessor
void BuilderManager::IncomingBuilderPing(AZ::u32 connId, AZ::u32 /*type*/, AZ::u32 serial, QByteArray payload, QString platform)
{
AssetBuilderSDK::BuilderHelloRequest requestPing;
AssetBuilderSDK::BuilderHelloResponse responsePing;
AssetBuilder::BuilderHelloRequest requestPing;
AssetBuilder::BuilderHelloResponse responsePing;
if (!AZ::Utils::LoadObjectFromBufferInPlace(payload.data(), payload.length(), requestPing))
{
@@ -476,7 +484,7 @@ namespace AssetProcessor
return builder;
}
BuilderRef BuilderManager::GetBuilder()
BuilderRef BuilderManager::GetBuilder(bool doRegistration)
{
AZStd::shared_ptr<Builder> newBuilder;
BuilderRef builderRef;
@@ -484,27 +492,30 @@ namespace AssetProcessor
{
AZStd::unique_lock<AZStd::mutex> lock(m_buildersMutex);
for (auto itr = m_builders.begin(); itr != m_builders.end(); )
if (!doRegistration)
{
auto& builder = itr->second;
if (!builder->m_busy)
for (auto itr = m_builders.begin(); itr != m_builders.end();)
{
builder->PumpCommunicator();
auto& builder = itr->second;
if (builder->IsValid())
if (!builder->m_busy)
{
return BuilderRef(builder);
builder->PumpCommunicator();
if (builder->IsValid())
{
return BuilderRef(builder);
}
else
{
itr = m_builders.erase(itr);
}
}
else
{
itr = m_builders.erase(itr);
++itr;
}
}
else
{
++itr;
}
}
AZ_TracePrintf("BuilderManager", "Starting new builder for job request\n");
@@ -516,7 +527,7 @@ namespace AssetProcessor
builderRef = BuilderRef(newBuilder);
}
if (!newBuilder->Start())
if (!newBuilder->Start(doRegistration))
{
AZ_Error("BuilderManager", false, "Builder failed to start");
Code/Tools/AssetProcessor/native/utilities/BuilderManager.h
+4 -4
View File
@@ -39,7 +39,7 @@ namespace AssetProcessor
virtual ~BuilderManagerBusTraits() = default;
//! Returns a builder for doing work
virtual BuilderRef GetBuilder() = 0;
virtual BuilderRef GetBuilder(bool doRegistration) = 0;
};
using BuilderManagerBus = AZ::EBus<BuilderManagerBusTraits>;
@@ -98,12 +98,12 @@ namespace AssetProcessor
private:
//! Starts the builder process and waits for it to connect
bool Start();
bool Start(bool doRegistration);
//! Sets the connection id and signals that the builder has connected
void SetConnection(AZ::u32 connId);
AZStd::vector<AZStd::string> BuildParams(const char* task, const char* moduleFilePath, const AZStd::string& builderGuid, const AZStd::string& jobDescriptionFile, const AZStd::string& jobResponseFile) const;
AZStd::vector<AZStd::string> BuildParams(const char* task, const char* moduleFilePath, const AZStd::string& builderGuid, const AZStd::string& jobDescriptionFile, const AZStd::string& jobResponseFile, bool doRegistration) const;
AZStd::unique_ptr<AzFramework::ProcessWatcher> LaunchProcess(const char* fullExePath, const AZStd::vector<AZStd::string>& params) const;
//! Waits for the builder exe to send the job response and pumps stdout/err
@@ -169,7 +169,7 @@ namespace AssetProcessor
void ConnectionLost(AZ::u32 connId);
//BuilderManagerBus
BuilderRef GetBuilder() override;
BuilderRef GetBuilder(bool doRegistration) override;
private:
Code/Tools/AssetProcessor/native/utilities/BuilderManager.inl
+2 -2
View File
@@ -50,7 +50,7 @@ namespace AssetProcessor
if (!netResponse.m_response.Succeeded() || s_createRequestFileForSuccessfulJob)
{
// we write the request out to disk for failure or debugging
// we write the request out to disk for failure or debugging
if (!DebugWriteRequestFile(tempFolderPath.c_str(), request, task, modulePath))
{
return BuilderRunJobOutcome::FailedToWriteDebugRequest;
@@ -83,7 +83,7 @@ namespace AssetProcessor
return false;
}
auto params = BuildParams(task.c_str(), modulePath.c_str(), "", jobRequestFile, jobResponseFile);
auto params = BuildParams(task.c_str(), modulePath.c_str(), "", jobRequestFile, jobResponseFile, false);
AZStd::string paramString;
AZ::StringFunc::Join(paramString, params.begin(), params.end(), " ");
Code/Tools/AssetProcessor/native/utilities/GUIApplicationManager.cpp
+2 -9
View File
@@ -318,16 +318,8 @@ bool GUIApplicationManager::Run()
qApp->setQuitOnLastWindowClosed(false);
QTimer::singleShot(0, this, [this]()
{
if (!PostActivate())
{
QuitRequested();
m_startedSuccessfully = false;
}
});
m_duringStartup = false;
m_startedSuccessfully = true;
int resultCode = qApp->exec(); // this blocks until the last window is closed.
@@ -483,6 +475,7 @@ bool GUIApplicationManager::PostActivate()
{
if (!ApplicationManagerBase::PostActivate())
{
m_startedSuccessfully = false;
return false;
}