From 6c17c7bfb3019812181785d0ab3549baf1c7ef40 Mon Sep 17 00:00:00 2001 From: Mike Balfour <82224783+mbalfour-amzn@users.noreply.github.com> Date: Wed, 26 May 2021 15:30:21 -0500 Subject: [PATCH] Add new API to convert absolute source paths to relative paths. (#930) There are already APIs for getting a relative product path from an absolute source path, or getting a relative source path for an *existing* source file, but there were no APIs for getting a relative source path for a *new* source file. Prefabs will need this ability to be able to correctly generate a relative source path inside the prefab file before the file has been saved. The logic for relative source paths is a little bit tricky because the paths are relative to the watch folders, and the watch folders can be nested, with different priorities to explain which should take precedence. The input paths can also include specifiers like "." and "..", which need to be reconciled before creating the final correct relative path. The included unit tests test all of the tricky edge cases that I was able to identify. --- .../Asset/AssetProcessorMessages.cpp | 50 +++++ .../Asset/AssetProcessorMessages.h | 39 ++++ .../Asset/AssetSystemComponent.cpp | 2 + .../API/EditorAssetSystemAPI.h | 12 +- .../Asset/AssetSystemComponent.cpp | 24 +++ .../Asset/AssetSystemComponent.h | 2 + .../AzToolsFramework/Tests/AssetSystemMocks.h | 2 + .../SliceStabilityTestFramework.h | 3 + .../native/AssetManager/AssetCatalog.cpp | 74 +++++++ .../native/AssetManager/AssetCatalog.h | 6 + .../AssetManager/AssetRequestHandler.cpp | 22 ++ .../AssetManager/assetProcessorManager.h | 5 + .../AssetCatalog/AssetCatalogUnitTests.cpp | 194 +++++++++++++++++- .../tests/AssetProcessorMessagesTests.cpp | 3 + .../AssetProcessorManagerUnitTests.cpp | 2 + .../RPI/Code/Tests/Common/AssetSystemStub.cpp | 10 +- .../RPI/Code/Tests/Common/AssetSystemStub.h | 2 + .../Builders/CopyDependencyBuilderTest.cpp | 3 + 18 files changed, 446 insertions(+), 9 deletions(-) diff --git a/Code/Framework/AzFramework/AzFramework/Asset/AssetProcessorMessages.cpp b/Code/Framework/AzFramework/AzFramework/Asset/AssetProcessorMessages.cpp index 020feffc47..7280c4af5c 100644 --- a/Code/Framework/AzFramework/AzFramework/Asset/AssetProcessorMessages.cpp +++ b/Code/Framework/AzFramework/AzFramework/Asset/AssetProcessorMessages.cpp @@ -308,6 +308,56 @@ namespace AzFramework } } + //--------------------------------------------------------------------- + GenerateRelativeSourcePathRequest::GenerateRelativeSourcePathRequest(const AZ::OSString& sourcePath) + { + AZ_Assert(!sourcePath.empty(), "GenerateRelativeSourcePathRequest: asset path is empty"); + m_sourcePath = sourcePath; + } + + unsigned int GenerateRelativeSourcePathRequest::GetMessageType() const + { + return MessageType; + } + + void GenerateRelativeSourcePathRequest::Reflect(AZ::ReflectContext* context) + { + auto serialize = azrtti_cast(context); + if (serialize) + { + serialize->Class() + ->Version(1) + ->Field("SourcePath", &GenerateRelativeSourcePathRequest::m_sourcePath); + } + } + + //--------------------------------------------------------------------- + GenerateRelativeSourcePathResponse::GenerateRelativeSourcePathResponse( + bool resolved, const AZ::OSString& relativeSourcePath, const AZ::OSString& rootFolder) + { + m_relativeSourcePath = relativeSourcePath; + m_resolved = resolved; + m_rootFolder = rootFolder; + } + + unsigned int GenerateRelativeSourcePathResponse::GetMessageType() const + { + return GenerateRelativeSourcePathRequest::MessageType; + } + + void GenerateRelativeSourcePathResponse::Reflect(AZ::ReflectContext* context) + { + auto serialize = azrtti_cast(context); + if (serialize) + { + serialize->Class() + ->Version(1) + ->Field("RelativeSourcePath", &GenerateRelativeSourcePathResponse::m_relativeSourcePath) + ->Field("RootFolder", &GenerateRelativeSourcePathResponse::m_rootFolder) + ->Field("Resolved", &GenerateRelativeSourcePathResponse::m_resolved); + } + } + //--------------------------------------------------------------------- GetFullSourcePathFromRelativeProductPathRequest::GetFullSourcePathFromRelativeProductPathRequest(const AZ::OSString& relativeProductPath) { diff --git a/Code/Framework/AzFramework/AzFramework/Asset/AssetProcessorMessages.h b/Code/Framework/AzFramework/AzFramework/Asset/AssetProcessorMessages.h index 9661e61828..c15f75e3e7 100644 --- a/Code/Framework/AzFramework/AzFramework/Asset/AssetProcessorMessages.h +++ b/Code/Framework/AzFramework/AzFramework/Asset/AssetProcessorMessages.h @@ -288,6 +288,45 @@ namespace AzFramework bool m_resolved; }; + ////////////////////////////////////////////////////////////////////////// + class GenerateRelativeSourcePathRequest : public BaseAssetProcessorMessage + { + public: + AZ_CLASS_ALLOCATOR(GenerateRelativeSourcePathRequest, AZ::OSAllocator, 0); + AZ_RTTI(GenerateRelativeSourcePathRequest, "{B3865033-F5A3-4749-8147-7B1AB04D5F6D}", + BaseAssetProcessorMessage); + static void Reflect(AZ::ReflectContext* context); + + // For people that are debugging the network messages and just see MessageType as a value, + // the CRC value below is 739777771 (0x2C181CEB) + static constexpr unsigned int MessageType = + AZ_CRC_CE("AssetSystem::GenerateRelativeSourcePathRequest"); + + GenerateRelativeSourcePathRequest() = default; + GenerateRelativeSourcePathRequest(const AZ::OSString& sourcePath); + unsigned int GetMessageType() const override; + + AZ::OSString m_sourcePath; + }; + + class GenerateRelativeSourcePathResponse : public BaseAssetProcessorMessage + { + public: + AZ_CLASS_ALLOCATOR(GenerateRelativeSourcePathResponse, AZ::OSAllocator, 0); + AZ_RTTI(GenerateRelativeSourcePathResponse, "{938D33DB-C8F6-4FA4-BC81-2F139A9BE1D7}", + BaseAssetProcessorMessage); + static void Reflect(AZ::ReflectContext* context); + + GenerateRelativeSourcePathResponse() = default; + GenerateRelativeSourcePathResponse( + bool resolved, const AZ::OSString& relativeSourcePath, const AZ::OSString& rootFolder); + unsigned int GetMessageType() const override; + + AZ::OSString m_relativeSourcePath; + AZ::OSString m_rootFolder; ///< This is the folder it was found in (the watched/scanned folder, such as gems /assets/ folder) + bool m_resolved; + }; + ////////////////////////////////////////////////////////////////////////// class GetFullSourcePathFromRelativeProductPathRequest : public BaseAssetProcessorMessage diff --git a/Code/Framework/AzFramework/AzFramework/Asset/AssetSystemComponent.cpp b/Code/Framework/AzFramework/AzFramework/Asset/AssetSystemComponent.cpp index 83c4907468..6b19084c2a 100644 --- a/Code/Framework/AzFramework/AzFramework/Asset/AssetSystemComponent.cpp +++ b/Code/Framework/AzFramework/AzFramework/Asset/AssetSystemComponent.cpp @@ -202,6 +202,7 @@ namespace AzFramework // Requests GetUnresolvedDependencyCountsRequest::Reflect(context); GetRelativeProductPathFromFullSourceOrProductPathRequest::Reflect(context); + GenerateRelativeSourcePathRequest::Reflect(context); GetFullSourcePathFromRelativeProductPathRequest::Reflect(context); SourceAssetInfoRequest::Reflect(context); AssetInfoRequest::Reflect(context); @@ -234,6 +235,7 @@ namespace AzFramework // Responses GetUnresolvedDependencyCountsResponse::Reflect(context); GetRelativeProductPathFromFullSourceOrProductPathResponse::Reflect(context); + GenerateRelativeSourcePathResponse::Reflect(context); GetFullSourcePathFromRelativeProductPathResponse::Reflect(context); SourceAssetInfoResponse::Reflect(context); AssetInfoResponse::Reflect(context); diff --git a/Code/Framework/AzToolsFramework/AzToolsFramework/API/EditorAssetSystemAPI.h b/Code/Framework/AzToolsFramework/AzToolsFramework/API/EditorAssetSystemAPI.h index 98e4c6b5eb..1599d29589 100644 --- a/Code/Framework/AzToolsFramework/AzToolsFramework/API/EditorAssetSystemAPI.h +++ b/Code/Framework/AzToolsFramework/AzToolsFramework/API/EditorAssetSystemAPI.h @@ -60,10 +60,20 @@ namespace AzToolsFramework //! and is generally checked into source control. virtual const char* GetAbsoluteDevRootFolderPath() = 0; - /// Convert a full source path like "c:\\dev\gamename\\blah\\test.tga" into a relative product path. + /// Convert a full source path like "c:\\dev\\gamename\\blah\\test.tga" into a relative product path. /// asset paths never mention their alias and are relative to the asset cache root virtual bool GetRelativeProductPathFromFullSourceOrProductPath(const AZStd::string& fullPath, AZStd::string& relativeProductPath) = 0; + /** Convert a source path like "c:\\dev\\gamename\\blah\\test.tga" into a relative source path, like "blah/test.tga". + * If no valid relative path could be created, the input source path will be returned in relativePath. + * @param sourcePath partial or full path to a source file. (The file doesn't need to exist) + * @param relativePath the output relative path for the source file, if a valid one could be created + * @param rootFilePath the root path that relativePath is relative to + * @return true if a valid relative path was created, false if it wasn't + */ + virtual bool GenerateRelativeSourcePath( + const AZStd::string& sourcePath, AZStd::string& relativePath, AZStd::string& rootFilePath) = 0; + /// Convert a relative asset path like "blah/test.tga" to a full source path path. /// Once the asset processor has finished building, this function is capable of handling even when the extension changes /// or when the source is in a different folder or in a different location (such as inside gems) diff --git a/Code/Framework/AzToolsFramework/AzToolsFramework/Asset/AssetSystemComponent.cpp b/Code/Framework/AzToolsFramework/AzToolsFramework/Asset/AssetSystemComponent.cpp index 5529829913..4966d9cce9 100644 --- a/Code/Framework/AzToolsFramework/AzToolsFramework/Asset/AssetSystemComponent.cpp +++ b/Code/Framework/AzToolsFramework/AzToolsFramework/Asset/AssetSystemComponent.cpp @@ -265,6 +265,30 @@ namespace AzToolsFramework return response.m_resolved; } + bool AssetSystemComponent::GenerateRelativeSourcePath( + const AZStd::string& sourcePath, AZStd::string& relativePath, AZStd::string& rootFilePath) + { + AzFramework::SocketConnection* engineConnection = AzFramework::SocketConnection::GetInstance(); + if (!engineConnection || !engineConnection->IsConnected()) + { + relativePath = sourcePath; + return false; + } + + AzFramework::AssetSystem::GenerateRelativeSourcePathRequest request(sourcePath); + AzFramework::AssetSystem::GenerateRelativeSourcePathResponse response; + if (!SendRequest(request, response)) + { + AZ_Error("Editor", false, "Failed to send GenerateRelativeSourcePath request for %s", sourcePath.c_str()); + relativePath = sourcePath; + return false; + } + + relativePath = response.m_relativeSourcePath; + rootFilePath = response.m_rootFolder; + return response.m_resolved; + } + bool AssetSystemComponent::GetFullSourcePathFromRelativeProductPath(const AZStd::string& relPath, AZStd::string& fullPath) { auto foundIt = m_assetSourceRelativePathToFullPathCache.find(relPath); diff --git a/Code/Framework/AzToolsFramework/AzToolsFramework/Asset/AssetSystemComponent.h b/Code/Framework/AzToolsFramework/AzToolsFramework/Asset/AssetSystemComponent.h index 399ee1ac9d..9d839c60f5 100644 --- a/Code/Framework/AzToolsFramework/AzToolsFramework/Asset/AssetSystemComponent.h +++ b/Code/Framework/AzToolsFramework/AzToolsFramework/Asset/AssetSystemComponent.h @@ -63,6 +63,8 @@ namespace AzToolsFramework const char* GetAbsoluteDevGameFolderPath() override; const char* GetAbsoluteDevRootFolderPath() override; bool GetRelativeProductPathFromFullSourceOrProductPath(const AZStd::string& fullPath, AZStd::string& outputPath) override; + bool GenerateRelativeSourcePath( + const AZStd::string& sourcePath, AZStd::string& outputPath, AZStd::string& watchFolder) override; bool GetFullSourcePathFromRelativeProductPath(const AZStd::string& relPath, AZStd::string& fullPath) override; bool GetAssetInfoById(const AZ::Data::AssetId& assetId, const AZ::Data::AssetType& assetType, const AZStd::string& platformName, AZ::Data::AssetInfo& assetInfo, AZStd::string& rootFilePath) override; bool GetSourceInfoBySourcePath(const char* sourcePath, AZ::Data::AssetInfo& assetInfo, AZStd::string& watchFolder) override; diff --git a/Code/Framework/AzToolsFramework/Tests/AssetSystemMocks.h b/Code/Framework/AzToolsFramework/Tests/AssetSystemMocks.h index 1e01229d73..8a394d3ab5 100644 --- a/Code/Framework/AzToolsFramework/Tests/AssetSystemMocks.h +++ b/Code/Framework/AzToolsFramework/Tests/AssetSystemMocks.h @@ -25,6 +25,8 @@ namespace UnitTests MOCK_METHOD0(GetAbsoluteDevGameFolderPath, const char* ()); MOCK_METHOD0(GetAbsoluteDevRootFolderPath, const char* ()); MOCK_METHOD2(GetRelativeProductPathFromFullSourceOrProductPath, bool(const AZStd::string& fullPath, AZStd::string& relativeProductPath)); + MOCK_METHOD3(GenerateRelativeSourcePath, + bool(const AZStd::string& sourcePath, AZStd::string& relativePath, AZStd::string& watchFolder)); MOCK_METHOD2(GetFullSourcePathFromRelativeProductPath, bool(const AZStd::string& relPath, AZStd::string& fullSourcePath)); MOCK_METHOD5(GetAssetInfoById, bool(const AZ::Data::AssetId& assetId, const AZ::Data::AssetType& assetType, const AZStd::string& platformName, AZ::Data::AssetInfo& assetInfo, AZStd::string& rootFilePath)); MOCK_METHOD3(GetSourceInfoBySourcePath, bool(const char* sourcePath, AZ::Data::AssetInfo& assetInfo, AZStd::string& watchFolder)); diff --git a/Code/Framework/AzToolsFramework/Tests/SliceStabilityTests/SliceStabilityTestFramework.h b/Code/Framework/AzToolsFramework/Tests/SliceStabilityTests/SliceStabilityTestFramework.h index 57ac16673d..7d3d312cdb 100644 --- a/Code/Framework/AzToolsFramework/Tests/SliceStabilityTests/SliceStabilityTestFramework.h +++ b/Code/Framework/AzToolsFramework/Tests/SliceStabilityTests/SliceStabilityTestFramework.h @@ -149,6 +149,9 @@ namespace UnitTest const char* GetAbsoluteDevGameFolderPath() override { return ""; } const char* GetAbsoluteDevRootFolderPath() override { return ""; } bool GetRelativeProductPathFromFullSourceOrProductPath([[maybe_unused]] const AZStd::string& fullPath, [[maybe_unused]] AZStd::string& relativeProductPath) override { return false; } + bool GenerateRelativeSourcePath( + [[maybe_unused]] const AZStd::string& sourcePath, [[maybe_unused]] AZStd::string& relativePath, + [[maybe_unused]] AZStd::string& watchFolder) override { return false; } bool GetFullSourcePathFromRelativeProductPath([[maybe_unused]] const AZStd::string& relPath, [[maybe_unused]] AZStd::string& fullSourcePath) override { return false; } bool GetAssetInfoById([[maybe_unused]] const AZ::Data::AssetId& assetId, [[maybe_unused]] const AZ::Data::AssetType& assetType, [[maybe_unused]] const AZStd::string& platformName, [[maybe_unused]] AZ::Data::AssetInfo& assetInfo, [[maybe_unused]] AZStd::string& rootFilePath) override { return false; } bool GetSourceInfoBySourcePath(const char* sourcePath, AZ::Data::AssetInfo& assetInfo, AZStd::string& watchFolder) override; diff --git a/Code/Tools/AssetProcessor/native/AssetManager/AssetCatalog.cpp b/Code/Tools/AssetProcessor/native/AssetManager/AssetCatalog.cpp index 196f6b0543..8bc6d0b6f7 100644 --- a/Code/Tools/AssetProcessor/native/AssetManager/AssetCatalog.cpp +++ b/Code/Tools/AssetProcessor/native/AssetManager/AssetCatalog.cpp @@ -655,6 +655,80 @@ namespace AssetProcessor return true; } + bool AssetCatalog::GenerateRelativeSourcePath( + const AZStd::string& sourcePath, AZStd::string& relativePath, AZStd::string& rootFolder) + { + QString normalizedSourcePath = AssetUtilities::NormalizeFilePath(sourcePath.c_str()); + QDir inputPath(normalizedSourcePath); + QString scanFolder; + QString relativeName; + + bool validResult = false; + + AZ_TracePrintf(AssetProcessor::DebugChannel, "ProcessGenerateRelativeSourcePathRequest: %s...\n", sourcePath.c_str()); + + if (sourcePath.empty()) + { + // For an empty input path, do nothing, we'll return an empty, invalid result. + // (We check fullPath instead of inputPath, because an empty fullPath actually produces "." for inputPath) + } + else if (inputPath.isAbsolute()) + { + // For an absolute path, try to convert it to a relative path, based on the existing scan folders. + // To get the inputPath, we use absolutePath() instead of path() so that any . or .. entries get collapsed. + validResult = m_platformConfig->ConvertToRelativePath(inputPath.absolutePath(), relativeName, scanFolder); + } + else if (inputPath.isRelative()) + { + // For a relative path, concatenate it with each scan folder, and see if a valid relative path emerges. + int scanFolders = m_platformConfig->GetScanFolderCount(); + for (int scanIdx = 0; scanIdx < scanFolders; scanIdx++) + { + auto& scanInfo = m_platformConfig->GetScanFolderAt(scanIdx); + QDir possibleRoot(scanInfo.ScanPath()); + QDir possibleAbsolutePath = possibleRoot.filePath(normalizedSourcePath); + // To get the inputPath, we use absolutePath() instead of path() so that any . or .. entries get collapsed. + if (m_platformConfig->ConvertToRelativePath(possibleAbsolutePath.absolutePath(), relativeName, scanFolder)) + { + validResult = true; + break; + } + } + } + + // The input has produced a valid relative path. However, the path might match multiple nested scan folders, + // so look to see if a higher-priority folder has a better match. + if (validResult) + { + QString overridingFile = m_platformConfig->GetOverridingFile(relativeName, scanFolder); + + if (!overridingFile.isEmpty()) + { + overridingFile = AssetUtilities::NormalizeFilePath(overridingFile); + validResult = m_platformConfig->ConvertToRelativePath(overridingFile, relativeName, scanFolder); + } + } + + if (!validResult) + { + // if we are here it means we have failed to determine the relativePath, so we will send back the original path + AZ_TracePrintf(AssetProcessor::DebugChannel, + "GenerateRelativeSourcePath found no valid result, returning original path: %s...\n", sourcePath.c_str()); + + rootFolder.clear(); + relativePath.clear(); + relativePath = sourcePath; + return false; + } + + relativePath = relativeName.toUtf8().data(); + rootFolder = scanFolder.toUtf8().data(); + + AZ_Assert(!relativePath.empty(), "ConvertToRelativePath returned true, but relativePath is empty"); + + return true; + } + bool AssetCatalog::GetFullSourcePathFromRelativeProductPath(const AZStd::string& relPath, AZStd::string& fullSourcePath) { ProcessGetFullSourcePathFromRelativeProductPathRequest(relPath, fullSourcePath); diff --git a/Code/Tools/AssetProcessor/native/AssetManager/AssetCatalog.h b/Code/Tools/AssetProcessor/native/AssetManager/AssetCatalog.h index f515fa3658..13dc7892b1 100644 --- a/Code/Tools/AssetProcessor/native/AssetManager/AssetCatalog.h +++ b/Code/Tools/AssetProcessor/native/AssetManager/AssetCatalog.h @@ -95,6 +95,12 @@ namespace AssetProcessor const char* GetAbsoluteDevGameFolderPath() override; const char* GetAbsoluteDevRootFolderPath() override; bool GetRelativeProductPathFromFullSourceOrProductPath(const AZStd::string& fullPath, AZStd::string& relativeProductPath) override; + + //! Given a partial or full source file path, respond with its relative path and the watch folder it is relative to. + //! The input source path does not need to exist, so this can be used for new files that haven't been saved yet. + bool GenerateRelativeSourcePath( + const AZStd::string& sourcePath, AZStd::string& relativePath, AZStd::string& watchFolder) override; + bool GetFullSourcePathFromRelativeProductPath(const AZStd::string& relPath, AZStd::string& fullSourcePath) override; bool GetAssetInfoById(const AZ::Data::AssetId& assetId, const AZ::Data::AssetType& assetType, const AZStd::string& platformName, AZ::Data::AssetInfo& assetInfo, AZStd::string& rootFilePath) override; bool GetSourceInfoBySourcePath(const char* sourcePath, AZ::Data::AssetInfo& assetInfo, AZStd::string& watchFolder) override; diff --git a/Code/Tools/AssetProcessor/native/AssetManager/AssetRequestHandler.cpp b/Code/Tools/AssetProcessor/native/AssetManager/AssetRequestHandler.cpp index 97b28691dc..2d22d3d136 100644 --- a/Code/Tools/AssetProcessor/native/AssetManager/AssetRequestHandler.cpp +++ b/Code/Tools/AssetProcessor/native/AssetManager/AssetRequestHandler.cpp @@ -104,6 +104,27 @@ namespace return GetRelativeProductPathFromFullSourceOrProductPathResponse(relPathFound, relProductPath); } + GenerateRelativeSourcePathResponse HandleGenerateRelativeSourcePathRequest( + MessageData messageData) + { + bool relPathFound = false; + AZStd::string relPath; + AZStd::string watchFolder; + + AzToolsFramework::AssetSystemRequestBus::BroadcastResult( + relPathFound, &AzToolsFramework::AssetSystemRequestBus::Events::GenerateRelativeSourcePath, + messageData.m_message->m_sourcePath, relPath, watchFolder); + + if (!relPathFound) + { + AZ_TracePrintf( + AssetProcessor::ConsoleChannel, "Could not find relative source path for the source file (%s).", + messageData.m_message->m_sourcePath.c_str()); + } + + return GenerateRelativeSourcePathResponse(relPathFound, relPath, watchFolder); + } + SourceAssetInfoResponse HandleSourceAssetInfoRequest(MessageData messageData) { SourceAssetInfoResponse response; @@ -407,6 +428,7 @@ AssetRequestHandler::AssetRequestHandler() m_requestRouter.RegisterMessageHandler(&HandleGetFullSourcePathFromRelativeProductPathRequest); m_requestRouter.RegisterMessageHandler(&HandleGetRelativeProductPathFromFullSourceOrProductPathRequest); + m_requestRouter.RegisterMessageHandler(&HandleGenerateRelativeSourcePathRequest); m_requestRouter.RegisterMessageHandler(&HandleSourceAssetInfoRequest); m_requestRouter.RegisterMessageHandler(&HandleSourceAssetProductsInfoRequest); m_requestRouter.RegisterMessageHandler(&HandleGetScanFoldersRequest); diff --git a/Code/Tools/AssetProcessor/native/AssetManager/assetProcessorManager.h b/Code/Tools/AssetProcessor/native/AssetManager/assetProcessorManager.h index 88f74c886b..3dc8bd7a00 100644 --- a/Code/Tools/AssetProcessor/native/AssetManager/assetProcessorManager.h +++ b/Code/Tools/AssetProcessor/native/AssetManager/assetProcessorManager.h @@ -57,6 +57,9 @@ namespace AzFramework class GetRelativeProductPathFromFullSourceOrProductPathRequest; class GetRelativeProductPathFromFullSourceOrProductPathResponse; + class GenerateRelativeSourcePathRequest; + class GenerateRelativeSourcePathResponse; + class GetFullSourcePathFromRelativeProductPathRequest; class GetFullSourcePathFromRelativeProductPathResponse; class AssetNotificationMessage; @@ -104,6 +107,8 @@ namespace AssetProcessor using GetAbsoluteAssetDatabaseLocationResponse = AzToolsFramework::AssetSystem::GetAbsoluteAssetDatabaseLocationResponse; using GetRelativeProductPathFromFullSourceOrProductPathRequest = AzFramework::AssetSystem::GetRelativeProductPathFromFullSourceOrProductPathRequest; using GetRelativeProductPathFromFullSourceOrProductPathResponse = AzFramework::AssetSystem::GetRelativeProductPathFromFullSourceOrProductPathResponse; + using GenerateRelativeSourcePathRequest = AzFramework::AssetSystem::GenerateRelativeSourcePathRequest; + using GenerateRelativeSourcePathResponse = AzFramework::AssetSystem::GenerateRelativeSourcePathResponse; using GetFullSourcePathFromRelativeProductPathRequest = AzFramework::AssetSystem::GetFullSourcePathFromRelativeProductPathRequest; using GetFullSourcePathFromRelativeProductPathResponse = AzFramework::AssetSystem::GetFullSourcePathFromRelativeProductPathResponse; diff --git a/Code/Tools/AssetProcessor/native/tests/AssetCatalog/AssetCatalogUnitTests.cpp b/Code/Tools/AssetProcessor/native/tests/AssetCatalog/AssetCatalogUnitTests.cpp index 3d9ecd3f5e..52bedc7744 100644 --- a/Code/Tools/AssetProcessor/native/tests/AssetCatalog/AssetCatalogUnitTests.cpp +++ b/Code/Tools/AssetProcessor/native/tests/AssetCatalog/AssetCatalogUnitTests.cpp @@ -224,6 +224,18 @@ namespace AssetProcessor dbConn->SetScanFolder(newScanFolder); } + virtual void AddScanFolders( + const QDir& tempPath, AssetDatabaseConnection* dbConn, PlatformConfiguration& config, + const AZStd::vector& platforms) + { + // PATH DisplayName PortKey root recurse platforms order + AddScanFolder(ScanFolderInfo(tempPath.filePath("subfolder4"), "subfolder4", "subfolder4", false, false, platforms, -6), config, dbConn); // subfolder 4 overrides subfolder3 + AddScanFolder(ScanFolderInfo(tempPath.filePath("subfolder3"), "subfolder3", "subfolder3", false, false, platforms, -5), config, dbConn); // subfolder 3 overrides subfolder2 + AddScanFolder(ScanFolderInfo(tempPath.filePath("subfolder2"), "subfolder2", "subfolder2", false, true, platforms, -2), config, dbConn); // subfolder 2 overrides subfolder1 + AddScanFolder(ScanFolderInfo(tempPath.filePath("subfolder1"), "subfolder1", "subfolder1", false, true, platforms, -1), config, dbConn); // subfolder1 overrides root + AddScanFolder(ScanFolderInfo(tempPath.absolutePath(), "temp", "tempfolder", true, false, platforms, 0), config, dbConn); // add the root + } + // build some default configs. void BuildConfig(const QDir& tempPath, AssetDatabaseConnection* dbConn, PlatformConfiguration& config) { @@ -232,12 +244,8 @@ namespace AssetProcessor config.EnablePlatform({ "fandango" ,{ "console", "renderer" } }, false); AZStd::vector platforms; config.PopulatePlatformsForScanFolder(platforms); - // PATH DisplayName PortKey root recurse platforms order - AddScanFolder(ScanFolderInfo(tempPath.filePath("subfolder4"), "subfolder4", "subfolder4", false, false, platforms, -6), config, dbConn); // subfolder 4 overrides subfolder3 - AddScanFolder(ScanFolderInfo(tempPath.filePath("subfolder3"), "subfolder3", "subfolder3", false, false, platforms, -5), config, dbConn); // subfolder 3 overrides subfolder2 - AddScanFolder(ScanFolderInfo(tempPath.filePath("subfolder2"), "subfolder2", "subfolder2", false, true, platforms, -2), config, dbConn); // subfolder 2 overrides subfolder1 - AddScanFolder(ScanFolderInfo(tempPath.filePath("subfolder1"), "subfolder1", "subfolder1", false, true, platforms, -1), config, dbConn); // subfolder1 overrides root - AddScanFolder(ScanFolderInfo(tempPath.absolutePath(), "temp", "tempfolder", true, false, platforms, 0), config, dbConn); // add the root + + AddScanFolders(tempPath, dbConn, config, platforms); config.AddMetaDataType("exportsettings", QString()); @@ -359,7 +367,8 @@ namespace AssetProcessor return false; } - // Calls the GetFullSourcePathFromRelativeProductPath function and checks the return results, returning true if it matches both of the expected results + // Calls the GetFullSourcePathFromRelativeProductPath function and checks the return results, returning true if it matches both of + // the expected results bool TestGetFullSourcePath(const QString& fileToCheck, const QDir& tempPath, bool expectToFind, const char* expectedPath) { bool fullPathfound = false; @@ -531,6 +540,177 @@ namespace AssetProcessor ASSERT_TRUE(TestGetRelativeProductPath(fileToCheck, true, { "aaa/basefile.txt" })); } + class AssetCatalogTestRelativeSourcePath : public AssetCatalogTest + { + public: + QDir GetRoot() + { + // Return an OS-friendly absolute root directory for our tests ("C:/sourceRoot" or "/sourceRoot"). It doesn't + // need to exist, it just needs to be an absolute path. + return QDir::root().filePath("sourceRoot"); + } + + // Set up custom scan folders for the "relative source path" tests, so that we can try out specific combinations of watch folders + void AddScanFolders( + [[maybe_unused]] const QDir& tempPath, AssetDatabaseConnection* dbConn, PlatformConfiguration& config, + const AZStd::vector& platforms) override + { + QDir root = GetRoot(); + + // This will set up the following watch folders, in highest to lowest priority: + + // /sourceRoot/recurseNested/nested (recurse) + // /sourceRoot/noRecurse (no recurse) + // /sourceRoot/recurseNotNested (recurse) + // /sourceRoot/recurseNested (recurse) + + AddScanFolder( + ScanFolderInfo(root.filePath("recurseNested/nested"), "nested", "nested", false, true, platforms, -4), config, dbConn); + AddScanFolder( + ScanFolderInfo(root.filePath("noRecurse"), "noRecurse", "noRecurse", false, false, platforms, -3), config, dbConn); + AddScanFolder( + ScanFolderInfo(root.filePath("recurseNotNested"), "recurseNotNested", "recurseNotNested", false, true, platforms, -2), + config, dbConn); + AddScanFolder( + ScanFolderInfo(root.filePath("recurseNested"), "recurseNested", "recurseNested", false, true, platforms, -1), + config, dbConn); + } + + // Calls the GenerateRelativeSourcePath function and validates that the results match the expected inputs. + void TestGetRelativeSourcePath( + const AZStd::string& sourcePath, bool expectedToFind, const AZStd::string& expectedPath, const AZStd::string& expectedRoot) + { + bool relPathFound = false; + AZStd::string relPath; + AZStd::string rootFolder; + + AzToolsFramework::AssetSystemRequestBus::BroadcastResult( + relPathFound, &AzToolsFramework::AssetSystem::AssetSystemRequest::GenerateRelativeSourcePath, sourcePath, + relPath, rootFolder); + + EXPECT_EQ(relPathFound, expectedToFind); + EXPECT_EQ(relPath, expectedPath); + EXPECT_EQ(rootFolder, expectedRoot); + } + }; + + TEST_F(AssetCatalogTestRelativeSourcePath, GenerateRelativeSourcePath_EmptySourcePath_ReturnsNoMatch) + { + // Test passes in an empty source path, which shouldn't produce a valid result. + // Input: empty source path + // Output: empty, not found result + TestGetRelativeSourcePath("", false, "", ""); + } + + TEST_F(AssetCatalogTestRelativeSourcePath, GenerateRelativeSourcePath_AbsolutePathOutsideWatchFolders_ReturnsNoMatch) + { + // Test passes in an invalid absolute source path, which shouldn't produce a valid result. + // Input: "/sourceRoot/noWatchFolder/test.txt" + // Output: not found result, which also returns the input as the relative file name + QDir watchFolder = GetRoot().filePath("noWatchFolder/"); + QString fileToCheck = watchFolder.filePath("test.txt"); + + TestGetRelativeSourcePath(fileToCheck.toUtf8().constData(), false, fileToCheck.toUtf8().constData(), ""); + } + + TEST_F(AssetCatalogTestRelativeSourcePath, GenerateRelativeSourcePath_AbsolutePathUnderWatchFolder_ReturnsRelativePath) + { + // Test passes in a valid absolute source path, which should produce a valid relative path + // Input: "/sourceRoot/noRecurse/test.txt" + // Output: "test.txt" in folder "/sourceRoot/noRecurse/" + QDir watchFolder = GetRoot().filePath("noRecurse/"); + QString fileToCheck = watchFolder.filePath("test.txt"); + + TestGetRelativeSourcePath(fileToCheck.toUtf8().constData(), true, "test.txt", watchFolder.path().toUtf8().constData()); + } + + TEST_F(AssetCatalogTestRelativeSourcePath, GenerateRelativeSourcePath_AbsolutePathUnderNestedWatchFolders_ReturnsRelativePath) + { + // Test passes in a valid absolute source path that matches a watch folder and a nested watch folder. + // The output relative path should match the nested folder, because the nested folder has a higher priority registered with the AP. + // Input: "/sourceRoot/recurseNested/nested/test.txt" + // Output: "test.txt" in folder "/sourceRoot/recurseNested/nested/" + QDir watchFolder = GetRoot().filePath("recurseNested/nested/"); + QString fileToCheck = watchFolder.filePath("test.txt"); + + TestGetRelativeSourcePath(fileToCheck.toUtf8().constData(), true, "test.txt", watchFolder.path().toUtf8().constData()); + } + + TEST_F(AssetCatalogTestRelativeSourcePath, GenerateRelativeSourcePath_BareFileNameValidInWatchFolder_ReturnsHighestPriorityWatchFolder) + { + // Test passes in a simple file name. The output should be relative to the highest-priority watch folder. + // Input: "test.txt" + // Output: "test.txt" in folder "/sourceRoot/recurseNested/nested/" + QDir watchFolder = GetRoot().filePath("recurseNested/nested/"); + + TestGetRelativeSourcePath("test.txt", true, "test.txt", watchFolder.path().toUtf8().constData()); + } + + TEST_F(AssetCatalogTestRelativeSourcePath, GenerateRelativeSourcePath_RelativePathValidInWatchFolder_ReturnsHighestPriorityWatchFolder) + { + // Test passes in a relative path. The output should preserve the relative path, but list it as relative to the highest-priority + // watch folder. + // Input: "a/b/c/test.txt" + // Output: "a/b/c/test.txt" in folder "/sourceRoot/recurseNested/nested/" + QDir watchFolder = GetRoot().filePath("recurseNested/nested/"); + + TestGetRelativeSourcePath("a/b/c/test.txt", true, "a/b/c/test.txt", watchFolder.path().toUtf8().constData()); + } + + TEST_F(AssetCatalogTestRelativeSourcePath, GenerateRelativeSourcePath_RelativePathNotInWatchFolder_ReturnsNoMatch) + { + // Test passes in a relative path that "backs up" two directories. This will be invalid, because no matter which watch directory + // we start at, the result will be outside of any watch directory. + // Input: "../../test.txt" + // Output: not found result, which also returns the input as the relative file name + TestGetRelativeSourcePath("../../test.txt", false, "../../test.txt", ""); + } + + TEST_F(AssetCatalogTestRelativeSourcePath, GenerateRelativeSourcePath_RelativePathValidFromNestedWatchFolder_ReturnsOuterFolder) + { + // Test passes in a relative path that "backs up" one directory. This will produce a valid result, because we can back up from + // the "recurseNested/nested/" watch folder to "recurseNested", which is also a valid watch folder. + // Input: "../test.txt" + // Output: "test.txt" in folder "/sourceRoot/recurseNested" + QDir watchFolder = GetRoot().filePath("recurseNested/"); + TestGetRelativeSourcePath("../test.txt", true, "test.txt", watchFolder.path().toUtf8().constData()); + } + + TEST_F(AssetCatalogTestRelativeSourcePath, GenerateRelativeSourcePath_RelativePathMovesToParentWatchFolder_ReturnsOuterFolder) + { + // Test passes in a relative path that backs up one directory and then forward into a directory. This will produce a valid + // result, because it can validly start in the highest-priority watch folder (recurseNested/nested), move back one into the + // outer watch folder (recurseNested), and then have a subdirectory within it. + // Note that it would also be valid to move from recurseNested to recurseNotNested, but that won't be the result of this test + // because that's a lower-priority match. + // Input: "../recurseNotNested/test.txt" + // Output: "recurseNotNested/test.txt" in folder "/sourceRoot/recurseNested/" + QDir watchFolder = GetRoot().filePath("recurseNested/"); + + TestGetRelativeSourcePath("../recurseNotNested/test.txt", true, "recurseNotNested/test.txt", watchFolder.path().toUtf8().constData()); + } + + TEST_F(AssetCatalogTestRelativeSourcePath, GenerateRelativeSourcePath_RelativePathMovesToSiblingWatchFolder_ReturnsSiblingFolder) + { + // Test passes in a relative path that backs up two directories and then forward into a directory. This will produce a valid + // result, because it can validly start in the recurseNested/nested folder, move back two folders, then forward into the sibling + // recurseNotNested folder. The result will be a relative path to the sibling folder. + // Input: "../../recurseNotNested/test.txt" + // Output: "test.txt" in folder "/sourceRoot/recurseNotNested/" + QDir watchFolder = GetRoot().filePath("recurseNotNested/"); + + TestGetRelativeSourcePath("../../recurseNotNested/test.txt", true, "test.txt", watchFolder.path().toUtf8().constData()); + } + + TEST_F(AssetCatalogTestRelativeSourcePath, GenerateRelativeSourcePath_RelativePathBacksOutOfWatchFolder_ReturnsNoMatch) + { + // Test passes in a relative path that adds a directory, then "backs up" three directories. This will be invalid, because no + // matter which watch directory we start at, the result will be outside of any watch directory. + // Input: "../test.txt" + // Output: "test.txt" in folder "/sourceRoot/recurseNested" + TestGetRelativeSourcePath("a/../../../test.txt", false, "a/../../../test.txt", ""); + } + class AssetCatalogTest_GetFullSourcePath : public AssetCatalogTest { diff --git a/Code/Tools/AssetProcessor/native/tests/AssetProcessorMessagesTests.cpp b/Code/Tools/AssetProcessor/native/tests/AssetProcessorMessagesTests.cpp index c33943f9a7..04949c82a9 100644 --- a/Code/Tools/AssetProcessor/native/tests/AssetProcessorMessagesTests.cpp +++ b/Code/Tools/AssetProcessor/native/tests/AssetProcessorMessagesTests.cpp @@ -265,6 +265,9 @@ namespace AssetProcessorMessagesTests addPairFunc(new GetFullSourcePathFromRelativeProductPathRequest(), new GetFullSourcePathFromRelativeProductPathResponse()); addPairFunc(new GetRelativeProductPathFromFullSourceOrProductPathRequest(), new GetRelativeProductPathFromFullSourceOrProductPathResponse()); + addPairFunc( + new GenerateRelativeSourcePathRequest(), + new GenerateRelativeSourcePathResponse()); addPairFunc(new SourceAssetInfoRequest(), new SourceAssetInfoResponse()); addPairFunc(new SourceAssetProductsInfoRequest(), new SourceAssetProductsInfoResponse()); addPairFunc(new GetScanFoldersRequest(), new GetScanFoldersResponse()); diff --git a/Code/Tools/AssetProcessor/native/unittests/AssetProcessorManagerUnitTests.cpp b/Code/Tools/AssetProcessor/native/unittests/AssetProcessorManagerUnitTests.cpp index 5940caadb7..c632dc8a7a 100644 --- a/Code/Tools/AssetProcessor/native/unittests/AssetProcessorManagerUnitTests.cpp +++ b/Code/Tools/AssetProcessor/native/unittests/AssetProcessorManagerUnitTests.cpp @@ -51,6 +51,8 @@ namespace AssetProcessor public: using GetRelativeProductPathFromFullSourceOrProductPathRequest = AzFramework::AssetSystem::GetRelativeProductPathFromFullSourceOrProductPathRequest; using GetRelativeProductPathFromFullSourceOrProductPathResponse = AzFramework::AssetSystem::GetRelativeProductPathFromFullSourceOrProductPathResponse; + using GenerateRelativeSourcePathRequest = AzFramework::AssetSystem::GenerateRelativeSourcePathRequest; + using GenerateRelativeSourcePathResponse = AzFramework::AssetSystem::GenerateRelativeSourcePathResponse; using GetFullSourcePathFromRelativeProductPathRequest = AzFramework::AssetSystem::GetFullSourcePathFromRelativeProductPathRequest; using GetFullSourcePathFromRelativeProductPathResponse = AzFramework::AssetSystem::GetFullSourcePathFromRelativeProductPathResponse; }; diff --git a/Gems/Atom/RPI/Code/Tests/Common/AssetSystemStub.cpp b/Gems/Atom/RPI/Code/Tests/Common/AssetSystemStub.cpp index 84e8a4ba65..6b9c503ec2 100644 --- a/Gems/Atom/RPI/Code/Tests/Common/AssetSystemStub.cpp +++ b/Gems/Atom/RPI/Code/Tests/Common/AssetSystemStub.cpp @@ -72,7 +72,15 @@ namespace UnitTest return false; } - bool AssetSystemStub::GetFullSourcePathFromRelativeProductPath([[maybe_unused]] const AZStd::string& relPath, [[maybe_unused]] AZStd::string& fullSourcePath) + bool AssetSystemStub::GenerateRelativeSourcePath( + [[maybe_unused]] const AZStd::string& sourcePath, [[maybe_unused]] AZStd::string& relativePath, + [[maybe_unused]] AZStd::string& watchFolder) + { + return false; + } + + bool AssetSystemStub::GetFullSourcePathFromRelativeProductPath( + [[maybe_unused]] const AZStd::string& relPath, [[maybe_unused]] AZStd::string& fullSourcePath) { return false; } diff --git a/Gems/Atom/RPI/Code/Tests/Common/AssetSystemStub.h b/Gems/Atom/RPI/Code/Tests/Common/AssetSystemStub.h index c6f4ac891f..48609ed0cb 100644 --- a/Gems/Atom/RPI/Code/Tests/Common/AssetSystemStub.h +++ b/Gems/Atom/RPI/Code/Tests/Common/AssetSystemStub.h @@ -63,6 +63,8 @@ namespace UnitTest const char* GetAbsoluteDevGameFolderPath() override; const char* GetAbsoluteDevRootFolderPath() override; bool GetRelativeProductPathFromFullSourceOrProductPath(const AZStd::string& fullPath, AZStd::string& relativeProductPath) override; + bool GenerateRelativeSourcePath( + const AZStd::string& sourcePath, AZStd::string& relativePath, AZStd::string& watchFolder) override; bool GetFullSourcePathFromRelativeProductPath(const AZStd::string& relPath, AZStd::string& fullSourcePath) override; bool GetAssetInfoById(const AZ::Data::AssetId& assetId, const AZ::Data::AssetType& assetType, const AZStd::string& platformName, AZ::Data::AssetInfo& assetInfo, AZStd::string& rootFilePath) override; bool GetSourceInfoBySourceUUID(const AZ::Uuid& sourceUuid, AZ::Data::AssetInfo& assetInfo, AZStd::string& watchFolder) override; diff --git a/Gems/LmbrCentral/Code/Tests/Builders/CopyDependencyBuilderTest.cpp b/Gems/LmbrCentral/Code/Tests/Builders/CopyDependencyBuilderTest.cpp index f082ef74f9..44fe70af80 100644 --- a/Gems/LmbrCentral/Code/Tests/Builders/CopyDependencyBuilderTest.cpp +++ b/Gems/LmbrCentral/Code/Tests/Builders/CopyDependencyBuilderTest.cpp @@ -217,6 +217,9 @@ protected: const char* GetAbsoluteDevGameFolderPath() override { return ""; } const char* GetAbsoluteDevRootFolderPath() override { return ""; } bool GetRelativeProductPathFromFullSourceOrProductPath([[maybe_unused]] const AZStd::string& fullPath, [[maybe_unused]] AZStd::string& relativeProductPath) { return true; } + bool GenerateRelativeSourcePath( + [[maybe_unused]] const AZStd::string& sourcePath, [[maybe_unused]] AZStd::string& relativePath, + [[maybe_unused]] AZStd::string& watchFolder) { return true; } bool GetFullSourcePathFromRelativeProductPath([[maybe_unused]] const AZStd::string& relPath, [[maybe_unused]] AZStd::string& fullSourcePath) { return true; } bool GetAssetInfoById([[maybe_unused]] const AZ::Data::AssetId& assetId, [[maybe_unused]] const AZ::Data::AssetType& assetType, [[maybe_unused]] const AZStd::string& platformName, [[maybe_unused]] AZ::Data::AssetInfo& assetInfo, [[maybe_unused]] AZStd::string& rootFilePath) { return true; } bool GetSourceInfoBySourcePath([[maybe_unused]] const char* sourcePath, [[maybe_unused]] AZ::Data::AssetInfo& assetInfo, [[maybe_unused]] AZStd::string& watchFolder) { return true; }