From ae50187fba06f39a2679a9312c88155b6072cc1d Mon Sep 17 00:00:00 2001 From: amzn-mike <80125227+amzn-mike@users.noreply.github.com> Date: Thu, 18 Nov 2021 13:37:48 -0600 Subject: [PATCH] [LYN-7520] Wildcard Source Dependencies include files in cache/excluded files (#5349) * Add folder exclusion for wildcard source dependencies Signed-off-by: amzn-mike <80125227+amzn-mike@users.noreply.github.com> * Exclude ignored files. Add unit tests Signed-off-by: amzn-mike <80125227+amzn-mike@users.noreply.github.com> * Add handling for ignored folders being added/removed Add unit tests Signed-off-by: amzn-mike <80125227+amzn-mike@users.noreply.github.com> * Add ExcludedFolderCacheInterface to cmake Signed-off-by: amzn-mike <80125227+amzn-mike@users.noreply.github.com> * Fix include Signed-off-by: amzn-mike <80125227+amzn-mike@users.noreply.github.com> * Add error message Signed-off-by: amzn-mike <80125227+amzn-mike@users.noreply.github.com> * Cleanup includes Signed-off-by: amzn-mike <80125227+amzn-mike@users.noreply.github.com> * Revert traits include Signed-off-by: amzn-mike <80125227+amzn-mike@users.noreply.github.com> * Fix missing include, minor cleanup Signed-off-by: amzn-mike <80125227+amzn-mike@users.noreply.github.com> * Add missing includes Signed-off-by: amzn-mike <80125227+amzn-mike@users.noreply.github.com> --- .../assetprocessor_static_files.cmake | 3 + .../AssetManager/ExcludedFolderCache.cpp | 154 ++++++++++++++++++ .../native/AssetManager/ExcludedFolderCache.h | 39 +++++ .../ExcludedFolderCacheInterface.h | 29 ++++ .../native/AssetManager/FileStateCache.cpp | 22 +++ .../native/AssetManager/FileStateCache.h | 15 +- .../AssetManager/assetProcessorManager.cpp | 27 ++- .../AssetManager/assetProcessorManager.h | 7 +- .../native/tests/AssetProcessorTest.h | 8 +- .../AssetProcessorManagerTest.cpp | 127 ++++++++++++++- .../utilities/ApplicationManagerBase.cpp | 2 + .../utilities/PlatformConfiguration.cpp | 73 ++++++++- .../native/utilities/PlatformConfiguration.h | 14 +- 13 files changed, 506 insertions(+), 14 deletions(-) create mode 100644 Code/Tools/AssetProcessor/native/AssetManager/ExcludedFolderCache.cpp create mode 100644 Code/Tools/AssetProcessor/native/AssetManager/ExcludedFolderCache.h create mode 100644 Code/Tools/AssetProcessor/native/AssetManager/ExcludedFolderCacheInterface.h diff --git a/Code/Tools/AssetProcessor/assetprocessor_static_files.cmake b/Code/Tools/AssetProcessor/assetprocessor_static_files.cmake index 0c1347517f..bce2458477 100644 --- a/Code/Tools/AssetProcessor/assetprocessor_static_files.cmake +++ b/Code/Tools/AssetProcessor/assetprocessor_static_files.cmake @@ -29,6 +29,9 @@ set(FILES native/AssetManager/SourceFileRelocator.h native/AssetManager/ControlRequestHandler.cpp native/AssetManager/ControlRequestHandler.h + native/AssetManager/ExcludedFolderCache.cpp + native/AssetManager/ExcludedFolderCache.h + native/AssetManager/ExcludedFolderCacheInterface.h native/assetprocessor.h native/connection/connection.cpp native/connection/connection.h diff --git a/Code/Tools/AssetProcessor/native/AssetManager/ExcludedFolderCache.cpp b/Code/Tools/AssetProcessor/native/AssetManager/ExcludedFolderCache.cpp new file mode 100644 index 0000000000..2c8dc844dd --- /dev/null +++ b/Code/Tools/AssetProcessor/native/AssetManager/ExcludedFolderCache.cpp @@ -0,0 +1,154 @@ +/* + * Copyright (c) Contributors to the Open 3D Engine Project. + * For complete copyright and license terms please see the LICENSE at the root of this distribution. + * + * SPDX-License-Identifier: Apache-2.0 OR MIT + * + */ + +#include +#include +#include +#include +#include +#include + +namespace AssetProcessor +{ + ExcludedFolderCache::ExcludedFolderCache(const PlatformConfiguration* platformConfig) : m_platformConfig(platformConfig) + { + AZ::Interface::Register(this); + } + + ExcludedFolderCache::~ExcludedFolderCache() + { + AZ::Interface::Unregister(this); + } + + const AZStd::unordered_set& ExcludedFolderCache::GetExcludedFolders() + { + if (!m_builtCache) + { + for (int i = 0; i < m_platformConfig->GetScanFolderCount(); ++i) + { + const auto& scanFolderInfo = m_platformConfig->GetScanFolderAt(i); + QDir rooted(scanFolderInfo.ScanPath()); + QString absolutePath = rooted.absolutePath(); + AZStd::stack dirs; + dirs.push(absolutePath); + + while (!dirs.empty()) + { + absolutePath = dirs.top(); + dirs.pop(); + + // Scan only folders, do not recurse so we have the chance to ignore a subfolder before going deeper + QDirIterator dirIterator(absolutePath, QDir::Dirs | QDir::NoSymLinks | QDir::NoDotAndDotDot); + + // Loop all the folders in this directory + while (dirIterator.hasNext()) + { + dirIterator.next(); + QString pathMatch = rooted.absoluteFilePath(dirIterator.filePath()); + + if (m_platformConfig->IsFileExcluded(pathMatch)) + { + // Add the folder to the list and do not proceed any deeper + m_excludedFolders.emplace(pathMatch.toUtf8().constData()); + } + else if (scanFolderInfo.RecurseSubFolders()) + { + // Folder is not excluded and recurse is enabled, add to the list of folders to check + dirs.push(pathMatch); + } + } + } + } + + // Add the cache to the list as well + AZStd::string projectCacheRootValue; + AZ::SettingsRegistry::Get()->Get(projectCacheRootValue, AZ::SettingsRegistryMergeUtils::FilePathKey_CacheProjectRootFolder); + projectCacheRootValue = AssetUtilities::NormalizeFilePath(projectCacheRootValue.c_str()).toUtf8().constData(); + m_excludedFolders.emplace(projectCacheRootValue); + + // Register to be notified about deletes so we can remove old ignored folders + auto fileStateCache = AZ::Interface::Get(); + + if (fileStateCache) + { + m_handler = AZ::Event::Handler([this](FileStateInfo fileInfo) + { + if (fileInfo.m_isDirectory) + { + AZStd::scoped_lock lock(m_pendingNewFolderMutex); + + m_pendingDeletes.emplace(fileInfo.m_absolutePath.toUtf8().constData()); + } + }); + + fileStateCache->RegisterForDeleteEvent(m_handler); + } + else + { + AZ_Error("ExcludedFolderCache", false, "Failed to find IFileStateRequests interface"); + } + + m_builtCache = true; + } + + // Incorporate any pending folders + AZStd::unordered_set pendingAdds; + AZStd::unordered_set pendingDeletes; + + { + AZStd::scoped_lock lock(m_pendingNewFolderMutex); + pendingAdds.swap(m_pendingNewFolders); + pendingDeletes.swap(m_pendingDeletes); + } + + if (!pendingAdds.empty()) + { + m_excludedFolders.insert(pendingAdds.begin(), pendingAdds.end()); + } + + if (!pendingDeletes.empty()) + { + for (const auto& pendingDelete : pendingDeletes) + { + m_excludedFolders.erase(pendingDelete); + } + } + + return m_excludedFolders; + } + + void ExcludedFolderCache::FileAdded(QString path) + { + QString relativePath, scanFolderPath; + + if (!m_platformConfig->ConvertToRelativePath(path, relativePath, scanFolderPath)) + { + AZ_Error("ExcludedFolderCache", false, "Failed to get relative path for newly added file %s", path.toUtf8().constData()); + return; + } + + AZ::IO::Path azPath(relativePath.toUtf8().constData()); + AZ::IO::Path absolutePath(scanFolderPath.toUtf8().constData()); + + for (const auto& pathPart : azPath) + { + absolutePath /= pathPart; + + QString normalized = AssetUtilities::NormalizeFilePath(absolutePath.c_str()); + + if (m_platformConfig->IsFileExcluded(normalized)) + { + // Add the folder to a pending list, since this callback runs on another thread + AZStd::scoped_lock lock(m_pendingNewFolderMutex); + + m_pendingNewFolders.emplace(normalized.toUtf8().constData()); + break; + } + } + } +} diff --git a/Code/Tools/AssetProcessor/native/AssetManager/ExcludedFolderCache.h b/Code/Tools/AssetProcessor/native/AssetManager/ExcludedFolderCache.h new file mode 100644 index 0000000000..b0c70946b6 --- /dev/null +++ b/Code/Tools/AssetProcessor/native/AssetManager/ExcludedFolderCache.h @@ -0,0 +1,39 @@ +/* + * Copyright (c) Contributors to the Open 3D Engine Project. + * For complete copyright and license terms please see the LICENSE at the root of this distribution. + * + * SPDX-License-Identifier: Apache-2.0 OR MIT + * + */ + +#pragma once + +#include +#include + +namespace AssetProcessor +{ + class PlatformConfiguration; + + struct ExcludedFolderCache : ExcludedFolderCacheInterface + { + explicit ExcludedFolderCache(const PlatformConfiguration* platformConfig); + ~ExcludedFolderCache() override; + + // Gets a set of absolute paths to folder which have been excluded according to the platform configuration rules + // Note - not thread safe + const AZStd::unordered_set& GetExcludedFolders() override; + + void FileAdded(QString path) override; + + private: + bool m_builtCache = false; + const PlatformConfiguration* m_platformConfig{}; + AZStd::unordered_set m_excludedFolders; + + AZStd::recursive_mutex m_pendingNewFolderMutex; + AZStd::unordered_set m_pendingNewFolders; // Newly ignored folders waiting to be added to m_excludedFolders + AZStd::unordered_set m_pendingDeletes; + AZ::Event::Handler m_handler; + }; +} diff --git a/Code/Tools/AssetProcessor/native/AssetManager/ExcludedFolderCacheInterface.h b/Code/Tools/AssetProcessor/native/AssetManager/ExcludedFolderCacheInterface.h new file mode 100644 index 0000000000..1bdc4cc855 --- /dev/null +++ b/Code/Tools/AssetProcessor/native/AssetManager/ExcludedFolderCacheInterface.h @@ -0,0 +1,29 @@ +/* + * Copyright (c) Contributors to the Open 3D Engine Project. + * For complete copyright and license terms please see the LICENSE at the root of this distribution. + * + * SPDX-License-Identifier: Apache-2.0 OR MIT + * + */ + +#pragma once + +#include +#include + +namespace AssetProcessor +{ + class PlatformConfiguration; + + struct ExcludedFolderCacheInterface + { + AZ_RTTI(ExcludedFolderCacheInterface, "{3AC471B6-C9F8-49CF-9E9D-237BDF63328C}"); + AZ_DISABLE_COPY_MOVE(ExcludedFolderCacheInterface); + + ExcludedFolderCacheInterface() = default; + virtual ~ExcludedFolderCacheInterface() = default; + + virtual const AZStd::unordered_set& GetExcludedFolders() = 0; + virtual void FileAdded(QString path) = 0; + }; +} diff --git a/Code/Tools/AssetProcessor/native/AssetManager/FileStateCache.cpp b/Code/Tools/AssetProcessor/native/AssetManager/FileStateCache.cpp index d21aca35f5..e49d1bbb47 100644 --- a/Code/Tools/AssetProcessor/native/AssetManager/FileStateCache.cpp +++ b/Code/Tools/AssetProcessor/native/AssetManager/FileStateCache.cpp @@ -63,6 +63,11 @@ namespace AssetProcessor return true; } + void FileStateCache::RegisterForDeleteEvent(AZ::Event::Handler& handler) + { + handler.Connect(m_deleteEvent); + } + void FileStateCache::AddInfoSet(QSet infoSet) { LockGuardType scopeLock(m_mapMutex); @@ -103,6 +108,8 @@ namespace AssetProcessor if (itr != m_fileInfoMap.end()) { + m_deleteEvent.Signal(itr.value()); + bool isDirectory = itr.value().m_isDirectory; QString parentPath = itr.value().m_absolutePath; m_fileInfoMap.erase(itr); @@ -205,6 +212,21 @@ namespace AssetProcessor return true; } + void FileStatePassthrough::RegisterForDeleteEvent(AZ::Event::Handler& handler) + { + handler.Connect(m_deleteEvent); + } + + void FileStatePassthrough::SignalDeleteEvent(const QString& absolutePath) const + { + FileStateInfo info; + + if (GetFileInfo(absolutePath, &info)) + { + m_deleteEvent.Signal(info); + } + } + bool FileStateInfo::operator==(const FileStateInfo& rhs) const { return m_absolutePath == rhs.m_absolutePath diff --git a/Code/Tools/AssetProcessor/native/AssetManager/FileStateCache.h b/Code/Tools/AssetProcessor/native/AssetManager/FileStateCache.h index ba663aef0e..56ec03aa7d 100644 --- a/Code/Tools/AssetProcessor/native/AssetManager/FileStateCache.h +++ b/Code/Tools/AssetProcessor/native/AssetManager/FileStateCache.h @@ -14,6 +14,7 @@ #include #include #include +#include namespace AssetProcessor { @@ -51,10 +52,11 @@ namespace AssetProcessor /// Convenience function to check if a file or directory exists. virtual bool Exists(const QString& absolutePath) const = 0; virtual bool GetHash(const QString& absolutePath, FileHash* foundHash) = 0; + virtual void RegisterForDeleteEvent(AZ::Event::Handler& handler) = 0; AZ_DISABLE_COPY_MOVE(IFileStateRequests); }; - + class FileStateBase : public IFileStateRequests { @@ -89,11 +91,11 @@ namespace AssetProcessor { public: - // FileStateRequestBus implementation bool GetFileInfo(const QString& absolutePath, FileStateInfo* foundFileInfo) const override; bool Exists(const QString& absolutePath) const override; bool GetHash(const QString& absolutePath, FileHash* foundHash) override; + void RegisterForDeleteEvent(AZ::Event::Handler& handler) override; void AddInfoSet(QSet infoSet) override; void AddFile(const QString& absolutePath) override; @@ -116,9 +118,11 @@ namespace AssetProcessor mutable AZStd::recursive_mutex m_mapMutex; QHash m_fileInfoMap; - + QHash m_fileHashMap; + AZ::Event m_deleteEvent; + using LockGuardType = AZStd::lock_guard; }; @@ -131,5 +135,10 @@ namespace AssetProcessor bool GetFileInfo(const QString& absolutePath, FileStateInfo* foundFileInfo) const override; bool Exists(const QString& absolutePath) const override; bool GetHash(const QString& absolutePath, FileHash* foundHash) override; + void RegisterForDeleteEvent(AZ::Event::Handler& handler) override; + + void SignalDeleteEvent(const QString& absolutePath) const; + protected: + AZ::Event m_deleteEvent; }; } // namespace AssetProcessor diff --git a/Code/Tools/AssetProcessor/native/AssetManager/assetProcessorManager.cpp b/Code/Tools/AssetProcessor/native/AssetManager/assetProcessorManager.cpp index 367a296bd4..05fe760bcb 100644 --- a/Code/Tools/AssetProcessor/native/AssetManager/assetProcessorManager.cpp +++ b/Code/Tools/AssetProcessor/native/AssetManager/assetProcessorManager.cpp @@ -18,8 +18,8 @@ #include - #include "native/AssetManager/assetProcessorManager.h" + #include #include @@ -66,8 +66,10 @@ namespace AssetProcessor m_sourceFileRelocator = AZStd::make_unique(m_stateData, m_platformConfig); - PopulateJobStateCache(); + m_excludedFolderCache = AZStd::make_unique(m_platformConfig); + PopulateJobStateCache(); + AssetProcessor::ProcessingJobInfoBus::Handler::BusConnect(); } @@ -3573,6 +3575,8 @@ namespace AssetProcessor QString knownPathBeforeWildcard = encodedFileData.left(slashBeforeWildcardIndex + 1); // include the slash QString relativeSearch = encodedFileData.mid(slashBeforeWildcardIndex + 1); // skip the slash + const auto& excludedFolders = m_excludedFolderCache->GetExcludedFolders(); + // Absolute path, just check the 1 scan folder if (AZ::IO::PathView(encodedFileData.toUtf8().constData()).IsAbsolute()) { @@ -3592,7 +3596,8 @@ namespace AssetProcessor QString scanFolderAndKnownSubPath = rooted.absoluteFilePath(knownPathBeforeWildcard); resolvedDependencyList.append(m_platformConfig->FindWildcardMatches( - scanFolderAndKnownSubPath, relativeSearch, false, scanFolderInfo->RecurseSubFolders())); + scanFolderAndKnownSubPath, relativeSearch, + excludedFolders, false, scanFolderInfo->RecurseSubFolders())); } } else // Relative path, check every scan folder @@ -3610,7 +3615,21 @@ namespace AssetProcessor QString absolutePath = rooted.absoluteFilePath(knownPathBeforeWildcard); resolvedDependencyList.append(m_platformConfig->FindWildcardMatches( - absolutePath, relativeSearch, false, scanFolderInfo->RecurseSubFolders())); + absolutePath, relativeSearch, + excludedFolders, false, scanFolderInfo->RecurseSubFolders())); + } + } + + // Filter out any excluded files + for (auto itr = resolvedDependencyList.begin(); itr != resolvedDependencyList.end();) + { + if (m_platformConfig->IsFileExcluded(*itr)) + { + itr = resolvedDependencyList.erase(itr); + } + else + { + ++itr; } } diff --git a/Code/Tools/AssetProcessor/native/AssetManager/assetProcessorManager.h b/Code/Tools/AssetProcessor/native/AssetManager/assetProcessorManager.h index 891e091f91..fe77396760 100644 --- a/Code/Tools/AssetProcessor/native/AssetManager/assetProcessorManager.h +++ b/Code/Tools/AssetProcessor/native/AssetManager/assetProcessorManager.h @@ -40,6 +40,8 @@ #include "AssetRequestHandler.h" #include "native/utilities/JobDiagnosticTracker.h" #include "SourceFileRelocator.h" + +#include #endif class FileWatcher; @@ -341,7 +343,8 @@ namespace AssetProcessor void CleanEmptyFolder(QString folder, QString root); void ProcessBuilders(QString normalizedPath, QString relativePathToFile, const ScanFolderInfo* scanFolder, const AssetProcessor::BuilderInfoList& builderInfoList); - + AZStd::vector GetExcludedFolders(); + struct SourceInfo { QString m_watchFolder; @@ -552,6 +555,8 @@ namespace AssetProcessor // when true, a flag will be sent to builders process job indicating debug output/mode should be used bool m_builderDebugFlag = false; + AZStd::unique_ptr m_excludedFolderCache{}; + protected Q_SLOTS: void FinishAnalysis(AZStd::string fileToCheck); ////////////////////////////////////////////////////////// diff --git a/Code/Tools/AssetProcessor/native/tests/AssetProcessorTest.h b/Code/Tools/AssetProcessor/native/tests/AssetProcessorTest.h index 8f26155fd3..fcfd831d0b 100644 --- a/Code/Tools/AssetProcessor/native/tests/AssetProcessorTest.h +++ b/Code/Tools/AssetProcessor/native/tests/AssetProcessorTest.h @@ -26,7 +26,7 @@ namespace AssetProcessor { protected: AZStd::unique_ptr m_errorAbsorber{}; - FileStatePassthrough m_fileStateCache; + AZStd::unique_ptr m_fileStateCache{}; void SetUp() override { @@ -40,9 +40,10 @@ namespace AssetProcessor m_ownsSysAllocator = true; AZ::AllocatorInstance::Create(); } - m_errorAbsorber = AZStd::make_unique(); + m_errorAbsorber = AZStd::make_unique(); m_application = AZStd::make_unique(); + m_fileStateCache = AZStd::make_unique(); // Inject the AutomatedTesting project as a project path into test fixture using FixedValueString = AZ::SettingsRegistryInterface::FixedValueString; @@ -60,7 +61,8 @@ namespace AssetProcessor void TearDown() override { AssetUtilities::ResetAssetRoot(); - + + m_fileStateCache.reset(); m_application.reset(); m_errorAbsorber.reset(); diff --git a/Code/Tools/AssetProcessor/native/tests/assetmanager/AssetProcessorManagerTest.cpp b/Code/Tools/AssetProcessor/native/tests/assetmanager/AssetProcessorManagerTest.cpp index 93344caf6e..58f76f8da6 100644 --- a/Code/Tools/AssetProcessor/native/tests/assetmanager/AssetProcessorManagerTest.cpp +++ b/Code/Tools/AssetProcessor/native/tests/assetmanager/AssetProcessorManagerTest.cpp @@ -5364,13 +5364,29 @@ AZStd::vector WildcardSourceDependencyTest::FileAddedTest(const Q void WildcardSourceDependencyTest::SetUp() { AssetProcessorManagerTest::SetUp(); - + QDir tempPath(m_tempDir.path()); // Add a non-recursive scan folder. Only files directly inside of this folder should be picked up, subfolders are ignored m_config->AddScanFolder(ScanFolderInfo(tempPath.filePath("no_recurse"), "no_recurse", "no_recurse", false, false, m_config->GetEnabledPlatforms(), 1)); + { + ExcludeAssetRecognizer excludeFolder; + excludeFolder.m_name = "Exclude ignored Folder"; + excludeFolder.m_patternMatcher = + AssetBuilderSDK::FilePatternMatcher(R"REGEX(^(.*\/)?ignored(\/.*)?$)REGEX", AssetBuilderSDK::AssetBuilderPattern::Regex); + m_config->AddExcludeRecognizer(excludeFolder); + } + + { + ExcludeAssetRecognizer excludeFile; + excludeFile.m_name = "Exclude z.foo Files"; + excludeFile.m_patternMatcher = + AssetBuilderSDK::FilePatternMatcher(R"REGEX(^(.*\/)?z\.foo$)REGEX", AssetBuilderSDK::AssetBuilderPattern::Regex); + m_config->AddExcludeRecognizer(excludeFile); + } + UnitTestUtils::CreateDummyFile(tempPath.absoluteFilePath("subfolder1/1a.foo")); UnitTestUtils::CreateDummyFile(tempPath.absoluteFilePath("subfolder1/1b.foo")); UnitTestUtils::CreateDummyFile(tempPath.absoluteFilePath("subfolder2/redirected/a.foo")); @@ -5384,6 +5400,19 @@ void WildcardSourceDependencyTest::SetUp() // Add a file in the non-recursive scanfolder. Since its not directly in the scan folder, it should always be ignored UnitTestUtils::CreateDummyFile(tempPath.absoluteFilePath("no_recurse/one/two/three/f.foo")); + // Add a file to an ignored folder + UnitTestUtils::CreateDummyFile(tempPath.absoluteFilePath("subfolder2/redirected/folder/ignored/g.foo")); + + // Add an ignored file + UnitTestUtils::CreateDummyFile(tempPath.absoluteFilePath("subfolder2/redirected/folder/one/z.foo")); + + // Add a file in the cache + AZStd::string projectCacheRootValue; + AZ::SettingsRegistry::Get()->Get(projectCacheRootValue, AZ::SettingsRegistryMergeUtils::FilePathKey_CacheProjectRootFolder); + projectCacheRootValue = AssetUtilities::NormalizeFilePath(projectCacheRootValue.c_str()).toUtf8().constData(); + auto path = AZ::IO::Path(projectCacheRootValue) / "cache.foo"; + UnitTestUtils::CreateDummyFile(path.c_str()); + AzToolsFramework::AssetDatabase::SourceFileDependencyEntryContainer dependencies; // Relative path wildcard dependency @@ -5518,6 +5547,102 @@ TEST_F(WildcardSourceDependencyTest, Absolute_NoWildcard) ASSERT_THAT(resolvedPaths, ::testing::UnorderedElementsAre()); } +TEST_F(WildcardSourceDependencyTest, Relative_IgnoredFolder) +{ + AZStd::vector resolvedPaths; + + ASSERT_TRUE(Test("*g.foo", resolvedPaths)); + ASSERT_THAT(resolvedPaths, ::testing::UnorderedElementsAre()); +} + +TEST_F(WildcardSourceDependencyTest, Absolute_IgnoredFolder) +{ + AZStd::vector resolvedPaths; + QDir tempPath(m_tempDir.path()); + + ASSERT_TRUE(Test(tempPath.absoluteFilePath("*g.foo").toUtf8().constData(), resolvedPaths)); + ASSERT_THAT(resolvedPaths, ::testing::UnorderedElementsAre()); +} + +TEST_F(WildcardSourceDependencyTest, Relative_IgnoredFile) +{ + AZStd::vector resolvedPaths; + + ASSERT_TRUE(Test("*z.foo", resolvedPaths)); + ASSERT_THAT(resolvedPaths, ::testing::UnorderedElementsAre()); +} + +TEST_F(WildcardSourceDependencyTest, Absolute_IgnoredFile) +{ + AZStd::vector resolvedPaths; + QDir tempPath(m_tempDir.path()); + + ASSERT_TRUE(Test(tempPath.absoluteFilePath("*z.foo").toUtf8().constData(), resolvedPaths)); + ASSERT_THAT(resolvedPaths, ::testing::UnorderedElementsAre()); +} + +TEST_F(WildcardSourceDependencyTest, Relative_CacheFolder) +{ + AZStd::vector resolvedPaths; + QDir tempPath(m_tempDir.path()); + + ASSERT_TRUE(Test("*cache.foo", resolvedPaths)); + ASSERT_THAT(resolvedPaths, ::testing::UnorderedElementsAre()); +} + +TEST_F(WildcardSourceDependencyTest, FilesAddedAfterInitialCache) +{ + AZStd::vector resolvedPaths; + QDir tempPath(m_tempDir.path()); + + auto excludedFolderCacheInterface = AZ::Interface::Get(); + + ASSERT_TRUE(excludedFolderCacheInterface); + + { + const auto& excludedFolders = excludedFolderCacheInterface->GetExcludedFolders(); + + ASSERT_EQ(excludedFolders.size(), 2); + } + + // Add a file to a new ignored folder + QString newFilePath = tempPath.absoluteFilePath("subfolder2/redirected/folder/two/ignored/three/new.foo"); + UnitTestUtils::CreateDummyFile(newFilePath); + + excludedFolderCacheInterface->FileAdded(newFilePath); + + const auto& excludedFolders = excludedFolderCacheInterface->GetExcludedFolders(); + + ASSERT_EQ(excludedFolders.size(), 3); + ASSERT_THAT(excludedFolders, ::testing::Contains(AZStd::string(tempPath.absoluteFilePath("subfolder2/redirected/folder/two/ignored").toUtf8().constData()))); +} + +TEST_F(WildcardSourceDependencyTest, FilesRemovedAfterInitialCache) +{ + AZStd::vector resolvedPaths; + QDir tempPath(m_tempDir.path()); + + // Add a file to a new ignored folder + QString newFilePath = tempPath.absoluteFilePath("subfolder2/redirected/folder/two/ignored/three/new.foo"); + UnitTestUtils::CreateDummyFile(newFilePath); + + auto excludedFolderCacheInterface = AZ::Interface::Get(); + + ASSERT_TRUE(excludedFolderCacheInterface); + + { + const auto& excludedFolders = excludedFolderCacheInterface->GetExcludedFolders(); + + ASSERT_EQ(excludedFolders.size(), 3); + } + + m_fileStateCache->SignalDeleteEvent(tempPath.absoluteFilePath("subfolder2/redirected/folder/two/ignored")); + + const auto& excludedFolders = excludedFolderCacheInterface->GetExcludedFolders(); + + ASSERT_EQ(excludedFolders.size(), 2); +} + TEST_F(WildcardSourceDependencyTest, NewFile_MatchesSavedRelativeDependency) { QDir tempPath(m_tempDir.path()); diff --git a/Code/Tools/AssetProcessor/native/utilities/ApplicationManagerBase.cpp b/Code/Tools/AssetProcessor/native/utilities/ApplicationManagerBase.cpp index 4e0bc7e434..966b0f6723 100644 --- a/Code/Tools/AssetProcessor/native/utilities/ApplicationManagerBase.cpp +++ b/Code/Tools/AssetProcessor/native/utilities/ApplicationManagerBase.cpp @@ -454,6 +454,8 @@ void ApplicationManagerBase::InitFileMonitor() QObject::connect(newFolderWatch, &FolderWatchCallbackEx::fileModified, [this](QString path) { m_fileStateCache->UpdateFile(path); }); QObject::connect(newFolderWatch, &FolderWatchCallbackEx::fileRemoved, [this](QString path) { m_fileStateCache->RemoveFile(path); }); + QObject::connect(newFolderWatch, &FolderWatchCallbackEx::fileAdded, [](QString path) { AZ::Interface::Get()->FileAdded(path); }); + QObject::connect(newFolderWatch, &FolderWatchCallbackEx::fileAdded, m_fileProcessor.get(), &AssetProcessor::FileProcessor::AssessAddedFile); QObject::connect(newFolderWatch, &FolderWatchCallbackEx::fileRemoved, diff --git a/Code/Tools/AssetProcessor/native/utilities/PlatformConfiguration.cpp b/Code/Tools/AssetProcessor/native/utilities/PlatformConfiguration.cpp index bc516d5da6..7543e9ec8b 100644 --- a/Code/Tools/AssetProcessor/native/utilities/PlatformConfiguration.cpp +++ b/Code/Tools/AssetProcessor/native/utilities/PlatformConfiguration.cpp @@ -1285,6 +1285,13 @@ namespace AssetProcessor return m_scanFolders[index]; } + const AssetProcessor::ScanFolderInfo& PlatformConfiguration::GetScanFolderAt(int index) const + { + Q_ASSERT(index >= 0); + Q_ASSERT(index < m_scanFolders.size()); + return m_scanFolders[index]; + } + void PlatformConfiguration::AddScanFolder(const AssetProcessor::ScanFolderInfo& source, bool isUnitTesting) { if (isUnitTesting) @@ -1436,7 +1443,10 @@ namespace AssetProcessor } QStringList PlatformConfiguration::FindWildcardMatches( - const QString& sourceFolder, QString relativeName, bool includeFolders, bool recursiveSearch) const + const QString& sourceFolder, + QString relativeName, + bool includeFolders, + bool recursiveSearch) const { if (relativeName.isEmpty()) { @@ -1469,6 +1479,67 @@ namespace AssetProcessor return returnList; } + QStringList PlatformConfiguration::FindWildcardMatches( + const QString& sourceFolder, + QString relativeName, + const AZStd::unordered_set& excludedFolders, + bool includeFolders, + bool recursiveSearch) const + { + if (relativeName.isEmpty()) + { + return QStringList(); + } + + QDir sourceFolderDir(sourceFolder); + + QString posixRelativeName = QDir::fromNativeSeparators(relativeName); + + QStringList returnList; + QRegExp nameMatch{ posixRelativeName, Qt::CaseInsensitive, QRegExp::Wildcard }; + AZStd::stack dirs; + dirs.push(sourceFolderDir.absolutePath()); + + while (!dirs.empty()) + { + QString absolutePath = dirs.top(); + dirs.pop(); + + if (excludedFolders.contains(absolutePath.toUtf8().constData())) + { + continue; + } + + QDirIterator dirIterator(absolutePath, QDir::AllEntries | QDir::NoSymLinks | QDir::NoDotAndDotDot); + + while (dirIterator.hasNext()) + { + dirIterator.next(); + + if (!dirIterator.fileInfo().isFile()) + { + if (recursiveSearch) + { + dirs.push(dirIterator.filePath()); + } + + if (!includeFolders) + { + continue; + } + } + + QString pathMatch{ sourceFolderDir.relativeFilePath(dirIterator.filePath()) }; + if (nameMatch.exactMatch(pathMatch)) + { + returnList.append(QDir::fromNativeSeparators(dirIterator.filePath())); + } + } + } + + return returnList; + } + const AssetProcessor::ScanFolderInfo* PlatformConfiguration::GetScanFolderForFile(const QString& fullFileName) const { QString normalized = AssetUtilities::NormalizeFilePath(fullFileName); diff --git a/Code/Tools/AssetProcessor/native/utilities/PlatformConfiguration.h b/Code/Tools/AssetProcessor/native/utilities/PlatformConfiguration.h index 2c04ca1fad..a57416da10 100644 --- a/Code/Tools/AssetProcessor/native/utilities/PlatformConfiguration.h +++ b/Code/Tools/AssetProcessor/native/utilities/PlatformConfiguration.h @@ -256,6 +256,9 @@ namespace AssetProcessor //! Retrieve the scan folder at a given index. AssetProcessor::ScanFolderInfo& GetScanFolderAt(int index); + //! Retrieve the scan folder at a given index. + const AssetProcessor::ScanFolderInfo& GetScanFolderAt(int index) const; + //! Manually add a scan folder. Also used for testing. void AddScanFolder(const AssetProcessor::ScanFolderInfo& source, bool isUnitTesting = false); @@ -298,7 +301,16 @@ namespace AssetProcessor QString FindFirstMatchingFile(QString relativeName) const; //! given a relative name with wildcard characters (* allowed) find a set of matching files or optionally folders - QStringList FindWildcardMatches(const QString& sourceFolder, QString relativeName, bool includeFolders = false, bool recursiveSearch = true) const; + QStringList FindWildcardMatches(const QString& sourceFolder, QString relativeName, bool includeFolders = false, + bool recursiveSearch = true) const; + + //! given a relative name with wildcard characters (* allowed) find a set of matching files or optionally folders + QStringList FindWildcardMatches( + const QString& sourceFolder, + QString relativeName, + const AZStd::unordered_set& excludedFolders, + bool includeFolders = false, + bool recursiveSearch = true) const; //! given a fileName (as a full path), return the database source name which includes the output prefix. //!