[LYN-7520] Wildcard Source Dependencies include files in cache/excluded files (#5349)

* Add folder exclusion for wildcard source dependencies Signed-off-by: amzn-mike <80125227+amzn-mike@users.noreply.github.com> * Exclude ignored files. Add unit tests Signed-off-by: amzn-mike <80125227+amzn-mike@users.noreply.github.com> * Add handling for ignored folders being added/removed Add unit tests Signed-off-by: amzn-mike <80125227+amzn-mike@users.noreply.github.com> * Add ExcludedFolderCacheInterface to cmake Signed-off-by: amzn-mike <80125227+amzn-mike@users.noreply.github.com> * Fix include Signed-off-by: amzn-mike <80125227+amzn-mike@users.noreply.github.com> * Add error message Signed-off-by: amzn-mike <80125227+amzn-mike@users.noreply.github.com> * Cleanup includes Signed-off-by: amzn-mike <80125227+amzn-mike@users.noreply.github.com> * Revert traits include Signed-off-by: amzn-mike <80125227+amzn-mike@users.noreply.github.com> * Fix missing include, minor cleanup Signed-off-by: amzn-mike <80125227+amzn-mike@users.noreply.github.com> * Add missing includes Signed-off-by: amzn-mike <80125227+amzn-mike@users.noreply.github.com>
2021-11-18 13:37:48 -06:00
parent 8e02a82866
commit ae50187fba
13 changed files with 506 additions and 14 deletions
@@ -29,6 +29,9 @@ set(FILES
    native/AssetManager/SourceFileRelocator.h
    native/AssetManager/ControlRequestHandler.cpp
    native/AssetManager/ControlRequestHandler.h
+    native/AssetManager/ExcludedFolderCache.cpp
+    native/AssetManager/ExcludedFolderCache.h
+    native/AssetManager/ExcludedFolderCacheInterface.h
    native/assetprocessor.h
    native/connection/connection.cpp
    native/connection/connection.h
@@ -0,0 +1,154 @@
+/*
+ * Copyright (c) Contributors to the Open 3D Engine Project.
+ * For complete copyright and license terms please see the LICENSE at the root of this distribution.
+ *
+ * SPDX-License-Identifier: Apache-2.0 OR MIT
+ *
+ */
+
+#include <QDirIterator>
+#include <AzCore/Settings/SettingsRegistryMergeUtils.h>
+#include <native/AssetManager/ExcludedFolderCache.h>
+#include <utilities/assetUtils.h>
+#include <utilities/PlatformConfiguration.h>
+#include <AzCore/IO/Path/Path.h>
+
+namespace AssetProcessor
+{
+    ExcludedFolderCache::ExcludedFolderCache(const PlatformConfiguration* platformConfig) : m_platformConfig(platformConfig)
+    {
+        AZ::Interface<ExcludedFolderCacheInterface>::Register(this);
+    }
+
+    ExcludedFolderCache::~ExcludedFolderCache()
+    {
+        AZ::Interface<ExcludedFolderCacheInterface>::Unregister(this);
+    }
+
+    const AZStd::unordered_set<AZStd::string>& ExcludedFolderCache::GetExcludedFolders()
+    {
+        if (!m_builtCache)
+        {
+            for (int i = 0; i < m_platformConfig->GetScanFolderCount(); ++i)
+            {
+                const auto& scanFolderInfo = m_platformConfig->GetScanFolderAt(i);
+                QDir rooted(scanFolderInfo.ScanPath());
+                QString absolutePath = rooted.absolutePath();
+                AZStd::stack<QString> dirs;
+                dirs.push(absolutePath);
+                
+                while (!dirs.empty())
+                {
+                    absolutePath = dirs.top();
+                    dirs.pop();
+
+                    // Scan only folders, do not recurse so we have the chance to ignore a subfolder before going deeper
+                    QDirIterator dirIterator(absolutePath, QDir::Dirs | QDir::NoSymLinks | QDir::NoDotAndDotDot);
+
+                    // Loop all the folders in this directory
+                    while (dirIterator.hasNext())
+                    {
+                        dirIterator.next();
+                        QString pathMatch = rooted.absoluteFilePath(dirIterator.filePath());
+
+                        if (m_platformConfig->IsFileExcluded(pathMatch))
+                        {
+                            // Add the folder to the list and do not proceed any deeper
+                            m_excludedFolders.emplace(pathMatch.toUtf8().constData());
+                        }
+                        else if (scanFolderInfo.RecurseSubFolders())
+                        {
+                            // Folder is not excluded and recurse is enabled, add to the list of folders to check
+                            dirs.push(pathMatch);
+                        }
+                    }
+                }
+            }
+
+            // Add the cache to the list as well
+            AZStd::string projectCacheRootValue;
+            AZ::SettingsRegistry::Get()->Get(projectCacheRootValue, AZ::SettingsRegistryMergeUtils::FilePathKey_CacheProjectRootFolder);
+            projectCacheRootValue = AssetUtilities::NormalizeFilePath(projectCacheRootValue.c_str()).toUtf8().constData();
+            m_excludedFolders.emplace(projectCacheRootValue);
+
+            // Register to be notified about deletes so we can remove old ignored folders
+            auto fileStateCache = AZ::Interface<IFileStateRequests>::Get();
+
+            if (fileStateCache)
+            {
+                m_handler = AZ::Event<FileStateInfo>::Handler([this](FileStateInfo fileInfo)
+                {
+                    if (fileInfo.m_isDirectory)
+                    {
+                        AZStd::scoped_lock lock(m_pendingNewFolderMutex);
+
+                        m_pendingDeletes.emplace(fileInfo.m_absolutePath.toUtf8().constData());
+                    }
+                });
+
+                fileStateCache->RegisterForDeleteEvent(m_handler);
+            }
+            else
+            {
+                AZ_Error("ExcludedFolderCache", false, "Failed to find IFileStateRequests interface");
+            }
+
+            m_builtCache = true;
+        }
+
+        // Incorporate any pending folders
+        AZStd::unordered_set<AZStd::string> pendingAdds;
+        AZStd::unordered_set<AZStd::string> pendingDeletes;
+        
+        {
+            AZStd::scoped_lock lock(m_pendingNewFolderMutex);
+            pendingAdds.swap(m_pendingNewFolders);
+            pendingDeletes.swap(m_pendingDeletes);
+        }
+
+        if (!pendingAdds.empty())
+        {
+            m_excludedFolders.insert(pendingAdds.begin(), pendingAdds.end());
+        }
+
+        if (!pendingDeletes.empty())
+        {
+            for (const auto& pendingDelete : pendingDeletes)
+            {
+                m_excludedFolders.erase(pendingDelete);
+            }
+        }
+
+        return m_excludedFolders;
+    }
+
+    void ExcludedFolderCache::FileAdded(QString path)
+    {
+        QString relativePath, scanFolderPath;
+        
+        if (!m_platformConfig->ConvertToRelativePath(path, relativePath, scanFolderPath))
+        {
+            AZ_Error("ExcludedFolderCache", false, "Failed to get relative path for newly added file %s", path.toUtf8().constData());
+            return;
+        }
+
+        AZ::IO::Path azPath(relativePath.toUtf8().constData());
+        AZ::IO::Path absolutePath(scanFolderPath.toUtf8().constData());
+        
+        for (const auto& pathPart : azPath)
+        {
+            absolutePath /= pathPart;
+
+            QString normalized = AssetUtilities::NormalizeFilePath(absolutePath.c_str());
+
+            if (m_platformConfig->IsFileExcluded(normalized))
+            {
+                // Add the folder to a pending list, since this callback runs on another thread
+                AZStd::scoped_lock lock(m_pendingNewFolderMutex);
+
+                m_pendingNewFolders.emplace(normalized.toUtf8().constData());
+                break;
+            }
+        }
+    }
+}
@@ -0,0 +1,39 @@
+/*
+ * Copyright (c) Contributors to the Open 3D Engine Project.
+ * For complete copyright and license terms please see the LICENSE at the root of this distribution.
+ *
+ * SPDX-License-Identifier: Apache-2.0 OR MIT
+ *
+ */
+
+#pragma once
+
+#include <AssetManager/ExcludedFolderCacheInterface.h>
+#include <AssetManager/FileStateCache.h>
+
+namespace AssetProcessor
+{
+    class PlatformConfiguration;
+
+    struct ExcludedFolderCache : ExcludedFolderCacheInterface
+    {
+        explicit ExcludedFolderCache(const PlatformConfiguration* platformConfig);
+        ~ExcludedFolderCache() override;
+
+        // Gets a set of absolute paths to folder which have been excluded according to the platform configuration rules
+        // Note - not thread safe
+        const AZStd::unordered_set<AZStd::string>& GetExcludedFolders() override;
+
+        void FileAdded(QString path) override;
+
+    private:
+        bool m_builtCache = false;
+        const PlatformConfiguration* m_platformConfig{};
+        AZStd::unordered_set<AZStd::string> m_excludedFolders;
+
+        AZStd::recursive_mutex m_pendingNewFolderMutex;
+        AZStd::unordered_set<AZStd::string> m_pendingNewFolders; // Newly ignored folders waiting to be added to m_excludedFolders
+        AZStd::unordered_set<AZStd::string> m_pendingDeletes;
+        AZ::Event<FileStateInfo>::Handler m_handler;
+    };
+}
@@ -0,0 +1,29 @@
+/*
+ * Copyright (c) Contributors to the Open 3D Engine Project.
+ * For complete copyright and license terms please see the LICENSE at the root of this distribution.
+ *
+ * SPDX-License-Identifier: Apache-2.0 OR MIT
+ *
+ */
+
+#pragma once
+
+#include <AzCore/std/containers/unordered_set.h>
+#include <QString>
+
+namespace AssetProcessor
+{
+    class PlatformConfiguration;
+
+    struct ExcludedFolderCacheInterface
+    {
+        AZ_RTTI(ExcludedFolderCacheInterface, "{3AC471B6-C9F8-49CF-9E9D-237BDF63328C}");
+        AZ_DISABLE_COPY_MOVE(ExcludedFolderCacheInterface);
+
+        ExcludedFolderCacheInterface() = default;
+        virtual ~ExcludedFolderCacheInterface() = default;
+
+        virtual const AZStd::unordered_set<AZStd::string>& GetExcludedFolders() = 0;
+        virtual void FileAdded(QString path) = 0;
+    };
+}
@@ -63,6 +63,11 @@ namespace AssetProcessor
        return true;
    }

+    void FileStateCache::RegisterForDeleteEvent(AZ::Event<FileStateInfo>::Handler& handler)
+    {
+        handler.Connect(m_deleteEvent);
+    }
+
    void FileStateCache::AddInfoSet(QSet<AssetFileInfo> infoSet)
    {
        LockGuardType scopeLock(m_mapMutex);
@@ -103,6 +108,8 @@ namespace AssetProcessor

        if (itr != m_fileInfoMap.end())
        {
+            m_deleteEvent.Signal(itr.value());
+
            bool isDirectory = itr.value().m_isDirectory;
            QString parentPath = itr.value().m_absolutePath;
            m_fileInfoMap.erase(itr);
@@ -205,6 +212,21 @@ namespace AssetProcessor
        return true;
    }

+    void FileStatePassthrough::RegisterForDeleteEvent(AZ::Event<FileStateInfo>::Handler& handler)
+    {
+        handler.Connect(m_deleteEvent);
+    }
+
+    void FileStatePassthrough::SignalDeleteEvent(const QString& absolutePath) const
+    {
+        FileStateInfo info;
+
+        if (GetFileInfo(absolutePath, &info))
+        {
+            m_deleteEvent.Signal(info);
+        }
+    }
+
    bool FileStateInfo::operator==(const FileStateInfo& rhs) const
    {
        return m_absolutePath == rhs.m_absolutePath
@@ -14,6 +14,7 @@
 #include <QSet>
 #include <QFileInfo>
 #include <AzCore/Interface/Interface.h>
+#include <AzCore/EBus/Event.h>

 namespace AssetProcessor
 {
@@ -51,10 +52,11 @@ namespace AssetProcessor
        /// Convenience function to check if a file or directory exists.
        virtual bool Exists(const QString& absolutePath) const = 0;
        virtual bool GetHash(const QString& absolutePath, FileHash* foundHash) = 0;
+        virtual void RegisterForDeleteEvent(AZ::Event<FileStateInfo>::Handler& handler) = 0;

        AZ_DISABLE_COPY_MOVE(IFileStateRequests);
    };
-    
+
    class FileStateBase
        : public IFileStateRequests
    {
@@ -89,11 +91,11 @@ namespace AssetProcessor
    {
    public:

-
        // FileStateRequestBus implementation
        bool GetFileInfo(const QString& absolutePath, FileStateInfo* foundFileInfo) const override;
        bool Exists(const QString& absolutePath) const override;
        bool GetHash(const QString& absolutePath, FileHash* foundHash) override;
+        void RegisterForDeleteEvent(AZ::Event<FileStateInfo>::Handler& handler) override;

        void AddInfoSet(QSet<AssetFileInfo> infoSet) override;
        void AddFile(const QString& absolutePath) override;
@@ -116,9 +118,11 @@ namespace AssetProcessor

        mutable AZStd::recursive_mutex m_mapMutex;
        QHash<QString, FileStateInfo> m_fileInfoMap;
-        
+
        QHash<QString, FileHash> m_fileHashMap;

+        AZ::Event<FileStateInfo> m_deleteEvent;
+
        using LockGuardType = AZStd::lock_guard<decltype(m_mapMutex)>;
    };

@@ -131,5 +135,10 @@ namespace AssetProcessor
        bool GetFileInfo(const QString& absolutePath, FileStateInfo* foundFileInfo) const override;
        bool Exists(const QString& absolutePath) const override;
        bool GetHash(const QString& absolutePath, FileHash* foundHash) override;
+        void RegisterForDeleteEvent(AZ::Event<FileStateInfo>::Handler& handler) override;
+
+        void SignalDeleteEvent(const QString& absolutePath) const;
+    protected:
+        AZ::Event<FileStateInfo> m_deleteEvent;
    };
 } // namespace AssetProcessor
@@ -18,8 +18,8 @@

 #include <AzToolsFramework/Debug/TraceContext.h>

-
 #include "native/AssetManager/assetProcessorManager.h"
+
 #include <AzCore/std/sort.h>
 #include <AzToolsFramework/API/AssetDatabaseBus.h>

@@ -66,8 +66,10 @@ namespace AssetProcessor

        m_sourceFileRelocator = AZStd::make_unique<SourceFileRelocator>(m_stateData, m_platformConfig);

-        PopulateJobStateCache();
+        m_excludedFolderCache = AZStd::make_unique<ExcludedFolderCache>(m_platformConfig);

+        PopulateJobStateCache();
+        
        AssetProcessor::ProcessingJobInfoBus::Handler::BusConnect();
    }

@@ -3573,6 +3575,8 @@ namespace AssetProcessor
                    QString knownPathBeforeWildcard = encodedFileData.left(slashBeforeWildcardIndex + 1); // include the slash
                    QString relativeSearch = encodedFileData.mid(slashBeforeWildcardIndex + 1); // skip the slash

+                    const auto& excludedFolders = m_excludedFolderCache->GetExcludedFolders();
+
                    // Absolute path, just check the 1 scan folder
                    if (AZ::IO::PathView(encodedFileData.toUtf8().constData()).IsAbsolute())
                    {
@@ -3592,7 +3596,8 @@ namespace AssetProcessor
                            QString scanFolderAndKnownSubPath = rooted.absoluteFilePath(knownPathBeforeWildcard);

                            resolvedDependencyList.append(m_platformConfig->FindWildcardMatches(
-                                scanFolderAndKnownSubPath, relativeSearch, false, scanFolderInfo->RecurseSubFolders()));
+                                scanFolderAndKnownSubPath, relativeSearch,
+                                excludedFolders, false, scanFolderInfo->RecurseSubFolders()));
                        }
                    }
                    else // Relative path, check every scan folder
@@ -3610,7 +3615,21 @@ namespace AssetProcessor
                            QString absolutePath = rooted.absoluteFilePath(knownPathBeforeWildcard);

                            resolvedDependencyList.append(m_platformConfig->FindWildcardMatches(
-                                absolutePath, relativeSearch, false, scanFolderInfo->RecurseSubFolders()));
+                                absolutePath, relativeSearch,
+                                excludedFolders, false, scanFolderInfo->RecurseSubFolders()));
+                        }
+                    }
+
+                    // Filter out any excluded files
+                    for (auto itr = resolvedDependencyList.begin(); itr != resolvedDependencyList.end();)
+                    {
+                        if (m_platformConfig->IsFileExcluded(*itr))
+                        {
+                            itr = resolvedDependencyList.erase(itr);
+                        }
+                        else
+                        {
+                            ++itr;
                        }
                    }

@@ -40,6 +40,8 @@
 #include "AssetRequestHandler.h"
 #include "native/utilities/JobDiagnosticTracker.h"
 #include "SourceFileRelocator.h"
+
+#include <AssetManager/ExcludedFolderCache.h>
 #endif

 class FileWatcher;
@@ -341,7 +343,8 @@ namespace AssetProcessor
        void CleanEmptyFolder(QString folder, QString root);

        void ProcessBuilders(QString normalizedPath, QString relativePathToFile, const ScanFolderInfo* scanFolder, const AssetProcessor::BuilderInfoList& builderInfoList);
-        
+        AZStd::vector<AZStd::string> GetExcludedFolders();
+
        struct SourceInfo
        {
            QString m_watchFolder;
@@ -552,6 +555,8 @@ namespace AssetProcessor
        // when true, a flag will be sent to builders process job indicating debug output/mode should be used
        bool m_builderDebugFlag = false;

+        AZStd::unique_ptr<ExcludedFolderCache> m_excludedFolderCache{};
+
 protected Q_SLOTS:
        void FinishAnalysis(AZStd::string fileToCheck);
        //////////////////////////////////////////////////////////
@@ -26,7 +26,7 @@ namespace AssetProcessor
    {
    protected:
        AZStd::unique_ptr<UnitTestUtils::AssertAbsorber> m_errorAbsorber{};
-        FileStatePassthrough m_fileStateCache;
+        AZStd::unique_ptr<FileStatePassthrough> m_fileStateCache{};

        void SetUp() override
        {
@@ -40,9 +40,10 @@ namespace AssetProcessor
                m_ownsSysAllocator = true;
                AZ::AllocatorInstance<AZ::SystemAllocator>::Create();
            }
-            m_errorAbsorber = AZStd::make_unique<UnitTestUtils::AssertAbsorber>();

+            m_errorAbsorber = AZStd::make_unique<UnitTestUtils::AssertAbsorber>();
            m_application = AZStd::make_unique<AzFramework::Application>();
+            m_fileStateCache = AZStd::make_unique<FileStatePassthrough>();

            // Inject the AutomatedTesting project as a project path into test fixture
            using FixedValueString = AZ::SettingsRegistryInterface::FixedValueString;
@@ -60,7 +61,8 @@ namespace AssetProcessor
        void TearDown() override
        {
            AssetUtilities::ResetAssetRoot();
-            
+
+            m_fileStateCache.reset();
            m_application.reset();
            m_errorAbsorber.reset();

@@ -5364,13 +5364,29 @@ AZStd::vector<AZStd::string> WildcardSourceDependencyTest::FileAddedTest(const Q
 void WildcardSourceDependencyTest::SetUp()
 {
    AssetProcessorManagerTest::SetUp();
-
+    
    QDir tempPath(m_tempDir.path());

    // Add a non-recursive scan folder.  Only files directly inside of this folder should be picked up, subfolders are ignored
    m_config->AddScanFolder(ScanFolderInfo(tempPath.filePath("no_recurse"), "no_recurse",
        "no_recurse", false, false, m_config->GetEnabledPlatforms(), 1));

+    {
+        ExcludeAssetRecognizer excludeFolder;
+        excludeFolder.m_name = "Exclude ignored Folder";
+        excludeFolder.m_patternMatcher =
+            AssetBuilderSDK::FilePatternMatcher(R"REGEX(^(.*\/)?ignored(\/.*)?$)REGEX", AssetBuilderSDK::AssetBuilderPattern::Regex);
+        m_config->AddExcludeRecognizer(excludeFolder);
+    }
+
+    {
+        ExcludeAssetRecognizer excludeFile;
+        excludeFile.m_name = "Exclude z.foo Files";
+        excludeFile.m_patternMatcher =
+            AssetBuilderSDK::FilePatternMatcher(R"REGEX(^(.*\/)?z\.foo$)REGEX", AssetBuilderSDK::AssetBuilderPattern::Regex);
+        m_config->AddExcludeRecognizer(excludeFile);
+    }
+
    UnitTestUtils::CreateDummyFile(tempPath.absoluteFilePath("subfolder1/1a.foo"));
    UnitTestUtils::CreateDummyFile(tempPath.absoluteFilePath("subfolder1/1b.foo"));
    UnitTestUtils::CreateDummyFile(tempPath.absoluteFilePath("subfolder2/redirected/a.foo"));
@@ -5384,6 +5400,19 @@ void WildcardSourceDependencyTest::SetUp()
    // Add a file in the non-recursive scanfolder.  Since its not directly in the scan folder, it should always be ignored
    UnitTestUtils::CreateDummyFile(tempPath.absoluteFilePath("no_recurse/one/two/three/f.foo"));

+    // Add a file to an ignored folder
+    UnitTestUtils::CreateDummyFile(tempPath.absoluteFilePath("subfolder2/redirected/folder/ignored/g.foo"));
+
+    // Add an ignored file
+    UnitTestUtils::CreateDummyFile(tempPath.absoluteFilePath("subfolder2/redirected/folder/one/z.foo"));
+
+    // Add a file in the cache
+    AZStd::string projectCacheRootValue;
+    AZ::SettingsRegistry::Get()->Get(projectCacheRootValue, AZ::SettingsRegistryMergeUtils::FilePathKey_CacheProjectRootFolder);
+    projectCacheRootValue = AssetUtilities::NormalizeFilePath(projectCacheRootValue.c_str()).toUtf8().constData();
+    auto path = AZ::IO::Path(projectCacheRootValue) / "cache.foo";
+    UnitTestUtils::CreateDummyFile(path.c_str());
+
    AzToolsFramework::AssetDatabase::SourceFileDependencyEntryContainer dependencies;

    // Relative path wildcard dependency
@@ -5518,6 +5547,102 @@ TEST_F(WildcardSourceDependencyTest, Absolute_NoWildcard)
    ASSERT_THAT(resolvedPaths, ::testing::UnorderedElementsAre());
 }

+TEST_F(WildcardSourceDependencyTest, Relative_IgnoredFolder)
+{
+    AZStd::vector<AZStd::string> resolvedPaths;
+
+    ASSERT_TRUE(Test("*g.foo", resolvedPaths));
+    ASSERT_THAT(resolvedPaths, ::testing::UnorderedElementsAre());
+}
+
+TEST_F(WildcardSourceDependencyTest, Absolute_IgnoredFolder)
+{
+    AZStd::vector<AZStd::string> resolvedPaths;
+    QDir tempPath(m_tempDir.path());
+
+    ASSERT_TRUE(Test(tempPath.absoluteFilePath("*g.foo").toUtf8().constData(), resolvedPaths));
+    ASSERT_THAT(resolvedPaths, ::testing::UnorderedElementsAre());
+}
+
+TEST_F(WildcardSourceDependencyTest, Relative_IgnoredFile)
+{
+    AZStd::vector<AZStd::string> resolvedPaths;
+
+    ASSERT_TRUE(Test("*z.foo", resolvedPaths));
+    ASSERT_THAT(resolvedPaths, ::testing::UnorderedElementsAre());
+}
+
+TEST_F(WildcardSourceDependencyTest, Absolute_IgnoredFile)
+{
+    AZStd::vector<AZStd::string> resolvedPaths;
+    QDir tempPath(m_tempDir.path());
+
+    ASSERT_TRUE(Test(tempPath.absoluteFilePath("*z.foo").toUtf8().constData(), resolvedPaths));
+    ASSERT_THAT(resolvedPaths, ::testing::UnorderedElementsAre());
+}
+
+TEST_F(WildcardSourceDependencyTest, Relative_CacheFolder)
+{
+    AZStd::vector<AZStd::string> resolvedPaths;
+    QDir tempPath(m_tempDir.path());
+    
+    ASSERT_TRUE(Test("*cache.foo", resolvedPaths));
+    ASSERT_THAT(resolvedPaths, ::testing::UnorderedElementsAre());
+}
+
+TEST_F(WildcardSourceDependencyTest, FilesAddedAfterInitialCache)
+{
+    AZStd::vector<AZStd::string> resolvedPaths;
+    QDir tempPath(m_tempDir.path());
+
+    auto excludedFolderCacheInterface = AZ::Interface<ExcludedFolderCacheInterface>::Get();
+
+    ASSERT_TRUE(excludedFolderCacheInterface);
+
+    {
+        const auto& excludedFolders = excludedFolderCacheInterface->GetExcludedFolders();
+
+        ASSERT_EQ(excludedFolders.size(), 2);
+    }
+
+    // Add a file to a new ignored folder
+    QString newFilePath = tempPath.absoluteFilePath("subfolder2/redirected/folder/two/ignored/three/new.foo");
+    UnitTestUtils::CreateDummyFile(newFilePath);
+
+    excludedFolderCacheInterface->FileAdded(newFilePath);
+
+    const auto& excludedFolders = excludedFolderCacheInterface->GetExcludedFolders();
+
+    ASSERT_EQ(excludedFolders.size(), 3);
+    ASSERT_THAT(excludedFolders, ::testing::Contains(AZStd::string(tempPath.absoluteFilePath("subfolder2/redirected/folder/two/ignored").toUtf8().constData())));
+}
+
+TEST_F(WildcardSourceDependencyTest, FilesRemovedAfterInitialCache)
+{
+    AZStd::vector<AZStd::string> resolvedPaths;
+    QDir tempPath(m_tempDir.path());
+
+    // Add a file to a new ignored folder
+    QString newFilePath = tempPath.absoluteFilePath("subfolder2/redirected/folder/two/ignored/three/new.foo");
+    UnitTestUtils::CreateDummyFile(newFilePath);
+
+    auto excludedFolderCacheInterface = AZ::Interface<ExcludedFolderCacheInterface>::Get();
+
+    ASSERT_TRUE(excludedFolderCacheInterface);
+
+    {
+        const auto& excludedFolders = excludedFolderCacheInterface->GetExcludedFolders();
+
+        ASSERT_EQ(excludedFolders.size(), 3);
+    }
+    
+    m_fileStateCache->SignalDeleteEvent(tempPath.absoluteFilePath("subfolder2/redirected/folder/two/ignored"));
+
+    const auto& excludedFolders = excludedFolderCacheInterface->GetExcludedFolders();
+
+    ASSERT_EQ(excludedFolders.size(), 2);
+}
+
 TEST_F(WildcardSourceDependencyTest, NewFile_MatchesSavedRelativeDependency)
 {
    QDir tempPath(m_tempDir.path());
@@ -454,6 +454,8 @@ void ApplicationManagerBase::InitFileMonitor()
        QObject::connect(newFolderWatch, &FolderWatchCallbackEx::fileModified, [this](QString path) { m_fileStateCache->UpdateFile(path); });
        QObject::connect(newFolderWatch, &FolderWatchCallbackEx::fileRemoved, [this](QString path) { m_fileStateCache->RemoveFile(path); });

+        QObject::connect(newFolderWatch, &FolderWatchCallbackEx::fileAdded, [](QString path) { AZ::Interface<AssetProcessor::ExcludedFolderCacheInterface>::Get()->FileAdded(path); });
+
        QObject::connect(newFolderWatch, &FolderWatchCallbackEx::fileAdded,
            m_fileProcessor.get(), &AssetProcessor::FileProcessor::AssessAddedFile);
        QObject::connect(newFolderWatch, &FolderWatchCallbackEx::fileRemoved,
@@ -1285,6 +1285,13 @@ namespace AssetProcessor
        return m_scanFolders[index];
    }

+    const AssetProcessor::ScanFolderInfo& PlatformConfiguration::GetScanFolderAt(int index) const
+    {
+        Q_ASSERT(index >= 0);
+        Q_ASSERT(index < m_scanFolders.size());
+        return m_scanFolders[index];
+    }
+
    void PlatformConfiguration::AddScanFolder(const AssetProcessor::ScanFolderInfo& source, bool isUnitTesting)
    {
        if (isUnitTesting)
@@ -1436,7 +1443,10 @@ namespace AssetProcessor
    }

    QStringList PlatformConfiguration::FindWildcardMatches(
-        const QString& sourceFolder, QString relativeName, bool includeFolders, bool recursiveSearch) const
+        const QString& sourceFolder,
+        QString relativeName,
+        bool includeFolders,
+        bool recursiveSearch) const
    {
        if (relativeName.isEmpty())
        {
@@ -1469,6 +1479,67 @@ namespace AssetProcessor
        return returnList;
    }

+    QStringList PlatformConfiguration::FindWildcardMatches(
+        const QString& sourceFolder,
+        QString relativeName,
+        const AZStd::unordered_set<AZStd::string>& excludedFolders,
+        bool includeFolders,
+        bool recursiveSearch) const
+    {
+        if (relativeName.isEmpty())
+        {
+            return QStringList();
+        }
+
+        QDir sourceFolderDir(sourceFolder);
+
+        QString posixRelativeName = QDir::fromNativeSeparators(relativeName);
+
+        QStringList returnList;
+        QRegExp nameMatch{ posixRelativeName, Qt::CaseInsensitive, QRegExp::Wildcard };
+        AZStd::stack<QString> dirs;
+        dirs.push(sourceFolderDir.absolutePath());
+        
+        while (!dirs.empty())
+        {
+            QString absolutePath = dirs.top();
+            dirs.pop();
+
+            if (excludedFolders.contains(absolutePath.toUtf8().constData()))
+            {
+                continue;
+            }
+
+            QDirIterator dirIterator(absolutePath, QDir::AllEntries | QDir::NoSymLinks | QDir::NoDotAndDotDot);
+
+            while (dirIterator.hasNext())
+            {
+                dirIterator.next();
+
+                if (!dirIterator.fileInfo().isFile())
+                {
+                    if (recursiveSearch)
+                    {
+                        dirs.push(dirIterator.filePath());
+                    }
+
+                    if (!includeFolders)
+                    {
+                        continue;
+                    }
+                }
+                
+                QString pathMatch{ sourceFolderDir.relativeFilePath(dirIterator.filePath()) };
+                if (nameMatch.exactMatch(pathMatch))
+                {
+                    returnList.append(QDir::fromNativeSeparators(dirIterator.filePath()));
+                }
+            }
+        }
+
+        return returnList;
+    }
+
    const AssetProcessor::ScanFolderInfo* PlatformConfiguration::GetScanFolderForFile(const QString& fullFileName) const
    {
        QString normalized = AssetUtilities::NormalizeFilePath(fullFileName);
@@ -256,6 +256,9 @@ namespace AssetProcessor
        //! Retrieve the scan folder at a given index.
        AssetProcessor::ScanFolderInfo& GetScanFolderAt(int index);

+        //! Retrieve the scan folder at a given index.
+        const AssetProcessor::ScanFolderInfo& GetScanFolderAt(int index) const;
+
        //!  Manually add a scan folder.  Also used for testing.
        void AddScanFolder(const AssetProcessor::ScanFolderInfo& source, bool isUnitTesting = false);

@@ -298,7 +301,16 @@ namespace AssetProcessor
        QString FindFirstMatchingFile(QString relativeName) const;

        //! given a relative name with wildcard characters (* allowed) find a set of matching files or optionally folders
-        QStringList FindWildcardMatches(const QString& sourceFolder, QString relativeName, bool includeFolders = false, bool recursiveSearch = true) const;
+        QStringList FindWildcardMatches(const QString& sourceFolder, QString relativeName, bool includeFolders = false,
+            bool recursiveSearch = true) const;
+
+        //! given a relative name with wildcard characters (* allowed) find a set of matching files or optionally folders
+        QStringList FindWildcardMatches(
+            const QString& sourceFolder,
+            QString relativeName,
+            const AZStd::unordered_set<AZStd::string>& excludedFolders,
+            bool includeFolders = false,
+            bool recursiveSearch = true) const;

        //! given a fileName (as a full path), return the database source name which includes the output prefix.
        //!