[LYN-7520] Wildcard Source Dependencies include files in cache/excluded files (#5349)

* Add folder exclusion for wildcard source dependencies Signed-off-by: amzn-mike <80125227+amzn-mike@users.noreply.github.com> * Exclude ignored files. Add unit tests Signed-off-by: amzn-mike <80125227+amzn-mike@users.noreply.github.com> * Add handling for ignored folders being added/removed Add unit tests Signed-off-by: amzn-mike <80125227+amzn-mike@users.noreply.github.com> * Add ExcludedFolderCacheInterface to cmake Signed-off-by: amzn-mike <80125227+amzn-mike@users.noreply.github.com> * Fix include Signed-off-by: amzn-mike <80125227+amzn-mike@users.noreply.github.com> * Add error message Signed-off-by: amzn-mike <80125227+amzn-mike@users.noreply.github.com> * Cleanup includes Signed-off-by: amzn-mike <80125227+amzn-mike@users.noreply.github.com> * Revert traits include Signed-off-by: amzn-mike <80125227+amzn-mike@users.noreply.github.com> * Fix missing include, minor cleanup Signed-off-by: amzn-mike <80125227+amzn-mike@users.noreply.github.com> * Add missing includes Signed-off-by: amzn-mike <80125227+amzn-mike@users.noreply.github.com>
2021-11-18 13:37:48 -06:00
parent 8e02a82866
commit ae50187fba
13 changed files with 506 additions and 14 deletions
@@ -0,0 +1,154 @@
+/*
+ * Copyright (c) Contributors to the Open 3D Engine Project.
+ * For complete copyright and license terms please see the LICENSE at the root of this distribution.
+ *
+ * SPDX-License-Identifier: Apache-2.0 OR MIT
+ *
+ */
+
+#include <QDirIterator>
+#include <AzCore/Settings/SettingsRegistryMergeUtils.h>
+#include <native/AssetManager/ExcludedFolderCache.h>
+#include <utilities/assetUtils.h>
+#include <utilities/PlatformConfiguration.h>
+#include <AzCore/IO/Path/Path.h>
+
+namespace AssetProcessor
+{
+    ExcludedFolderCache::ExcludedFolderCache(const PlatformConfiguration* platformConfig) : m_platformConfig(platformConfig)
+    {
+        AZ::Interface<ExcludedFolderCacheInterface>::Register(this);
+    }
+
+    ExcludedFolderCache::~ExcludedFolderCache()
+    {
+        AZ::Interface<ExcludedFolderCacheInterface>::Unregister(this);
+    }
+
+    const AZStd::unordered_set<AZStd::string>& ExcludedFolderCache::GetExcludedFolders()
+    {
+        if (!m_builtCache)
+        {
+            for (int i = 0; i < m_platformConfig->GetScanFolderCount(); ++i)
+            {
+                const auto& scanFolderInfo = m_platformConfig->GetScanFolderAt(i);
+                QDir rooted(scanFolderInfo.ScanPath());
+                QString absolutePath = rooted.absolutePath();
+                AZStd::stack<QString> dirs;
+                dirs.push(absolutePath);
+                
+                while (!dirs.empty())
+                {
+                    absolutePath = dirs.top();
+                    dirs.pop();
+
+                    // Scan only folders, do not recurse so we have the chance to ignore a subfolder before going deeper
+                    QDirIterator dirIterator(absolutePath, QDir::Dirs | QDir::NoSymLinks | QDir::NoDotAndDotDot);
+
+                    // Loop all the folders in this directory
+                    while (dirIterator.hasNext())
+                    {
+                        dirIterator.next();
+                        QString pathMatch = rooted.absoluteFilePath(dirIterator.filePath());
+
+                        if (m_platformConfig->IsFileExcluded(pathMatch))
+                        {
+                            // Add the folder to the list and do not proceed any deeper
+                            m_excludedFolders.emplace(pathMatch.toUtf8().constData());
+                        }
+                        else if (scanFolderInfo.RecurseSubFolders())
+                        {
+                            // Folder is not excluded and recurse is enabled, add to the list of folders to check
+                            dirs.push(pathMatch);
+                        }
+                    }
+                }
+            }
+
+            // Add the cache to the list as well
+            AZStd::string projectCacheRootValue;
+            AZ::SettingsRegistry::Get()->Get(projectCacheRootValue, AZ::SettingsRegistryMergeUtils::FilePathKey_CacheProjectRootFolder);
+            projectCacheRootValue = AssetUtilities::NormalizeFilePath(projectCacheRootValue.c_str()).toUtf8().constData();
+            m_excludedFolders.emplace(projectCacheRootValue);
+
+            // Register to be notified about deletes so we can remove old ignored folders
+            auto fileStateCache = AZ::Interface<IFileStateRequests>::Get();
+
+            if (fileStateCache)
+            {
+                m_handler = AZ::Event<FileStateInfo>::Handler([this](FileStateInfo fileInfo)
+                {
+                    if (fileInfo.m_isDirectory)
+                    {
+                        AZStd::scoped_lock lock(m_pendingNewFolderMutex);
+
+                        m_pendingDeletes.emplace(fileInfo.m_absolutePath.toUtf8().constData());
+                    }
+                });
+
+                fileStateCache->RegisterForDeleteEvent(m_handler);
+            }
+            else
+            {
+                AZ_Error("ExcludedFolderCache", false, "Failed to find IFileStateRequests interface");
+            }
+
+            m_builtCache = true;
+        }
+
+        // Incorporate any pending folders
+        AZStd::unordered_set<AZStd::string> pendingAdds;
+        AZStd::unordered_set<AZStd::string> pendingDeletes;
+        
+        {
+            AZStd::scoped_lock lock(m_pendingNewFolderMutex);
+            pendingAdds.swap(m_pendingNewFolders);
+            pendingDeletes.swap(m_pendingDeletes);
+        }
+
+        if (!pendingAdds.empty())
+        {
+            m_excludedFolders.insert(pendingAdds.begin(), pendingAdds.end());
+        }
+
+        if (!pendingDeletes.empty())
+        {
+            for (const auto& pendingDelete : pendingDeletes)
+            {
+                m_excludedFolders.erase(pendingDelete);
+            }
+        }
+
+        return m_excludedFolders;
+    }
+
+    void ExcludedFolderCache::FileAdded(QString path)
+    {
+        QString relativePath, scanFolderPath;
+        
+        if (!m_platformConfig->ConvertToRelativePath(path, relativePath, scanFolderPath))
+        {
+            AZ_Error("ExcludedFolderCache", false, "Failed to get relative path for newly added file %s", path.toUtf8().constData());
+            return;
+        }
+
+        AZ::IO::Path azPath(relativePath.toUtf8().constData());
+        AZ::IO::Path absolutePath(scanFolderPath.toUtf8().constData());
+        
+        for (const auto& pathPart : azPath)
+        {
+            absolutePath /= pathPart;
+
+            QString normalized = AssetUtilities::NormalizeFilePath(absolutePath.c_str());
+
+            if (m_platformConfig->IsFileExcluded(normalized))
+            {
+                // Add the folder to a pending list, since this callback runs on another thread
+                AZStd::scoped_lock lock(m_pendingNewFolderMutex);
+
+                m_pendingNewFolders.emplace(normalized.toUtf8().constData());
+                break;
+            }
+        }
+    }
+}
@@ -0,0 +1,39 @@
+/*
+ * Copyright (c) Contributors to the Open 3D Engine Project.
+ * For complete copyright and license terms please see the LICENSE at the root of this distribution.
+ *
+ * SPDX-License-Identifier: Apache-2.0 OR MIT
+ *
+ */
+
+#pragma once
+
+#include <AssetManager/ExcludedFolderCacheInterface.h>
+#include <AssetManager/FileStateCache.h>
+
+namespace AssetProcessor
+{
+    class PlatformConfiguration;
+
+    struct ExcludedFolderCache : ExcludedFolderCacheInterface
+    {
+        explicit ExcludedFolderCache(const PlatformConfiguration* platformConfig);
+        ~ExcludedFolderCache() override;
+
+        // Gets a set of absolute paths to folder which have been excluded according to the platform configuration rules
+        // Note - not thread safe
+        const AZStd::unordered_set<AZStd::string>& GetExcludedFolders() override;
+
+        void FileAdded(QString path) override;
+
+    private:
+        bool m_builtCache = false;
+        const PlatformConfiguration* m_platformConfig{};
+        AZStd::unordered_set<AZStd::string> m_excludedFolders;
+
+        AZStd::recursive_mutex m_pendingNewFolderMutex;
+        AZStd::unordered_set<AZStd::string> m_pendingNewFolders; // Newly ignored folders waiting to be added to m_excludedFolders
+        AZStd::unordered_set<AZStd::string> m_pendingDeletes;
+        AZ::Event<FileStateInfo>::Handler m_handler;
+    };
+}
@@ -0,0 +1,29 @@
+/*
+ * Copyright (c) Contributors to the Open 3D Engine Project.
+ * For complete copyright and license terms please see the LICENSE at the root of this distribution.
+ *
+ * SPDX-License-Identifier: Apache-2.0 OR MIT
+ *
+ */
+
+#pragma once
+
+#include <AzCore/std/containers/unordered_set.h>
+#include <QString>
+
+namespace AssetProcessor
+{
+    class PlatformConfiguration;
+
+    struct ExcludedFolderCacheInterface
+    {
+        AZ_RTTI(ExcludedFolderCacheInterface, "{3AC471B6-C9F8-49CF-9E9D-237BDF63328C}");
+        AZ_DISABLE_COPY_MOVE(ExcludedFolderCacheInterface);
+
+        ExcludedFolderCacheInterface() = default;
+        virtual ~ExcludedFolderCacheInterface() = default;
+
+        virtual const AZStd::unordered_set<AZStd::string>& GetExcludedFolders() = 0;
+        virtual void FileAdded(QString path) = 0;
+    };
+}
@@ -63,6 +63,11 @@ namespace AssetProcessor
        return true;
    }

+    void FileStateCache::RegisterForDeleteEvent(AZ::Event<FileStateInfo>::Handler& handler)
+    {
+        handler.Connect(m_deleteEvent);
+    }
+
    void FileStateCache::AddInfoSet(QSet<AssetFileInfo> infoSet)
    {
        LockGuardType scopeLock(m_mapMutex);
@@ -103,6 +108,8 @@ namespace AssetProcessor

        if (itr != m_fileInfoMap.end())
        {
+            m_deleteEvent.Signal(itr.value());
+
            bool isDirectory = itr.value().m_isDirectory;
            QString parentPath = itr.value().m_absolutePath;
            m_fileInfoMap.erase(itr);
@@ -205,6 +212,21 @@ namespace AssetProcessor
        return true;
    }

+    void FileStatePassthrough::RegisterForDeleteEvent(AZ::Event<FileStateInfo>::Handler& handler)
+    {
+        handler.Connect(m_deleteEvent);
+    }
+
+    void FileStatePassthrough::SignalDeleteEvent(const QString& absolutePath) const
+    {
+        FileStateInfo info;
+
+        if (GetFileInfo(absolutePath, &info))
+        {
+            m_deleteEvent.Signal(info);
+        }
+    }
+
    bool FileStateInfo::operator==(const FileStateInfo& rhs) const
    {
        return m_absolutePath == rhs.m_absolutePath
@@ -14,6 +14,7 @@
 #include <QSet>
 #include <QFileInfo>
 #include <AzCore/Interface/Interface.h>
+#include <AzCore/EBus/Event.h>

 namespace AssetProcessor
 {
@@ -51,10 +52,11 @@ namespace AssetProcessor
        /// Convenience function to check if a file or directory exists.
        virtual bool Exists(const QString& absolutePath) const = 0;
        virtual bool GetHash(const QString& absolutePath, FileHash* foundHash) = 0;
+        virtual void RegisterForDeleteEvent(AZ::Event<FileStateInfo>::Handler& handler) = 0;

        AZ_DISABLE_COPY_MOVE(IFileStateRequests);
    };
-    
+
    class FileStateBase
        : public IFileStateRequests
    {
@@ -89,11 +91,11 @@ namespace AssetProcessor
    {
    public:

-
        // FileStateRequestBus implementation
        bool GetFileInfo(const QString& absolutePath, FileStateInfo* foundFileInfo) const override;
        bool Exists(const QString& absolutePath) const override;
        bool GetHash(const QString& absolutePath, FileHash* foundHash) override;
+        void RegisterForDeleteEvent(AZ::Event<FileStateInfo>::Handler& handler) override;

        void AddInfoSet(QSet<AssetFileInfo> infoSet) override;
        void AddFile(const QString& absolutePath) override;
@@ -116,9 +118,11 @@ namespace AssetProcessor

        mutable AZStd::recursive_mutex m_mapMutex;
        QHash<QString, FileStateInfo> m_fileInfoMap;
-        
+
        QHash<QString, FileHash> m_fileHashMap;

+        AZ::Event<FileStateInfo> m_deleteEvent;
+
        using LockGuardType = AZStd::lock_guard<decltype(m_mapMutex)>;
    };

@@ -131,5 +135,10 @@ namespace AssetProcessor
        bool GetFileInfo(const QString& absolutePath, FileStateInfo* foundFileInfo) const override;
        bool Exists(const QString& absolutePath) const override;
        bool GetHash(const QString& absolutePath, FileHash* foundHash) override;
+        void RegisterForDeleteEvent(AZ::Event<FileStateInfo>::Handler& handler) override;
+
+        void SignalDeleteEvent(const QString& absolutePath) const;
+    protected:
+        AZ::Event<FileStateInfo> m_deleteEvent;
    };
 } // namespace AssetProcessor
@@ -18,8 +18,8 @@

 #include <AzToolsFramework/Debug/TraceContext.h>

-
 #include "native/AssetManager/assetProcessorManager.h"
+
 #include <AzCore/std/sort.h>
 #include <AzToolsFramework/API/AssetDatabaseBus.h>

@@ -66,8 +66,10 @@ namespace AssetProcessor

        m_sourceFileRelocator = AZStd::make_unique<SourceFileRelocator>(m_stateData, m_platformConfig);

-        PopulateJobStateCache();
+        m_excludedFolderCache = AZStd::make_unique<ExcludedFolderCache>(m_platformConfig);

+        PopulateJobStateCache();
+        
        AssetProcessor::ProcessingJobInfoBus::Handler::BusConnect();
    }

@@ -3573,6 +3575,8 @@ namespace AssetProcessor
                    QString knownPathBeforeWildcard = encodedFileData.left(slashBeforeWildcardIndex + 1); // include the slash
                    QString relativeSearch = encodedFileData.mid(slashBeforeWildcardIndex + 1); // skip the slash

+                    const auto& excludedFolders = m_excludedFolderCache->GetExcludedFolders();
+
                    // Absolute path, just check the 1 scan folder
                    if (AZ::IO::PathView(encodedFileData.toUtf8().constData()).IsAbsolute())
                    {
@@ -3592,7 +3596,8 @@ namespace AssetProcessor
                            QString scanFolderAndKnownSubPath = rooted.absoluteFilePath(knownPathBeforeWildcard);

                            resolvedDependencyList.append(m_platformConfig->FindWildcardMatches(
-                                scanFolderAndKnownSubPath, relativeSearch, false, scanFolderInfo->RecurseSubFolders()));
+                                scanFolderAndKnownSubPath, relativeSearch,
+                                excludedFolders, false, scanFolderInfo->RecurseSubFolders()));
                        }
                    }
                    else // Relative path, check every scan folder
@@ -3610,7 +3615,21 @@ namespace AssetProcessor
                            QString absolutePath = rooted.absoluteFilePath(knownPathBeforeWildcard);

                            resolvedDependencyList.append(m_platformConfig->FindWildcardMatches(
-                                absolutePath, relativeSearch, false, scanFolderInfo->RecurseSubFolders()));
+                                absolutePath, relativeSearch,
+                                excludedFolders, false, scanFolderInfo->RecurseSubFolders()));
+                        }
+                    }
+
+                    // Filter out any excluded files
+                    for (auto itr = resolvedDependencyList.begin(); itr != resolvedDependencyList.end();)
+                    {
+                        if (m_platformConfig->IsFileExcluded(*itr))
+                        {
+                            itr = resolvedDependencyList.erase(itr);
+                        }
+                        else
+                        {
+                            ++itr;
                        }
                    }

@@ -40,6 +40,8 @@
 #include "AssetRequestHandler.h"
 #include "native/utilities/JobDiagnosticTracker.h"
 #include "SourceFileRelocator.h"
+
+#include <AssetManager/ExcludedFolderCache.h>
 #endif

 class FileWatcher;
@@ -341,7 +343,8 @@ namespace AssetProcessor
        void CleanEmptyFolder(QString folder, QString root);

        void ProcessBuilders(QString normalizedPath, QString relativePathToFile, const ScanFolderInfo* scanFolder, const AssetProcessor::BuilderInfoList& builderInfoList);
-        
+        AZStd::vector<AZStd::string> GetExcludedFolders();
+
        struct SourceInfo
        {
            QString m_watchFolder;
@@ -552,6 +555,8 @@ namespace AssetProcessor
        // when true, a flag will be sent to builders process job indicating debug output/mode should be used
        bool m_builderDebugFlag = false;

+        AZStd::unique_ptr<ExcludedFolderCache> m_excludedFolderCache{};
+
 protected Q_SLOTS:
        void FinishAnalysis(AZStd::string fileToCheck);
        //////////////////////////////////////////////////////////