Merging last dev

Signed-off-by: Gene Walters <genewalt@amazon.com>
This commit is contained in:
Gene Walters
2021-11-19 23:12:59 -08:00
2657 changed files with 249197 additions and 30535 deletions
@@ -0,0 +1,154 @@
/*
* Copyright (c) Contributors to the Open 3D Engine Project.
* For complete copyright and license terms please see the LICENSE at the root of this distribution.
*
* SPDX-License-Identifier: Apache-2.0 OR MIT
*
*/
#include <QDirIterator>
#include <AzCore/Settings/SettingsRegistryMergeUtils.h>
#include <native/AssetManager/ExcludedFolderCache.h>
#include <utilities/assetUtils.h>
#include <utilities/PlatformConfiguration.h>
#include <AzCore/IO/Path/Path.h>
namespace AssetProcessor
{
ExcludedFolderCache::ExcludedFolderCache(const PlatformConfiguration* platformConfig) : m_platformConfig(platformConfig)
{
AZ::Interface<ExcludedFolderCacheInterface>::Register(this);
}
ExcludedFolderCache::~ExcludedFolderCache()
{
AZ::Interface<ExcludedFolderCacheInterface>::Unregister(this);
}
const AZStd::unordered_set<AZStd::string>& ExcludedFolderCache::GetExcludedFolders()
{
if (!m_builtCache)
{
for (int i = 0; i < m_platformConfig->GetScanFolderCount(); ++i)
{
const auto& scanFolderInfo = m_platformConfig->GetScanFolderAt(i);
QDir rooted(scanFolderInfo.ScanPath());
QString absolutePath = rooted.absolutePath();
AZStd::stack<QString> dirs;
dirs.push(absolutePath);
while (!dirs.empty())
{
absolutePath = dirs.top();
dirs.pop();
// Scan only folders, do not recurse so we have the chance to ignore a subfolder before going deeper
QDirIterator dirIterator(absolutePath, QDir::Dirs | QDir::NoSymLinks | QDir::NoDotAndDotDot);
// Loop all the folders in this directory
while (dirIterator.hasNext())
{
dirIterator.next();
QString pathMatch = rooted.absoluteFilePath(dirIterator.filePath());
if (m_platformConfig->IsFileExcluded(pathMatch))
{
// Add the folder to the list and do not proceed any deeper
m_excludedFolders.emplace(pathMatch.toUtf8().constData());
}
else if (scanFolderInfo.RecurseSubFolders())
{
// Folder is not excluded and recurse is enabled, add to the list of folders to check
dirs.push(pathMatch);
}
}
}
}
// Add the cache to the list as well
AZStd::string projectCacheRootValue;
AZ::SettingsRegistry::Get()->Get(projectCacheRootValue, AZ::SettingsRegistryMergeUtils::FilePathKey_CacheProjectRootFolder);
projectCacheRootValue = AssetUtilities::NormalizeFilePath(projectCacheRootValue.c_str()).toUtf8().constData();
m_excludedFolders.emplace(projectCacheRootValue);
// Register to be notified about deletes so we can remove old ignored folders
auto fileStateCache = AZ::Interface<IFileStateRequests>::Get();
if (fileStateCache)
{
m_handler = AZ::Event<FileStateInfo>::Handler([this](FileStateInfo fileInfo)
{
if (fileInfo.m_isDirectory)
{
AZStd::scoped_lock lock(m_pendingNewFolderMutex);
m_pendingDeletes.emplace(fileInfo.m_absolutePath.toUtf8().constData());
}
});
fileStateCache->RegisterForDeleteEvent(m_handler);
}
else
{
AZ_Error("ExcludedFolderCache", false, "Failed to find IFileStateRequests interface");
}
m_builtCache = true;
}
// Incorporate any pending folders
AZStd::unordered_set<AZStd::string> pendingAdds;
AZStd::unordered_set<AZStd::string> pendingDeletes;
{
AZStd::scoped_lock lock(m_pendingNewFolderMutex);
pendingAdds.swap(m_pendingNewFolders);
pendingDeletes.swap(m_pendingDeletes);
}
if (!pendingAdds.empty())
{
m_excludedFolders.insert(pendingAdds.begin(), pendingAdds.end());
}
if (!pendingDeletes.empty())
{
for (const auto& pendingDelete : pendingDeletes)
{
m_excludedFolders.erase(pendingDelete);
}
}
return m_excludedFolders;
}
void ExcludedFolderCache::FileAdded(QString path)
{
QString relativePath, scanFolderPath;
if (!m_platformConfig->ConvertToRelativePath(path, relativePath, scanFolderPath))
{
AZ_Error("ExcludedFolderCache", false, "Failed to get relative path for newly added file %s", path.toUtf8().constData());
return;
}
AZ::IO::Path azPath(relativePath.toUtf8().constData());
AZ::IO::Path absolutePath(scanFolderPath.toUtf8().constData());
for (const auto& pathPart : azPath)
{
absolutePath /= pathPart;
QString normalized = AssetUtilities::NormalizeFilePath(absolutePath.c_str());
if (m_platformConfig->IsFileExcluded(normalized))
{
// Add the folder to a pending list, since this callback runs on another thread
AZStd::scoped_lock lock(m_pendingNewFolderMutex);
m_pendingNewFolders.emplace(normalized.toUtf8().constData());
break;
}
}
}
}
@@ -0,0 +1,39 @@
/*
* Copyright (c) Contributors to the Open 3D Engine Project.
* For complete copyright and license terms please see the LICENSE at the root of this distribution.
*
* SPDX-License-Identifier: Apache-2.0 OR MIT
*
*/
#pragma once
#include <AssetManager/ExcludedFolderCacheInterface.h>
#include <AssetManager/FileStateCache.h>
namespace AssetProcessor
{
class PlatformConfiguration;
struct ExcludedFolderCache : ExcludedFolderCacheInterface
{
explicit ExcludedFolderCache(const PlatformConfiguration* platformConfig);
~ExcludedFolderCache() override;
// Gets a set of absolute paths to folder which have been excluded according to the platform configuration rules
// Note - not thread safe
const AZStd::unordered_set<AZStd::string>& GetExcludedFolders() override;
void FileAdded(QString path) override;
private:
bool m_builtCache = false;
const PlatformConfiguration* m_platformConfig{};
AZStd::unordered_set<AZStd::string> m_excludedFolders;
AZStd::recursive_mutex m_pendingNewFolderMutex;
AZStd::unordered_set<AZStd::string> m_pendingNewFolders; // Newly ignored folders waiting to be added to m_excludedFolders
AZStd::unordered_set<AZStd::string> m_pendingDeletes;
AZ::Event<FileStateInfo>::Handler m_handler;
};
}
@@ -0,0 +1,29 @@
/*
* Copyright (c) Contributors to the Open 3D Engine Project.
* For complete copyright and license terms please see the LICENSE at the root of this distribution.
*
* SPDX-License-Identifier: Apache-2.0 OR MIT
*
*/
#pragma once
#include <AzCore/std/containers/unordered_set.h>
#include <QString>
namespace AssetProcessor
{
class PlatformConfiguration;
struct ExcludedFolderCacheInterface
{
AZ_RTTI(ExcludedFolderCacheInterface, "{3AC471B6-C9F8-49CF-9E9D-237BDF63328C}");
AZ_DISABLE_COPY_MOVE(ExcludedFolderCacheInterface);
ExcludedFolderCacheInterface() = default;
virtual ~ExcludedFolderCacheInterface() = default;
virtual const AZStd::unordered_set<AZStd::string>& GetExcludedFolders() = 0;
virtual void FileAdded(QString path) = 0;
};
}
@@ -63,6 +63,11 @@ namespace AssetProcessor
return true;
}
void FileStateCache::RegisterForDeleteEvent(AZ::Event<FileStateInfo>::Handler& handler)
{
handler.Connect(m_deleteEvent);
}
void FileStateCache::AddInfoSet(QSet<AssetFileInfo> infoSet)
{
LockGuardType scopeLock(m_mapMutex);
@@ -103,6 +108,8 @@ namespace AssetProcessor
if (itr != m_fileInfoMap.end())
{
m_deleteEvent.Signal(itr.value());
bool isDirectory = itr.value().m_isDirectory;
QString parentPath = itr.value().m_absolutePath;
m_fileInfoMap.erase(itr);
@@ -205,6 +212,21 @@ namespace AssetProcessor
return true;
}
void FileStatePassthrough::RegisterForDeleteEvent(AZ::Event<FileStateInfo>::Handler& handler)
{
handler.Connect(m_deleteEvent);
}
void FileStatePassthrough::SignalDeleteEvent(const QString& absolutePath) const
{
FileStateInfo info;
if (GetFileInfo(absolutePath, &info))
{
m_deleteEvent.Signal(info);
}
}
bool FileStateInfo::operator==(const FileStateInfo& rhs) const
{
return m_absolutePath == rhs.m_absolutePath
@@ -14,6 +14,7 @@
#include <QSet>
#include <QFileInfo>
#include <AzCore/Interface/Interface.h>
#include <AzCore/EBus/Event.h>
namespace AssetProcessor
{
@@ -51,10 +52,11 @@ namespace AssetProcessor
/// Convenience function to check if a file or directory exists.
virtual bool Exists(const QString& absolutePath) const = 0;
virtual bool GetHash(const QString& absolutePath, FileHash* foundHash) = 0;
virtual void RegisterForDeleteEvent(AZ::Event<FileStateInfo>::Handler& handler) = 0;
AZ_DISABLE_COPY_MOVE(IFileStateRequests);
};
class FileStateBase
: public IFileStateRequests
{
@@ -89,11 +91,11 @@ namespace AssetProcessor
{
public:
// FileStateRequestBus implementation
bool GetFileInfo(const QString& absolutePath, FileStateInfo* foundFileInfo) const override;
bool Exists(const QString& absolutePath) const override;
bool GetHash(const QString& absolutePath, FileHash* foundHash) override;
void RegisterForDeleteEvent(AZ::Event<FileStateInfo>::Handler& handler) override;
void AddInfoSet(QSet<AssetFileInfo> infoSet) override;
void AddFile(const QString& absolutePath) override;
@@ -116,9 +118,11 @@ namespace AssetProcessor
mutable AZStd::recursive_mutex m_mapMutex;
QHash<QString, FileStateInfo> m_fileInfoMap;
QHash<QString, FileHash> m_fileHashMap;
AZ::Event<FileStateInfo> m_deleteEvent;
using LockGuardType = AZStd::lock_guard<decltype(m_mapMutex)>;
};
@@ -131,5 +135,10 @@ namespace AssetProcessor
bool GetFileInfo(const QString& absolutePath, FileStateInfo* foundFileInfo) const override;
bool Exists(const QString& absolutePath) const override;
bool GetHash(const QString& absolutePath, FileHash* foundHash) override;
void RegisterForDeleteEvent(AZ::Event<FileStateInfo>::Handler& handler) override;
void SignalDeleteEvent(const QString& absolutePath) const;
protected:
AZ::Event<FileStateInfo> m_deleteEvent;
};
} // namespace AssetProcessor
@@ -18,8 +18,8 @@
#include <AzToolsFramework/Debug/TraceContext.h>
#include "native/AssetManager/assetProcessorManager.h"
#include <AzCore/std/sort.h>
#include <AzToolsFramework/API/AssetDatabaseBus.h>
@@ -66,8 +66,10 @@ namespace AssetProcessor
m_sourceFileRelocator = AZStd::make_unique<SourceFileRelocator>(m_stateData, m_platformConfig);
PopulateJobStateCache();
m_excludedFolderCache = AZStd::make_unique<ExcludedFolderCache>(m_platformConfig);
PopulateJobStateCache();
AssetProcessor::ProcessingJobInfoBus::Handler::BusConnect();
}
@@ -3573,6 +3575,8 @@ namespace AssetProcessor
QString knownPathBeforeWildcard = encodedFileData.left(slashBeforeWildcardIndex + 1); // include the slash
QString relativeSearch = encodedFileData.mid(slashBeforeWildcardIndex + 1); // skip the slash
const auto& excludedFolders = m_excludedFolderCache->GetExcludedFolders();
// Absolute path, just check the 1 scan folder
if (AZ::IO::PathView(encodedFileData.toUtf8().constData()).IsAbsolute())
{
@@ -3592,7 +3596,8 @@ namespace AssetProcessor
QString scanFolderAndKnownSubPath = rooted.absoluteFilePath(knownPathBeforeWildcard);
resolvedDependencyList.append(m_platformConfig->FindWildcardMatches(
scanFolderAndKnownSubPath, relativeSearch, false, scanFolderInfo->RecurseSubFolders()));
scanFolderAndKnownSubPath, relativeSearch,
excludedFolders, false, scanFolderInfo->RecurseSubFolders()));
}
}
else // Relative path, check every scan folder
@@ -3610,7 +3615,21 @@ namespace AssetProcessor
QString absolutePath = rooted.absoluteFilePath(knownPathBeforeWildcard);
resolvedDependencyList.append(m_platformConfig->FindWildcardMatches(
absolutePath, relativeSearch, false, scanFolderInfo->RecurseSubFolders()));
absolutePath, relativeSearch,
excludedFolders, false, scanFolderInfo->RecurseSubFolders()));
}
}
// Filter out any excluded files
for (auto itr = resolvedDependencyList.begin(); itr != resolvedDependencyList.end();)
{
if (m_platformConfig->IsFileExcluded(*itr))
{
itr = resolvedDependencyList.erase(itr);
}
else
{
++itr;
}
}
@@ -40,6 +40,8 @@
#include "AssetRequestHandler.h"
#include "native/utilities/JobDiagnosticTracker.h"
#include "SourceFileRelocator.h"
#include <AssetManager/ExcludedFolderCache.h>
#endif
class FileWatcher;
@@ -341,7 +343,8 @@ namespace AssetProcessor
void CleanEmptyFolder(QString folder, QString root);
void ProcessBuilders(QString normalizedPath, QString relativePathToFile, const ScanFolderInfo* scanFolder, const AssetProcessor::BuilderInfoList& builderInfoList);
AZStd::vector<AZStd::string> GetExcludedFolders();
struct SourceInfo
{
QString m_watchFolder;
@@ -552,6 +555,8 @@ namespace AssetProcessor
// when true, a flag will be sent to builders process job indicating debug output/mode should be used
bool m_builderDebugFlag = false;
AZStd::unique_ptr<ExcludedFolderCache> m_excludedFolderCache{};
protected Q_SLOTS:
void FinishAnalysis(AZStd::string fileToCheck);
//////////////////////////////////////////////////////////
@@ -26,7 +26,7 @@ namespace AssetProcessor
{
protected:
AZStd::unique_ptr<UnitTestUtils::AssertAbsorber> m_errorAbsorber{};
FileStatePassthrough m_fileStateCache;
AZStd::unique_ptr<FileStatePassthrough> m_fileStateCache{};
void SetUp() override
{
@@ -40,9 +40,10 @@ namespace AssetProcessor
m_ownsSysAllocator = true;
AZ::AllocatorInstance<AZ::SystemAllocator>::Create();
}
m_errorAbsorber = AZStd::make_unique<UnitTestUtils::AssertAbsorber>();
m_errorAbsorber = AZStd::make_unique<UnitTestUtils::AssertAbsorber>();
m_application = AZStd::make_unique<AzFramework::Application>();
m_fileStateCache = AZStd::make_unique<FileStatePassthrough>();
// Inject the AutomatedTesting project as a project path into test fixture
using FixedValueString = AZ::SettingsRegistryInterface::FixedValueString;
@@ -60,7 +61,8 @@ namespace AssetProcessor
void TearDown() override
{
AssetUtilities::ResetAssetRoot();
m_fileStateCache.reset();
m_application.reset();
m_errorAbsorber.reset();
@@ -5364,13 +5364,29 @@ AZStd::vector<AZStd::string> WildcardSourceDependencyTest::FileAddedTest(const Q
void WildcardSourceDependencyTest::SetUp()
{
AssetProcessorManagerTest::SetUp();
QDir tempPath(m_tempDir.path());
// Add a non-recursive scan folder. Only files directly inside of this folder should be picked up, subfolders are ignored
m_config->AddScanFolder(ScanFolderInfo(tempPath.filePath("no_recurse"), "no_recurse",
"no_recurse", false, false, m_config->GetEnabledPlatforms(), 1));
{
ExcludeAssetRecognizer excludeFolder;
excludeFolder.m_name = "Exclude ignored Folder";
excludeFolder.m_patternMatcher =
AssetBuilderSDK::FilePatternMatcher(R"REGEX(^(.*\/)?ignored(\/.*)?$)REGEX", AssetBuilderSDK::AssetBuilderPattern::Regex);
m_config->AddExcludeRecognizer(excludeFolder);
}
{
ExcludeAssetRecognizer excludeFile;
excludeFile.m_name = "Exclude z.foo Files";
excludeFile.m_patternMatcher =
AssetBuilderSDK::FilePatternMatcher(R"REGEX(^(.*\/)?z\.foo$)REGEX", AssetBuilderSDK::AssetBuilderPattern::Regex);
m_config->AddExcludeRecognizer(excludeFile);
}
UnitTestUtils::CreateDummyFile(tempPath.absoluteFilePath("subfolder1/1a.foo"));
UnitTestUtils::CreateDummyFile(tempPath.absoluteFilePath("subfolder1/1b.foo"));
UnitTestUtils::CreateDummyFile(tempPath.absoluteFilePath("subfolder2/redirected/a.foo"));
@@ -5384,6 +5400,19 @@ void WildcardSourceDependencyTest::SetUp()
// Add a file in the non-recursive scanfolder. Since its not directly in the scan folder, it should always be ignored
UnitTestUtils::CreateDummyFile(tempPath.absoluteFilePath("no_recurse/one/two/three/f.foo"));
// Add a file to an ignored folder
UnitTestUtils::CreateDummyFile(tempPath.absoluteFilePath("subfolder2/redirected/folder/ignored/g.foo"));
// Add an ignored file
UnitTestUtils::CreateDummyFile(tempPath.absoluteFilePath("subfolder2/redirected/folder/one/z.foo"));
// Add a file in the cache
AZStd::string projectCacheRootValue;
AZ::SettingsRegistry::Get()->Get(projectCacheRootValue, AZ::SettingsRegistryMergeUtils::FilePathKey_CacheProjectRootFolder);
projectCacheRootValue = AssetUtilities::NormalizeFilePath(projectCacheRootValue.c_str()).toUtf8().constData();
auto path = AZ::IO::Path(projectCacheRootValue) / "cache.foo";
UnitTestUtils::CreateDummyFile(path.c_str());
AzToolsFramework::AssetDatabase::SourceFileDependencyEntryContainer dependencies;
// Relative path wildcard dependency
@@ -5518,6 +5547,102 @@ TEST_F(WildcardSourceDependencyTest, Absolute_NoWildcard)
ASSERT_THAT(resolvedPaths, ::testing::UnorderedElementsAre());
}
TEST_F(WildcardSourceDependencyTest, Relative_IgnoredFolder)
{
AZStd::vector<AZStd::string> resolvedPaths;
ASSERT_TRUE(Test("*g.foo", resolvedPaths));
ASSERT_THAT(resolvedPaths, ::testing::UnorderedElementsAre());
}
TEST_F(WildcardSourceDependencyTest, Absolute_IgnoredFolder)
{
AZStd::vector<AZStd::string> resolvedPaths;
QDir tempPath(m_tempDir.path());
ASSERT_TRUE(Test(tempPath.absoluteFilePath("*g.foo").toUtf8().constData(), resolvedPaths));
ASSERT_THAT(resolvedPaths, ::testing::UnorderedElementsAre());
}
TEST_F(WildcardSourceDependencyTest, Relative_IgnoredFile)
{
AZStd::vector<AZStd::string> resolvedPaths;
ASSERT_TRUE(Test("*z.foo", resolvedPaths));
ASSERT_THAT(resolvedPaths, ::testing::UnorderedElementsAre());
}
TEST_F(WildcardSourceDependencyTest, Absolute_IgnoredFile)
{
AZStd::vector<AZStd::string> resolvedPaths;
QDir tempPath(m_tempDir.path());
ASSERT_TRUE(Test(tempPath.absoluteFilePath("*z.foo").toUtf8().constData(), resolvedPaths));
ASSERT_THAT(resolvedPaths, ::testing::UnorderedElementsAre());
}
TEST_F(WildcardSourceDependencyTest, Relative_CacheFolder)
{
AZStd::vector<AZStd::string> resolvedPaths;
QDir tempPath(m_tempDir.path());
ASSERT_TRUE(Test("*cache.foo", resolvedPaths));
ASSERT_THAT(resolvedPaths, ::testing::UnorderedElementsAre());
}
TEST_F(WildcardSourceDependencyTest, FilesAddedAfterInitialCache)
{
AZStd::vector<AZStd::string> resolvedPaths;
QDir tempPath(m_tempDir.path());
auto excludedFolderCacheInterface = AZ::Interface<ExcludedFolderCacheInterface>::Get();
ASSERT_TRUE(excludedFolderCacheInterface);
{
const auto& excludedFolders = excludedFolderCacheInterface->GetExcludedFolders();
ASSERT_EQ(excludedFolders.size(), 2);
}
// Add a file to a new ignored folder
QString newFilePath = tempPath.absoluteFilePath("subfolder2/redirected/folder/two/ignored/three/new.foo");
UnitTestUtils::CreateDummyFile(newFilePath);
excludedFolderCacheInterface->FileAdded(newFilePath);
const auto& excludedFolders = excludedFolderCacheInterface->GetExcludedFolders();
ASSERT_EQ(excludedFolders.size(), 3);
ASSERT_THAT(excludedFolders, ::testing::Contains(AZStd::string(tempPath.absoluteFilePath("subfolder2/redirected/folder/two/ignored").toUtf8().constData())));
}
TEST_F(WildcardSourceDependencyTest, FilesRemovedAfterInitialCache)
{
AZStd::vector<AZStd::string> resolvedPaths;
QDir tempPath(m_tempDir.path());
// Add a file to a new ignored folder
QString newFilePath = tempPath.absoluteFilePath("subfolder2/redirected/folder/two/ignored/three/new.foo");
UnitTestUtils::CreateDummyFile(newFilePath);
auto excludedFolderCacheInterface = AZ::Interface<ExcludedFolderCacheInterface>::Get();
ASSERT_TRUE(excludedFolderCacheInterface);
{
const auto& excludedFolders = excludedFolderCacheInterface->GetExcludedFolders();
ASSERT_EQ(excludedFolders.size(), 3);
}
m_fileStateCache->SignalDeleteEvent(tempPath.absoluteFilePath("subfolder2/redirected/folder/two/ignored"));
const auto& excludedFolders = excludedFolderCacheInterface->GetExcludedFolders();
ASSERT_EQ(excludedFolders.size(), 2);
}
TEST_F(WildcardSourceDependencyTest, NewFile_MatchesSavedRelativeDependency)
{
QDir tempPath(m_tempDir.path());
@@ -52,11 +52,12 @@ TEST_F(PlatformConfigurationUnitTests, TestFailReadConfigFile_BadPlatform)
using namespace AssetProcessor;
const auto testExeFolder = AZ::IO::FileIOBase::GetInstance()->ResolvePath(TestAppRoot);
const AZ::IO::FixedMaxPath projectPath = (*testExeFolder) / EmptyDummyProjectName;
auto configRoot = AZ::IO::FileIOBase::GetInstance()->ResolvePath("@exefolder@/testdata/config_broken_badplatform");
ASSERT_TRUE(configRoot);
UnitTestPlatformConfiguration config;
m_absorber.Clear();
ASSERT_FALSE(config.InitializeFromConfigFiles(configRoot->c_str(), testExeFolder->c_str(), EmptyDummyProjectName, false, false));
ASSERT_FALSE(config.InitializeFromConfigFiles(configRoot->c_str(), testExeFolder->c_str(), projectPath.c_str(), false, false));
ASSERT_GT(m_absorber.m_numErrorsAbsorbed, 0);
}
@@ -67,11 +68,12 @@ TEST_F(PlatformConfigurationUnitTests, TestFailReadConfigFile_NoPlatform)
using namespace AssetProcessor;
const auto testExeFolder = AZ::IO::FileIOBase::GetInstance()->ResolvePath(TestAppRoot);
const AZ::IO::FixedMaxPath projectPath = (*testExeFolder) / EmptyDummyProjectName;
auto configRoot = AZ::IO::FileIOBase::GetInstance()->ResolvePath("@exefolder@/testdata/config_broken_noplatform");
ASSERT_TRUE(configRoot);
UnitTestPlatformConfiguration config;
m_absorber.Clear();
ASSERT_FALSE(config.InitializeFromConfigFiles(configRoot->c_str(), testExeFolder->c_str(), EmptyDummyProjectName, false, false));
ASSERT_FALSE(config.InitializeFromConfigFiles(configRoot->c_str(), testExeFolder->c_str(), projectPath.c_str(), false, false));
ASSERT_GT(m_absorber.m_numErrorsAbsorbed, 0);
}
@@ -81,11 +83,12 @@ TEST_F(PlatformConfigurationUnitTests, TestFailReadConfigFile_NoScanFolders)
using namespace AssetProcessor;
const auto testExeFolder = AZ::IO::FileIOBase::GetInstance()->ResolvePath(TestAppRoot);
const AZ::IO::FixedMaxPath projectPath = (*testExeFolder) / EmptyDummyProjectName;
auto configRoot = AZ::IO::FileIOBase::GetInstance()->ResolvePath("@exefolder@/testdata/config_broken_noscans");
ASSERT_TRUE(configRoot);
UnitTestPlatformConfiguration config;
m_absorber.Clear();
ASSERT_FALSE(config.InitializeFromConfigFiles(configRoot->c_str(), testExeFolder->c_str(), EmptyDummyProjectName, false, false));
ASSERT_FALSE(config.InitializeFromConfigFiles(configRoot->c_str(), testExeFolder->c_str(), projectPath.c_str(), false, false));
ASSERT_GT(m_absorber.m_numErrorsAbsorbed, 0);
}
@@ -95,11 +98,12 @@ TEST_F(PlatformConfigurationUnitTests, TestFailReadConfigFile_BrokenRecognizers)
using namespace AssetProcessor;
const auto testExeFolder = AZ::IO::FileIOBase::GetInstance()->ResolvePath(TestAppRoot);
const AZ::IO::FixedMaxPath projectPath = (*testExeFolder) / EmptyDummyProjectName;
auto configRoot = AZ::IO::FileIOBase::GetInstance()->ResolvePath("@exefolder@/testdata/config_broken_recognizers");
ASSERT_TRUE(configRoot);
UnitTestPlatformConfiguration config;
m_absorber.Clear();
ASSERT_FALSE(config.InitializeFromConfigFiles(configRoot->c_str(), testExeFolder->c_str(), EmptyDummyProjectName, false, false));
ASSERT_FALSE(config.InitializeFromConfigFiles(configRoot->c_str(), testExeFolder->c_str(), projectPath.c_str(), false, false));
ASSERT_GT(m_absorber.m_numErrorsAbsorbed, 0);
}
@@ -109,11 +113,12 @@ TEST_F(PlatformConfigurationUnitTests, TestFailReadConfigFile_Regular_Platforms)
using namespace AssetProcessor;
const auto testExeFolder = AZ::IO::FileIOBase::GetInstance()->ResolvePath(TestAppRoot);
const AZ::IO::FixedMaxPath projectPath = (*testExeFolder) / EmptyDummyProjectName;
auto configRoot = AZ::IO::FileIOBase::GetInstance()->ResolvePath("@exefolder@/testdata/config_regular");
ASSERT_TRUE(configRoot);
UnitTestPlatformConfiguration config;
m_absorber.Clear();
ASSERT_TRUE(config.InitializeFromConfigFiles(configRoot->c_str(), testExeFolder->c_str(), EmptyDummyProjectName, false, false));
ASSERT_TRUE(config.InitializeFromConfigFiles(configRoot->c_str(), testExeFolder->c_str(), projectPath.c_str(), false, false));
ASSERT_EQ(m_absorber.m_numErrorsAbsorbed, 0);
// verify the data.
@@ -322,12 +327,13 @@ TEST_F(PlatformConfigurationUnitTests, TestFailReadConfigFile_RegularScanfolder)
using namespace AssetProcessor;
const auto testExeFolder = AZ::IO::FileIOBase::GetInstance()->ResolvePath(TestAppRoot);
const AZ::IO::FixedMaxPath projectPath = (*testExeFolder) / EmptyDummyProjectName;
auto configRoot = AZ::IO::FileIOBase::GetInstance()->ResolvePath("@exefolder@/testdata/config_regular");
ASSERT_TRUE(configRoot);
UnitTestPlatformConfiguration config;
m_absorber.Clear();
AssetUtilities::ComputeProjectName(EmptyDummyProjectName, true);
ASSERT_TRUE(config.InitializeFromConfigFiles(configRoot->c_str(), testExeFolder->c_str(), EmptyDummyProjectName, false, false));
ASSERT_TRUE(config.InitializeFromConfigFiles(configRoot->c_str(), testExeFolder->c_str(), projectPath.c_str(), false, false));
ASSERT_EQ(m_absorber.m_numErrorsAbsorbed, 0);
ASSERT_EQ(config.GetScanFolderCount(), 3); // the two, and then the one that has the same data as prior but different identifier.
@@ -356,11 +362,12 @@ TEST_F(PlatformConfigurationUnitTests, TestFailReadConfigFile_RegularScanfolderP
using namespace AssetProcessor;
const auto testExeFolder = AZ::IO::FileIOBase::GetInstance()->ResolvePath(TestAppRoot);
const AZ::IO::FixedMaxPath projectPath = (*testExeFolder) / EmptyDummyProjectName;
auto configRoot = AZ::IO::FileIOBase::GetInstance()->ResolvePath("@exefolder@/testdata/config_regular_platform_scanfolder");
ASSERT_TRUE(configRoot);
UnitTestPlatformConfiguration config;
m_absorber.Clear();
ASSERT_TRUE(config.InitializeFromConfigFiles(configRoot->c_str(), testExeFolder->c_str(), EmptyDummyProjectName, false, false));
ASSERT_TRUE(config.InitializeFromConfigFiles(configRoot->c_str(), testExeFolder->c_str(), projectPath.c_str(), false, false));
ASSERT_EQ(m_absorber.m_numErrorsAbsorbed, 0);
ASSERT_EQ(config.GetScanFolderCount(), 5);
@@ -402,13 +409,14 @@ TEST_F(PlatformConfigurationUnitTests, TestFailReadConfigFile_RegularExcludes)
using namespace AssetProcessor;
const auto testExeFolder = AZ::IO::FileIOBase::GetInstance()->ResolvePath(TestAppRoot);
const AZ::IO::FixedMaxPath projectPath = (*testExeFolder) / EmptyDummyProjectName;
auto configRoot = AZ::IO::FileIOBase::GetInstance()->ResolvePath("@exefolder@/testdata/config_regular");
ASSERT_TRUE(configRoot);
UnitTestPlatformConfiguration config;
config.AddScanFolder(ScanFolderInfo("blahblah", "Blah ScanFolder", "sf2", true, true), true);
m_absorber.Clear();
ASSERT_TRUE(config.InitializeFromConfigFiles(configRoot->c_str(), testExeFolder->c_str(), EmptyDummyProjectName, false, false));
ASSERT_TRUE(config.InitializeFromConfigFiles(configRoot->c_str(), testExeFolder->c_str(), projectPath.c_str(), false, false));
ASSERT_EQ(m_absorber.m_numErrorsAbsorbed, 0);
ASSERT_TRUE(config.IsFileExcluded("blahblah/$tmp_01.test"));
@@ -429,11 +437,12 @@ TEST_F(PlatformConfigurationUnitTests, TestFailReadConfigFile_Recognizers)
#endif
const auto testExeFolder = AZ::IO::FileIOBase::GetInstance()->ResolvePath(TestAppRoot);
const AZ::IO::FixedMaxPath projectPath = (*testExeFolder) / EmptyDummyProjectName;
auto configRoot = AZ::IO::FileIOBase::GetInstance()->ResolvePath("@exefolder@/testdata/config_regular");
ASSERT_TRUE(configRoot);
UnitTestPlatformConfiguration config;
m_absorber.Clear();
ASSERT_TRUE(config.InitializeFromConfigFiles(configRoot->c_str(), testExeFolder->c_str(), EmptyDummyProjectName, false, false));
ASSERT_TRUE(config.InitializeFromConfigFiles(configRoot->c_str(), testExeFolder->c_str(), projectPath.c_str(), false, false));
ASSERT_EQ(m_absorber.m_numErrorsAbsorbed, 0);
const AssetProcessor::RecognizerContainer& recogs = config.GetAssetRecognizerContainer();
@@ -520,12 +529,13 @@ TEST_F(PlatformConfigurationUnitTests, TestFailReadConfigFile_Overrides)
using namespace AzToolsFramework::AssetSystem;
using namespace AssetProcessor;
const auto testExeFolder = AZ::IO::FileIOBase::GetInstance()->ResolvePath(TestAppRoot);
const AZ::IO::FixedMaxPath projectPath = (*testExeFolder) / DummyProjectName;
auto configRoot = AZ::IO::FileIOBase::GetInstance()->ResolvePath("@exefolder@/testdata/config_regular");
ASSERT_TRUE(configRoot);
UnitTestPlatformConfiguration config;
m_absorber.Clear();
ASSERT_TRUE(config.InitializeFromConfigFiles(configRoot->c_str(), testExeFolder->c_str(), DummyProjectName, false, false));
ASSERT_TRUE(config.InitializeFromConfigFiles(configRoot->c_str(), testExeFolder->c_str(), projectPath.c_str(), false, false));
ASSERT_EQ(m_absorber.m_numErrorsAbsorbed, 0);
const AssetProcessor::RecognizerContainer& recogs = config.GetAssetRecognizerContainer();
@@ -627,11 +637,12 @@ TEST_F(PlatformConfigurationUnitTests, ReadCheckServer_FromConfig_Valid)
using namespace AssetProcessor;
const auto testExeFolder = AZ::IO::FileIOBase::GetInstance()->ResolvePath(TestAppRoot);
const AZ::IO::FixedMaxPath projectPath = (*testExeFolder) / EmptyDummyProjectName;
auto configRoot = AZ::IO::FileIOBase::GetInstance()->ResolvePath("@exefolder@/testdata/config_regular");
ASSERT_TRUE(configRoot);
UnitTestPlatformConfiguration config;
m_absorber.Clear();
ASSERT_TRUE(config.InitializeFromConfigFiles(configRoot->c_str(), testExeFolder->c_str(), EmptyDummyProjectName, false, false));
ASSERT_TRUE(config.InitializeFromConfigFiles(configRoot->c_str(), testExeFolder->c_str(), projectPath.c_str(), false, false));
ASSERT_EQ(m_absorber.m_numErrorsAbsorbed, 0);
const AssetProcessor::RecognizerContainer& recogs = config.GetAssetRecognizerContainer();
@@ -676,11 +687,12 @@ TEST_F(PlatformConfigurationUnitTests, Test_MetaFileTypes_AssetImporterExtension
using namespace AssetProcessor;
const auto testExeFolder = AZ::IO::FileIOBase::GetInstance()->ResolvePath(TestAppRoot);
const AZ::IO::FixedMaxPath projectPath = (*testExeFolder) / EmptyDummyProjectName;
auto configRoot = AZ::IO::FileIOBase::GetInstance()->ResolvePath("@exefolder@/testdata/config_metadata");
ASSERT_TRUE(configRoot);
UnitTestPlatformConfiguration config;
m_absorber.Clear();
ASSERT_FALSE(config.InitializeFromConfigFiles(configRoot->c_str(), testExeFolder->c_str(), EmptyDummyProjectName, false, false));
ASSERT_FALSE(config.InitializeFromConfigFiles(configRoot->c_str(), testExeFolder->c_str(), projectPath.c_str(), false, false));
ASSERT_GT(m_absorber.m_numErrorsAbsorbed, 0);
ASSERT_TRUE(config.MetaDataFileTypesCount() == 2);
@@ -454,6 +454,8 @@ void ApplicationManagerBase::InitFileMonitor()
QObject::connect(newFolderWatch, &FolderWatchCallbackEx::fileModified, [this](QString path) { m_fileStateCache->UpdateFile(path); });
QObject::connect(newFolderWatch, &FolderWatchCallbackEx::fileRemoved, [this](QString path) { m_fileStateCache->RemoveFile(path); });
QObject::connect(newFolderWatch, &FolderWatchCallbackEx::fileAdded, [](QString path) { AZ::Interface<AssetProcessor::ExcludedFolderCacheInterface>::Get()->FileAdded(path); });
QObject::connect(newFolderWatch, &FolderWatchCallbackEx::fileAdded,
m_fileProcessor.get(), &AssetProcessor::FileProcessor::AssessAddedFile);
QObject::connect(newFolderWatch, &FolderWatchCallbackEx::fileRemoved,
@@ -749,7 +749,7 @@ namespace AssetProcessor
}
AZStd::vector<AZ::IO::Path> configFiles = AzToolsFramework::AssetUtils::GetConfigFiles(absoluteSystemRoot.toUtf8().constData(),
absoluteAssetRoot.toUtf8().constData(), projectPath.toUtf8().constData(),
projectPath.toUtf8().constData(),
addPlatformConfigs, addGemsConfigs && !noGemScanFolders, settingsRegistry);
// First Merge all Engine, Gem and Project specific AssetProcessor*Config.setreg/.inifiles
@@ -1285,6 +1285,13 @@ namespace AssetProcessor
return m_scanFolders[index];
}
const AssetProcessor::ScanFolderInfo& PlatformConfiguration::GetScanFolderAt(int index) const
{
Q_ASSERT(index >= 0);
Q_ASSERT(index < m_scanFolders.size());
return m_scanFolders[index];
}
void PlatformConfiguration::AddScanFolder(const AssetProcessor::ScanFolderInfo& source, bool isUnitTesting)
{
if (isUnitTesting)
@@ -1436,7 +1443,10 @@ namespace AssetProcessor
}
QStringList PlatformConfiguration::FindWildcardMatches(
const QString& sourceFolder, QString relativeName, bool includeFolders, bool recursiveSearch) const
const QString& sourceFolder,
QString relativeName,
bool includeFolders,
bool recursiveSearch) const
{
if (relativeName.isEmpty())
{
@@ -1469,6 +1479,67 @@ namespace AssetProcessor
return returnList;
}
QStringList PlatformConfiguration::FindWildcardMatches(
const QString& sourceFolder,
QString relativeName,
const AZStd::unordered_set<AZStd::string>& excludedFolders,
bool includeFolders,
bool recursiveSearch) const
{
if (relativeName.isEmpty())
{
return QStringList();
}
QDir sourceFolderDir(sourceFolder);
QString posixRelativeName = QDir::fromNativeSeparators(relativeName);
QStringList returnList;
QRegExp nameMatch{ posixRelativeName, Qt::CaseInsensitive, QRegExp::Wildcard };
AZStd::stack<QString> dirs;
dirs.push(sourceFolderDir.absolutePath());
while (!dirs.empty())
{
QString absolutePath = dirs.top();
dirs.pop();
if (excludedFolders.contains(absolutePath.toUtf8().constData()))
{
continue;
}
QDirIterator dirIterator(absolutePath, QDir::AllEntries | QDir::NoSymLinks | QDir::NoDotAndDotDot);
while (dirIterator.hasNext())
{
dirIterator.next();
if (!dirIterator.fileInfo().isFile())
{
if (recursiveSearch)
{
dirs.push(dirIterator.filePath());
}
if (!includeFolders)
{
continue;
}
}
QString pathMatch{ sourceFolderDir.relativeFilePath(dirIterator.filePath()) };
if (nameMatch.exactMatch(pathMatch))
{
returnList.append(QDir::fromNativeSeparators(dirIterator.filePath()));
}
}
}
return returnList;
}
const AssetProcessor::ScanFolderInfo* PlatformConfiguration::GetScanFolderForFile(const QString& fullFileName) const
{
QString normalized = AssetUtilities::NormalizeFilePath(fullFileName);
@@ -256,6 +256,9 @@ namespace AssetProcessor
//! Retrieve the scan folder at a given index.
AssetProcessor::ScanFolderInfo& GetScanFolderAt(int index);
//! Retrieve the scan folder at a given index.
const AssetProcessor::ScanFolderInfo& GetScanFolderAt(int index) const;
//! Manually add a scan folder. Also used for testing.
void AddScanFolder(const AssetProcessor::ScanFolderInfo& source, bool isUnitTesting = false);
@@ -298,7 +301,16 @@ namespace AssetProcessor
QString FindFirstMatchingFile(QString relativeName) const;
//! given a relative name with wildcard characters (* allowed) find a set of matching files or optionally folders
QStringList FindWildcardMatches(const QString& sourceFolder, QString relativeName, bool includeFolders = false, bool recursiveSearch = true) const;
QStringList FindWildcardMatches(const QString& sourceFolder, QString relativeName, bool includeFolders = false,
bool recursiveSearch = true) const;
//! given a relative name with wildcard characters (* allowed) find a set of matching files or optionally folders
QStringList FindWildcardMatches(
const QString& sourceFolder,
QString relativeName,
const AZStd::unordered_set<AZStd::string>& excludedFolders,
bool includeFolders = false,
bool recursiveSearch = true) const;
//! given a fileName (as a full path), return the database source name which includes the output prefix.
//!
@@ -1161,7 +1161,7 @@ namespace AssetUtilities
{
#ifndef AZ_TESTS_ENABLED
// Only used for unit tests, speed is critical for GetFileHash.
AZ_UNUSED(hashMsDelay);
hashMsDelay = 0;
#endif
bool useFileHashing = ShouldUseFileHashing();
@@ -1170,10 +1170,10 @@ namespace AssetUtilities
return 0;
}
AZ::u64 hash = 0;
if(!force)
{
auto* fileStateInterface = AZ::Interface<AssetProcessor::IFileStateRequests>::Get();
AZ::u64 hash = 0;
if (fileStateInterface && fileStateInterface->GetHash(filePath, &hash))
{
@@ -1181,64 +1181,8 @@ namespace AssetUtilities
}
}
char buffer[FileHashBufferSize];
constexpr bool ErrorOnReadFailure = true;
AZ::IO::FileIOStream readStream(filePath, AZ::IO::OpenMode::ModeRead | AZ::IO::OpenMode::ModeBinary, ErrorOnReadFailure);
if(readStream.IsOpen() && readStream.CanRead())
{
AZ::IO::SizeType bytesRead;
auto* state = XXH64_createState();
if(state == nullptr)
{
AZ_Assert(false, "Failed to create hash state");
return 0;
}
if (XXH64_reset(state, 0) == XXH_ERROR)
{
AZ_Assert(false, "Failed to reset hash state");
return 0;
}
do
{
// In edge cases where another process is writing to this file while this hashing is occuring and that file wasn't locked,
// the following read check can fail because it performs an end of file check, and asserts and shuts down if the read size
// was smaller than the buffer and the read is not at the end of the file. The logic used to check end of file internal to read
// will be out of date in the edge cases where another process is actively writing to this file while this hash is running.
// The stream's length ends up more accurate in this case, preventing this assert and shut down.
// One area this occurs is the navigation mesh file (mnmnavmission0.bai) that's temporarily created when exporting a level,
// the navigation system can still be writing to this file when hashing begins, causing the EoF marker to change.
AZ::IO::SizeType remainingToRead = AZStd::min(readStream.GetLength() - readStream.GetCurPos(), aznumeric_cast<AZ::IO::SizeType>(AZ_ARRAY_SIZE(buffer)));
bytesRead = readStream.Read(remainingToRead, buffer);
if(bytesReadOut)
{
*bytesReadOut += bytesRead;
}
XXH64_update(state, buffer, bytesRead);
#ifdef AZ_TESTS_ENABLED
// Used by unit tests to force the race condition mentioned above, to verify the crash fix.
if(hashMsDelay > 0)
{
AZStd::this_thread::sleep_for(AZStd::chrono::milliseconds(hashMsDelay));
}
#endif
} while (bytesRead > 0);
auto hash = XXH64_digest(state);
XXH64_freeState(state);
return hash;
}
return 0;
hash = AssetBuilderSDK::GetFileHash(filePath, bytesReadOut, hashMsDelay);
return hash;
}
AZ::u64 AdjustTimestamp(QDateTime timestamp)
@@ -238,7 +238,6 @@ namespace AssetUtilities
// hashMsDelay is only for automated tests to test that writing to a file while it's hashing does not cause a crash.
// hashMsDelay is not used in non-unit test builds.
AZ::u64 GetFileHash(const char* filePath, bool force = false, AZ::IO::SizeType* bytesReadOut = nullptr, int hashMsDelay = 0);
inline constexpr AZ::u64 FileHashBufferSize = 1024 * 64;
//! Adjusts a timestamp to fix timezone settings and account for any precision adjustment needed
AZ::u64 AdjustTimestamp(QDateTime timestamp);