birkeh/o3de
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Pass relative path IsFileExcluded so that only paths relative to the root scan folder are matched against the exclude filters (#4504)

Browse Source 
* Pass relative path IsFileExcluded so that only paths relative to the root scan folder are matched against the exclude filters.

Signed-off-by: amzn-sj <srikkant@amazon.com>

* Revert previous change. Remove the exclude filter for the Install directory.

Signed-off-by: amzn-sj <srikkant@amazon.com>

* Pass in relative path to the exclude filter as before. Fix the AssetScanner tests.

Signed-off-by: amzn-sj <srikkant@amazon.com>

* Prepend a ./ to the relative path in order to match the exclude patterns

Signed-off-by: amzn-sj <srikkant@amazon.com>

* Remove hack to prepend ./. Update the exclude patterns so that the hack is no longer required.

Signed-off-by: amzn-sj <srikkant@amazon.com>

* Add missing ? and remove whitespace

Signed-off-by: amzn-sj <srikkant@amazon.com>

* 1. IsFileExcluded() now converts the input path to a path that's relative to its corresponding scan folder.
2. Update regex patterns in gems and AutomatedTesting as well.
3. Remove unnecessary escaping for '/'.

Signed-off-by: amzn-sj <srikkant@amazon.com>

* Use ConvertToRelativePath() function to compute path relative to a scan folder.

Signed-off-by: amzn-sj <srikkant@amazon.com>

* More fixes to regex patterns

Signed-off-by: amzn-sj <srikkant@amazon.com>

* Remove test case which tests a hypothetical scenario that cannot occur. Fix another test case by adding scan folder.

Signed-off-by: amzn-sj <srikkant@amazon.com>

* Remove assert that's not needed since it's a valid scenario

Signed-off-by: amzn-sj <srikkant@amazon.com>
Signed-off-by: Gene Walters <genewalt@amazon.com>
This commit is contained in:
SJ
2021-10-26 09:43:03 -07:00
committed by Gene Walters
parent 769e2e3942
commit 104170879e
11 changed files with 47 additions and 43 deletions
Download Patch File Download Diff File Expand all files Collapse all files
AutomatedTesting/Gem/AssetProcessorGemConfig.setreg
+5 -5
View File
@@ -3,19 +3,19 @@
"AssetProcessor": {
"Settings": {
"Exclude PythonTest Benchmark Settings Assets": {
"pattern": ".*\\\\/PythonTests\\\\/.*benchmarksettings"
"pattern": "(^|.+/)PythonTests/.*benchmarksettings"
},
"Exclude fbx_tests": {
"pattern": ".*\\\\/fbx_tests\\\\/assets\\\\/.*"
"pattern": "(^|.+/)fbx_tests/assets(/.+)$"
},
"Exclude wwise_bank_dependency_tests": {
"pattern": ".*\\\\/wwise_bank_dependency_tests\\\\/assets\\\\/.*"
"pattern": "(^|.+/)wwise_bank_dependency_tests/assets(/.+)$"
},
"Exclude AssetProcessorTestAssets": {
"pattern": ".*\\\\/asset_processor_tests\\\\/assets\\\\/.*"
"pattern": "(^|.+/)asset_processor_tests/assets(/.+)$"
},
"Exclude Restricted AssetProcessorTestAssets": {
"pattern": ".*\\\\/asset_processor_tests\\\\/restricted\\\\/.*"
"pattern": "(^|.+/)asset_processor_tests/restricted(/.+)$"
}
}
}
Code/Tools/AssetProcessor/native/tests/AssetCatalog/AssetCatalogUnitTests.cpp
+1 -1
View File
@@ -281,7 +281,7 @@ namespace AssetProcessor
ExcludeAssetRecognizer excludeRecogniser;
excludeRecogniser.m_name = "backup";
excludeRecogniser.m_patternMatcher = AssetBuilderSDK::FilePatternMatcher(".*\\/savebackup\\/.*", AssetBuilderSDK::AssetBuilderPattern::Regex);
excludeRecogniser.m_patternMatcher = AssetBuilderSDK::FilePatternMatcher("(^|.+/)savebackup/.*", AssetBuilderSDK::AssetBuilderPattern::Regex);
config.AddExcludeRecognizer(excludeRecogniser);
}
Code/Tools/AssetProcessor/native/tests/assetscanner/AssetScannerTests.cpp
+2 -2
View File
@@ -123,7 +123,7 @@ namespace AssetProcessor
ExcludeAssetRecognizer excludeRecogniser;
excludeRecogniser.m_name = "backup";
// we are excluding all the files in the folder but not the folder itself
excludeRecogniser.m_patternMatcher = AssetBuilderSDK::FilePatternMatcher(".*\\/subfolder2\\/aaa\\/.*", AssetBuilderSDK::AssetBuilderPattern::Regex);
excludeRecogniser.m_patternMatcher = AssetBuilderSDK::FilePatternMatcher("(^|[^/]+/)aaa/.*", AssetBuilderSDK::AssetBuilderPattern::Regex);
m_platformConfig.get()->AddExcludeRecognizer(excludeRecogniser);
m_assetScanner.get()->StartScan();
@@ -144,7 +144,7 @@ namespace AssetProcessor
ExcludeAssetRecognizer excludeRecogniser;
excludeRecogniser.m_name = "backup";
// we are excluding the complete folder here
excludeRecogniser.m_patternMatcher = AssetBuilderSDK::FilePatternMatcher(".*\\/subfolder2\\/aaa", AssetBuilderSDK::AssetBuilderPattern::Regex);
excludeRecogniser.m_patternMatcher = AssetBuilderSDK::FilePatternMatcher("(^|[^/]+/)aaa", AssetBuilderSDK::AssetBuilderPattern::Regex);
m_platformConfig.get()->AddExcludeRecognizer(excludeRecogniser);
m_assetScanner.get()->StartScan();
Code/Tools/AssetProcessor/native/tests/platformconfiguration/platformconfigurationtests.cpp
+2
View File
@@ -405,6 +405,8 @@ TEST_F(PlatformConfigurationUnitTests, TestFailReadConfigFile_RegularExcludes)
auto configRoot = AZ::IO::FileIOBase::GetInstance()->ResolvePath("@exefolder@/testdata/config_regular");
ASSERT_TRUE(configRoot);
UnitTestPlatformConfiguration config;
config.AddScanFolder(ScanFolderInfo("blahblah", "Blah ScanFolder", "sf2", true, true), true);
m_absorber.Clear();
ASSERT_TRUE(config.InitializeFromConfigFiles(configRoot->c_str(), testExeFolder->c_str(), EmptyDummyProjectName, false, false));
ASSERT_EQ(m_absorber.m_numErrorsAbsorbed, 0);
Code/Tools/AssetProcessor/native/unittests/AssetProcessorManagerUnitTests.cpp
+2 -7
View File
@@ -347,7 +347,7 @@ namespace AssetProcessor
ExcludeAssetRecognizer excludeRecogniser;
excludeRecogniser.m_name = "backup";
excludeRecogniser.m_patternMatcher = AssetBuilderSDK::FilePatternMatcher(".*\\/savebackup\\/.*", AssetBuilderSDK::AssetBuilderPattern::Regex);
excludeRecogniser.m_patternMatcher = AssetBuilderSDK::FilePatternMatcher("(^|.+/)savebackup/.*", AssetBuilderSDK::AssetBuilderPattern::Regex);
config.AddExcludeRecognizer(excludeRecogniser);
AssetProcessorManager_Test apm(&config); // note, this will 'push' the scan folders in to the db.
@@ -1791,13 +1791,8 @@ namespace AssetProcessor
UNIT_TEST_EXPECT_TRUE((newfingerprintForPCAfterVersionChange != fingerprintForPC) || (newfingerprintForPCAfterVersionChange != newfingerprintForPC));//Fingerprints should be different
UNIT_TEST_EXPECT_TRUE((newfingerprintForANDROIDAfterVersionChange != fingerprintForANDROID) || (newfingerprintForANDROIDAfterVersionChange != newfingerprintForANDROID));//Fingerprints should be different
//------Test for Files which are excluded
processResults.clear();
absolutePath = AssetUtilities::NormalizeFilePath(tempPath.absoluteFilePath("subfolder3/savebackup/test.txt"));
QMetaObject::invokeMethod(&apm, "AssessModifiedFile", Qt::QueuedConnection, Q_ARG(QString, absolutePath));
UNIT_TEST_EXPECT_FALSE(BlockUntil(idling, 3000)); //Processing a file that will be excluded should not cause assetprocessor manager to emit the onBecameIdle signal because its state should not change
UNIT_TEST_EXPECT_TRUE(processResults.size() == 0);
// ------------- Test querying asset status -------------------
{
absolutePath = tempPath.absoluteFilePath("subfolder2/folder/ship.tiff");
Code/Tools/AssetProcessor/native/utilities/PlatformConfiguration.cpp
+8 -3
View File
@@ -1633,13 +1633,18 @@ namespace AssetProcessor
bool AssetProcessor::PlatformConfiguration::IsFileExcluded(QString fileName) const
{
for (const ExcludeAssetRecognizer& excludeRecognizer : m_excludeAssetRecognizers)
QString relPath, scanFolderName;
if (ConvertToRelativePath(fileName, relPath, scanFolderName))
{
if (excludeRecognizer.m_patternMatcher.MatchesPath(fileName.toUtf8().constData()))
for (const ExcludeAssetRecognizer& excludeRecognizer : m_excludeAssetRecognizers)
{
return true;
if (excludeRecognizer.m_patternMatcher.MatchesPath(relPath.toUtf8().constData()))
{
return true;
}
}
}
return false;
}
Code/Tools/AssetProcessor/testdata/config_regular/AssetProcessorPlatformConfig.setreg
Vendored
+2 -2
View File
@@ -50,10 +50,10 @@
"order": 6000
},
"Exclude HoldFiles": {
"pattern": ".*\\\\/Levels\\\\/.*_hold\\\\/.*"
"pattern": "(^|.+/)Levels/.*_hold(/.*)?$"
},
"Exclude TempFiles": {
"pattern": ".*\\\\/\\\\$tmp[0-9]*_.*"
"pattern": "(^|.+/)\\\\$tmp[0-9]*_.*"
},
"RC i_caf": {
"glob": "*.i_caf",
Code/Tools/AssetProcessor/testdata/config_regular_platform_scanfolder/AssetProcessorPlatformConfig.setreg
Vendored
+2 -2
View File
@@ -74,10 +74,10 @@
"include": "test"
},
"Exclude HoldFiles": {
"pattern": ".*\\\\/Levels\\\\/.*_hold\\\\/.*"
"pattern": "(^|.+/)Levels/.*_hold(/.*)?$"
},
"Exclude TempFiles": {
"pattern": ".*\\\\/\\\\$tmp[0-9]*_.*"
"pattern": "(^|.+/)\\\\$tmp[0-9]*_.*"
},
"RC i_caf": {
"glob": "*.i_caf",
Gems/AtomContent/Sponza/Registry/AssetProcessorPlatformConfig.setreg
+1 -1
View File
@@ -6,7 +6,7 @@
// Sample Gems, Block source folders
// ------------------------------------------------------------------------------
"Exclude Work In Progress Folders": {
"pattern": ".*\\\\/.[Ss]rc\\\\/.*"
"pattern": "(^|.+/).[Ss]rc(/.*)?$"
}
}
}
Gems/AudioEngineWwise/AssetProcessorGemConfig.setreg
+2 -2
View File
@@ -3,10 +3,10 @@
"AssetProcessor": {
"Settings": {
"Exclude AudioProject": {
"pattern": ".*\\\\/Sounds\\\\/.+_project.*"
"pattern": "(^|.+/)Sounds/.+_project.*"
},
"Exclude AudioTemp": {
"pattern": ".*\\\\/Sounds\\\\/.+\\\\.(txt|xml|dat)"
"pattern": "(^|.+/)Sounds/.+\\\\.(txt|xml|dat)"
},
"RC audio": {
"pattern": ".*\\\\.(wav|pcm)",
Registry/AssetProcessorPlatformConfig.setreg
+20 -18
View File
@@ -137,64 +137,66 @@
// Excludes files that match the pattern or glob
// if you use a pattern, remember to escape your backslashes (\\)
// The patterns are checked against a path relative to the entry's
// root scan folder.
"Exclude _LevelBackups": {
"pattern": ".*\\\\/Levels\\\\/.*\\\\/_savebackup\\\\/.*"
"pattern": "(^|.+/)Levels/.*/_savebackup(/.*)?$"
},
"Exclude _LevelAutoBackups": {
"pattern": ".*\\\\/Levels\\\\/.*\\\\/_autobackup\\\\/.*"
"pattern": "(^|.+/)Levels/.*/_autobackup(/.*)?$"
},
"Exclude HoldFiles": {
"pattern": ".*\\\\/Levels\\\\/.*_hold\\\\/.*"
"pattern": "(^|.+/)Levels/.*_hold(/.*)?$"
},
// note that $ has meaning to regex, so we escape it.
"Exclude TempFiles": {
"pattern": ".*\\\\/\\\\$tmp[0-9]*_.*"
"pattern": "(^|.+/)\\\\$tmp[0-9]*_.*"
},
"Exclude TmpAnimationCompression": {
"pattern": ".*\\\\/Editor\\\\/Tmp\\\\/AnimationCompression\\\\/.*"
"pattern": "(^|.+/)Editor/Tmp/AnimationCompression(/.*)?$"
},
"Exclude EventLog": {
"pattern": ".*\\\\/Editor\\\\/.*eventlog\\\\.xml"
"pattern": "(^|.+/)Editor/.*eventlog\\\\.xml"
},
"Exclude GameGemsCode": {
"pattern": ".*\\\\/Gem\\\\/Code\\\\/.*"
"pattern": "(^|.+/)Gem/Code(/.*)?$"
},
"Exclude GameGemsResources": {
"pattern": ".*\\\\/Gem\\\\/Resources\\\\/.*"
"pattern": "(^|.+/)Gem/Resources(/.*)?$"
},
"Exclude Private Certs": {
"pattern": ".*\\DynamicContent\\\\/Certificates\\\\/Private\\\\/.*"
"pattern": "(^|.+/)DynamicContent/Certificates/Private(/.*)?$"
},
"Exclude CMakeLists": {
"pattern": ".*\\\\/CMakeLists.txt"
"pattern": "(^|.+/)CMakeLists\\\\.txt"
},
"Exclude CMakeFiles": {
"pattern": ".*\\\\/.*\\\\.cmake"
"pattern": "(^|.+/).+\\\\.cmake"
},
"Exclude User": {
"pattern": ".*/[Uu]ser/.*"
"pattern": "^[Uu]ser(/.*)?$"
},
"Exclude Build": {
"pattern": ".*/[Bb]uild/.*"
"pattern": "^[Bb]uild(/.*)?$"
},
"Exclude Install": {
"pattern": ".*/[Ii]nstall/.*"
"pattern": "^[Ii]nstall(/.*)?$"
},
"Exclude UserSettings": {
"pattern": ".*/UserSettings.xml"
"pattern": "(^|[^/]+/)UserSettings\\\\.xml"
},
// ------------------------------------------------------------------------------
// Large Worlds Test
// ------------------------------------------------------------------------------
"Exclude Work In Progress Folders": {
"pattern": ".*\\\\/WIP\\\\/.*"
"pattern": "(^|[^/]+/)WIP(/.*)?"
},
"Exclude Content Source Folders": {
"pattern": ".*\\\\/CONTENT_SOURCE\\\\/.*"
"pattern": "(^|[^/]+/)CONTENT_SOURCE(/.*)?"
},
"Exclude Art Source Folders": {
"pattern": ".*\\\\/ArtSource\\\\/.*"
"pattern": "(^|[^/]+/)ArtSource(/.*)?"
},
//------------------------------------------------------------------------------
// Copying Files Automatically Into the Cache