birkeh/o3de
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Merging last dev

Browse Source 
Signed-off-by: Gene Walters <genewalt@amazon.com>
This commit is contained in:
Gene Walters
2021-11-19 23:12:59 -08:00
parent e1ace4a8f6 7100d48e16
commit 6005a4e08b
2657 changed files with 249197 additions and 30535 deletions
Download Patch File Download Diff File Expand all files Collapse all files
Code/Tools/AssetProcessor/native/utilities/ApplicationManagerBase.cpp
+2
View File
@@ -454,6 +454,8 @@ void ApplicationManagerBase::InitFileMonitor()
QObject::connect(newFolderWatch, &FolderWatchCallbackEx::fileModified, [this](QString path) { m_fileStateCache->UpdateFile(path); });
QObject::connect(newFolderWatch, &FolderWatchCallbackEx::fileRemoved, [this](QString path) { m_fileStateCache->RemoveFile(path); });
QObject::connect(newFolderWatch, &FolderWatchCallbackEx::fileAdded, [](QString path) { AZ::Interface<AssetProcessor::ExcludedFolderCacheInterface>::Get()->FileAdded(path); });
QObject::connect(newFolderWatch, &FolderWatchCallbackEx::fileAdded,
m_fileProcessor.get(), &AssetProcessor::FileProcessor::AssessAddedFile);
QObject::connect(newFolderWatch, &FolderWatchCallbackEx::fileRemoved,
Code/Tools/AssetProcessor/native/utilities/PlatformConfiguration.cpp
+73 -2
View File
@@ -749,7 +749,7 @@ namespace AssetProcessor
}
AZStd::vector<AZ::IO::Path> configFiles = AzToolsFramework::AssetUtils::GetConfigFiles(absoluteSystemRoot.toUtf8().constData(),
absoluteAssetRoot.toUtf8().constData(), projectPath.toUtf8().constData(),
projectPath.toUtf8().constData(),
addPlatformConfigs, addGemsConfigs && !noGemScanFolders, settingsRegistry);
// First Merge all Engine, Gem and Project specific AssetProcessor*Config.setreg/.inifiles
@@ -1285,6 +1285,13 @@ namespace AssetProcessor
return m_scanFolders[index];
}
const AssetProcessor::ScanFolderInfo& PlatformConfiguration::GetScanFolderAt(int index) const
{
Q_ASSERT(index >= 0);
Q_ASSERT(index < m_scanFolders.size());
return m_scanFolders[index];
}
void PlatformConfiguration::AddScanFolder(const AssetProcessor::ScanFolderInfo& source, bool isUnitTesting)
{
if (isUnitTesting)
@@ -1436,7 +1443,10 @@ namespace AssetProcessor
}
QStringList PlatformConfiguration::FindWildcardMatches(
const QString& sourceFolder, QString relativeName, bool includeFolders, bool recursiveSearch) const
const QString& sourceFolder,
QString relativeName,
bool includeFolders,
bool recursiveSearch) const
{
if (relativeName.isEmpty())
{
@@ -1469,6 +1479,67 @@ namespace AssetProcessor
return returnList;
}
QStringList PlatformConfiguration::FindWildcardMatches(
const QString& sourceFolder,
QString relativeName,
const AZStd::unordered_set<AZStd::string>& excludedFolders,
bool includeFolders,
bool recursiveSearch) const
{
if (relativeName.isEmpty())
{
return QStringList();
}
QDir sourceFolderDir(sourceFolder);
QString posixRelativeName = QDir::fromNativeSeparators(relativeName);
QStringList returnList;
QRegExp nameMatch{ posixRelativeName, Qt::CaseInsensitive, QRegExp::Wildcard };
AZStd::stack<QString> dirs;
dirs.push(sourceFolderDir.absolutePath());
while (!dirs.empty())
{
QString absolutePath = dirs.top();
dirs.pop();
if (excludedFolders.contains(absolutePath.toUtf8().constData()))
{
continue;
}
QDirIterator dirIterator(absolutePath, QDir::AllEntries | QDir::NoSymLinks | QDir::NoDotAndDotDot);
while (dirIterator.hasNext())
{
dirIterator.next();
if (!dirIterator.fileInfo().isFile())
{
if (recursiveSearch)
{
dirs.push(dirIterator.filePath());
}
if (!includeFolders)
{
continue;
}
}
QString pathMatch{ sourceFolderDir.relativeFilePath(dirIterator.filePath()) };
if (nameMatch.exactMatch(pathMatch))
{
returnList.append(QDir::fromNativeSeparators(dirIterator.filePath()));
}
}
}
return returnList;
}
const AssetProcessor::ScanFolderInfo* PlatformConfiguration::GetScanFolderForFile(const QString& fullFileName) const
{
QString normalized = AssetUtilities::NormalizeFilePath(fullFileName);
Code/Tools/AssetProcessor/native/utilities/PlatformConfiguration.h
+13 -1
View File
@@ -256,6 +256,9 @@ namespace AssetProcessor
//! Retrieve the scan folder at a given index.
AssetProcessor::ScanFolderInfo& GetScanFolderAt(int index);
//! Retrieve the scan folder at a given index.
const AssetProcessor::ScanFolderInfo& GetScanFolderAt(int index) const;
//! Manually add a scan folder. Also used for testing.
void AddScanFolder(const AssetProcessor::ScanFolderInfo& source, bool isUnitTesting = false);
@@ -298,7 +301,16 @@ namespace AssetProcessor
QString FindFirstMatchingFile(QString relativeName) const;
//! given a relative name with wildcard characters (* allowed) find a set of matching files or optionally folders
QStringList FindWildcardMatches(const QString& sourceFolder, QString relativeName, bool includeFolders = false, bool recursiveSearch = true) const;
QStringList FindWildcardMatches(const QString& sourceFolder, QString relativeName, bool includeFolders = false,
bool recursiveSearch = true) const;
//! given a relative name with wildcard characters (* allowed) find a set of matching files or optionally folders
QStringList FindWildcardMatches(
const QString& sourceFolder,
QString relativeName,
const AZStd::unordered_set<AZStd::string>& excludedFolders,
bool includeFolders = false,
bool recursiveSearch = true) const;
//! given a fileName (as a full path), return the database source name which includes the output prefix.
//!
Code/Tools/AssetProcessor/native/utilities/assetUtils.cpp
+4 -60
View File
@@ -1161,7 +1161,7 @@ namespace AssetUtilities
{
#ifndef AZ_TESTS_ENABLED
// Only used for unit tests, speed is critical for GetFileHash.
AZ_UNUSED(hashMsDelay);
hashMsDelay = 0;
#endif
bool useFileHashing = ShouldUseFileHashing();
@@ -1170,10 +1170,10 @@ namespace AssetUtilities
return 0;
}
AZ::u64 hash = 0;
if(!force)
{
auto* fileStateInterface = AZ::Interface<AssetProcessor::IFileStateRequests>::Get();
AZ::u64 hash = 0;
if (fileStateInterface && fileStateInterface->GetHash(filePath, &hash))
{
@@ -1181,64 +1181,8 @@ namespace AssetUtilities
}
}
char buffer[FileHashBufferSize];
constexpr bool ErrorOnReadFailure = true;
AZ::IO::FileIOStream readStream(filePath, AZ::IO::OpenMode::ModeRead | AZ::IO::OpenMode::ModeBinary, ErrorOnReadFailure);
if(readStream.IsOpen() && readStream.CanRead())
{
AZ::IO::SizeType bytesRead;
auto* state = XXH64_createState();
if(state == nullptr)
{
AZ_Assert(false, "Failed to create hash state");
return 0;
}
if (XXH64_reset(state, 0) == XXH_ERROR)
{
AZ_Assert(false, "Failed to reset hash state");
return 0;
}
do
{
// In edge cases where another process is writing to this file while this hashing is occuring and that file wasn't locked,
// the following read check can fail because it performs an end of file check, and asserts and shuts down if the read size
// was smaller than the buffer and the read is not at the end of the file. The logic used to check end of file internal to read
// will be out of date in the edge cases where another process is actively writing to this file while this hash is running.
// The stream's length ends up more accurate in this case, preventing this assert and shut down.
// One area this occurs is the navigation mesh file (mnmnavmission0.bai) that's temporarily created when exporting a level,
// the navigation system can still be writing to this file when hashing begins, causing the EoF marker to change.
AZ::IO::SizeType remainingToRead = AZStd::min(readStream.GetLength() - readStream.GetCurPos(), aznumeric_cast<AZ::IO::SizeType>(AZ_ARRAY_SIZE(buffer)));
bytesRead = readStream.Read(remainingToRead, buffer);
if(bytesReadOut)
{
*bytesReadOut += bytesRead;
}
XXH64_update(state, buffer, bytesRead);
#ifdef AZ_TESTS_ENABLED
// Used by unit tests to force the race condition mentioned above, to verify the crash fix.
if(hashMsDelay > 0)
{
AZStd::this_thread::sleep_for(AZStd::chrono::milliseconds(hashMsDelay));
}
#endif
} while (bytesRead > 0);
auto hash = XXH64_digest(state);
XXH64_freeState(state);
return hash;
}
return 0;
hash = AssetBuilderSDK::GetFileHash(filePath, bytesReadOut, hashMsDelay);
return hash;
}
AZ::u64 AdjustTimestamp(QDateTime timestamp)
Code/Tools/AssetProcessor/native/utilities/assetUtils.h
-1
View File
@@ -238,7 +238,6 @@ namespace AssetUtilities
// hashMsDelay is only for automated tests to test that writing to a file while it's hashing does not cause a crash.
// hashMsDelay is not used in non-unit test builds.
AZ::u64 GetFileHash(const char* filePath, bool force = false, AZ::IO::SizeType* bytesReadOut = nullptr, int hashMsDelay = 0);
inline constexpr AZ::u64 FileHashBufferSize = 1024 * 64;
//! Adjusts a timestamp to fix timezone settings and account for any precision adjustment needed
AZ::u64 AdjustTimestamp(QDateTime timestamp);