birkeh/o3de
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

ATOM-13512 Altering IBL ImageProcessor .preset Does Not Force Assets to Rebuild (#5639)

Browse Source 
Changes include:
- Move config files from ImageProcessingAtom/Config/ folder to ImageProcessingAtom/Assets/Config/ folder so it can be watched as source depencies.
- Change GetSuggestedPreset function so it can return certain preset for certain file name.
- Move file mask mappings from preset to ImageBuilder.settings. But the file masks in preset can still be used.
- Changed from using project config folder's imageBuilder.Settings for override to using it as json merge patch.
- Expose GetFileHash function in AssetBuilderSDK api.


Signed-off-by: Qing Tao <55564570+VickyAtAZ@users.noreply.github.com>
This commit is contained in:
Qing Tao
2021-11-16 15:25:13 -08:00
committed by GitHub
parent 6c1eefe605
commit 2ae8477683
55 changed files with 763 additions and 1118 deletions
Download Patch File Download Diff File Expand all files Collapse all files
Code/Tools/AssetProcessor/AssetBuilderSDK/AssetBuilderSDK/AssetBuilderSDK.cpp
+68
View File
@@ -23,6 +23,8 @@
#include <AzCore/RTTI/BehaviorContext.h>
//////////////////////////////////////////////////////////////////////////
#include <xxhash/xxhash.h>
namespace AssetBuilderSDK
{
const char* const ErrorWindow = "Error"; //Use this window name to log error messages.
@@ -1599,4 +1601,70 @@ namespace AssetBuilderSDK
{
return m_errorsOccurred;
}
AZ::u64 GetHashFromIOStream(AZ::IO::GenericStream& readStream, AZ::IO::SizeType* bytesReadOut, int hashMsDelay)
{
constexpr AZ::u64 HashBufferSize = 1024 * 64;
char buffer[HashBufferSize];
if(readStream.IsOpen() && readStream.CanRead())
{
AZ::IO::SizeType bytesRead;
auto* state = XXH64_createState();
if(state == nullptr)
{
AZ_Assert(false, "Failed to create hash state");
return 0;
}
if (XXH64_reset(state, 0) == XXH_ERROR)
{
AZ_Assert(false, "Failed to reset hash state");
return 0;
}
do
{
// In edge cases where another process is writing to this file while this hashing is occuring and that file wasn't locked,
// the following read check can fail because it performs an end of file check, and asserts and shuts down if the read size
// was smaller than the buffer and the read is not at the end of the file. The logic used to check end of file internal to read
// will be out of date in the edge cases where another process is actively writing to this file while this hash is running.
// The stream's length ends up more accurate in this case, preventing this assert and shut down.
// One area this occurs is the navigation mesh file (mnmnavmission0.bai) that's temporarily created when exporting a level,
// the navigation system can still be writing to this file when hashing begins, causing the EoF marker to change.
AZ::IO::SizeType remainingToRead = AZStd::min(readStream.GetLength() - readStream.GetCurPos(), aznumeric_cast<AZ::IO::SizeType>(AZ_ARRAY_SIZE(buffer)));
bytesRead = readStream.Read(remainingToRead, buffer);
if(bytesReadOut)
{
*bytesReadOut += bytesRead;
}
XXH64_update(state, buffer, bytesRead);
// Used by unit tests to force the race condition mentioned above, to verify the crash fix.
if(hashMsDelay > 0)
{
AZStd::this_thread::sleep_for(AZStd::chrono::milliseconds(hashMsDelay));
}
} while (bytesRead > 0);
auto hash = XXH64_digest(state);
XXH64_freeState(state);
return hash;
}
return 0;
}
AZ::u64 GetFileHash(const char* filePath, AZ::IO::SizeType* bytesReadOut, int hashMsDelay)
{
constexpr bool ErrorOnReadFailure = true;
AZ::IO::FileIOStream readStream(filePath, AZ::IO::OpenMode::ModeRead | AZ::IO::OpenMode::ModeBinary, ErrorOnReadFailure);
return GetHashFromIOStream(readStream, bytesReadOut, hashMsDelay);
}
}
Code/Tools/AssetProcessor/AssetBuilderSDK/AssetBuilderSDK/AssetBuilderSDK.h
+13
View File
@@ -910,6 +910,19 @@ namespace AssetBuilderSDK
//! There can be multiple builders running at once, so we need to filter out ones coming from other builders
AZStd::thread_id m_jobThreadId;
};
//! Get hash for a whole file
//! @filePath the path for the file
//! @bytesReadOut output the read file size in bytes
//! @hashMsDelay [Do not use except for unit test] add a delay in ms for between each block reading.
AZ::u64 GetFileHash(const char* filePath, AZ::IO::SizeType* bytesReadOut = nullptr, int hashMsDelay = 0);
//! Get hash for a generic IO stream
//! @readStream the input readable stream
//! @bytesReadOut output the read size in bytes
//! @hashMsDelay [Do not use except for unit test] add a delay in ms for between each block reading.
AZ::u64 GetHashFromIOStream(AZ::IO::GenericStream& readStream, AZ::IO::SizeType* bytesReadOut = nullptr, int hashMsDelay = 0);
} // namespace AssetBuilderSDK
namespace AZ
Code/Tools/AssetProcessor/AssetBuilderSDK/CMakeLists.txt
+1
View File
@@ -32,6 +32,7 @@ ly_add_target(
PUBLIC
AZ::AzFramework
AZ::AzToolsFramework
3rdParty::xxhash
)
ly_add_source_properties(
SOURCES AssetBuilderSDK/AssetBuilderSDK.cpp
Code/Tools/AssetProcessor/native/utilities/assetUtils.cpp
+4 -60
View File
@@ -1161,7 +1161,7 @@ namespace AssetUtilities
{
#ifndef AZ_TESTS_ENABLED
// Only used for unit tests, speed is critical for GetFileHash.
AZ_UNUSED(hashMsDelay);
hashMsDelay = 0;
#endif
bool useFileHashing = ShouldUseFileHashing();
@@ -1170,10 +1170,10 @@ namespace AssetUtilities
return 0;
}
AZ::u64 hash = 0;
if(!force)
{
auto* fileStateInterface = AZ::Interface<AssetProcessor::IFileStateRequests>::Get();
AZ::u64 hash = 0;
if (fileStateInterface && fileStateInterface->GetHash(filePath, &hash))
{
@@ -1181,64 +1181,8 @@ namespace AssetUtilities
}
}
char buffer[FileHashBufferSize];
constexpr bool ErrorOnReadFailure = true;
AZ::IO::FileIOStream readStream(filePath, AZ::IO::OpenMode::ModeRead | AZ::IO::OpenMode::ModeBinary, ErrorOnReadFailure);
if(readStream.IsOpen() && readStream.CanRead())
{
AZ::IO::SizeType bytesRead;
auto* state = XXH64_createState();
if(state == nullptr)
{
AZ_Assert(false, "Failed to create hash state");
return 0;
}
if (XXH64_reset(state, 0) == XXH_ERROR)
{
AZ_Assert(false, "Failed to reset hash state");
return 0;
}
do
{
// In edge cases where another process is writing to this file while this hashing is occuring and that file wasn't locked,
// the following read check can fail because it performs an end of file check, and asserts and shuts down if the read size
// was smaller than the buffer and the read is not at the end of the file. The logic used to check end of file internal to read
// will be out of date in the edge cases where another process is actively writing to this file while this hash is running.
// The stream's length ends up more accurate in this case, preventing this assert and shut down.
// One area this occurs is the navigation mesh file (mnmnavmission0.bai) that's temporarily created when exporting a level,
// the navigation system can still be writing to this file when hashing begins, causing the EoF marker to change.
AZ::IO::SizeType remainingToRead = AZStd::min(readStream.GetLength() - readStream.GetCurPos(), aznumeric_cast<AZ::IO::SizeType>(AZ_ARRAY_SIZE(buffer)));
bytesRead = readStream.Read(remainingToRead, buffer);
if(bytesReadOut)
{
*bytesReadOut += bytesRead;
}
XXH64_update(state, buffer, bytesRead);
#ifdef AZ_TESTS_ENABLED
// Used by unit tests to force the race condition mentioned above, to verify the crash fix.
if(hashMsDelay > 0)
{
AZStd::this_thread::sleep_for(AZStd::chrono::milliseconds(hashMsDelay));
}
#endif
} while (bytesRead > 0);
auto hash = XXH64_digest(state);
XXH64_freeState(state);
return hash;
}
return 0;
hash = AssetBuilderSDK::GetFileHash(filePath, bytesReadOut, hashMsDelay);
return hash;
}
AZ::u64 AdjustTimestamp(QDateTime timestamp)