ATOM-13512 Altering IBL ImageProcessor .preset Does Not Force Assets to Rebuild (#5639)

Changes include:
- Move config files from ImageProcessingAtom/Config/ folder to ImageProcessingAtom/Assets/Config/ folder so it can be watched as source depencies.
- Change GetSuggestedPreset function so it can return certain preset for certain file name.
- Move file mask mappings from preset to ImageBuilder.settings. But the file masks in preset can still be used.
- Changed from using project config folder's imageBuilder.Settings for override to using it as json merge patch.
- Expose GetFileHash function in AssetBuilderSDK api.


Signed-off-by: Qing Tao <55564570+VickyAtAZ@users.noreply.github.com>
This commit is contained in:
Qing Tao
2021-11-16 15:25:13 -08:00
committed by GitHub
parent 6c1eefe605
commit 2ae8477683
55 changed files with 763 additions and 1118 deletions
@@ -8,6 +8,7 @@
#include "BuilderSettingManager.h"
#include <QCoreApplication>
#include <QDirIterator>
#include <QFile>
#include <QFileInfo>
@@ -17,8 +18,9 @@
#include <BuilderSettings/CubemapSettings.h>
#include <BuilderSettings/TextureSettings.h>
#include <Converters/Cubemap.h>
#include <Processing/PixelFormatInfo.h>
#include <Processing/ImageToProcess.h>
#include <Processing/PixelFormatInfo.h>
#include <Processing/Utils.h>
#include <ImageLoader/ImageLoaders.h>
#include <ImageProcessing_Traits_Platform.h>
@@ -41,13 +43,18 @@
namespace ImageProcessingAtom
{
const char* BuilderSettingManager::s_defaultConfigRelativeFolder = "Gems/Atom/Asset/ImageProcessingAtom/Config/";
const char* BuilderSettingManager::s_defaultConfigRelativeFolder = "Gems/Atom/Asset/ImageProcessingAtom/Assets/Config/";
const char* BuilderSettingManager::s_projectConfigRelativeFolder = "Config/AtomImageBuilder/";
const char* BuilderSettingManager::s_builderSettingFileName = "ImageBuilder.settings";
const char* BuilderSettingManager::s_presetFileExtension = ".preset";
const char* BuilderSettingManager::s_presetFileExtension = "preset";
const char FileMaskDelimiter = '_';
namespace
{
static constexpr const char* const LogWindow = "Image Processing";
}
#if defined(AZ_TOOLS_EXPAND_FOR_RESTRICTED_PLATFORMS)
#define AZ_RESTRICTED_PLATFORM_EXPANSION(CodeName, CODENAME, codename, PrivateName, PRIVATENAME, privatename, PublicName, PUBLICNAME, publicname, PublicAuxName1, PublicAuxName2, PublicAuxName3) \
namespace ImageProcess##PrivateName \
@@ -69,13 +76,15 @@ namespace ImageProcessingAtom
if (serialize)
{
serialize->Class<BuilderSettingManager>()
->Version(1)
->Field("AnalysisFingerprint", &BuilderSettingManager::m_analysisFingerprint)
->Version(2)
->Field("BuildSettings", &BuilderSettingManager::m_builderSettings)
->Field("DefaultPresetsByFileMask", &BuilderSettingManager::m_defaultPresetByFileMask)
->Field("PresetsByFileMask", &BuilderSettingManager::m_presetFilterMap)
->Field("DefaultPreset", &BuilderSettingManager::m_defaultPreset)
->Field("DefaultPresetAlpha", &BuilderSettingManager::m_defaultPresetAlpha)
->Field("DefaultPresetNonePOT", &BuilderSettingManager::m_defaultPresetNonePOT);
->Field("DefaultPresetNonePOT", &BuilderSettingManager::m_defaultPresetNonePOT)
// deprecated properties
->Field("DefaultPresetsByFileMask", &BuilderSettingManager::m_defaultPresetByFileMask)
->Field("AnalysisFingerprint", &BuilderSettingManager::m_analysisFingerprint);
}
}
@@ -122,7 +131,7 @@ namespace ImageProcessingAtom
s_globalInstance.Reset();
}
const PresetSettings* BuilderSettingManager::GetPreset(const PresetName& presetName, const PlatformName& platform, AZStd::string_view* settingsFilePathOut)
const PresetSettings* BuilderSettingManager::GetPreset(const PresetName& presetName, const PlatformName& platform, AZStd::string_view* settingsFilePathOut) const
{
AZStd::lock_guard<AZStd::recursive_mutex> lock(m_presetMapLock);
auto itr = m_presets.find(presetName);
@@ -137,16 +146,17 @@ namespace ImageProcessingAtom
return nullptr;
}
const BuilderSettings* BuilderSettingManager::GetBuilderSetting(const PlatformName& platform)
const BuilderSettings* BuilderSettingManager::GetBuilderSetting(const PlatformName& platform) const
{
if (m_builderSettings.find(platform) != m_builderSettings.end())
auto itr = m_builderSettings.find(platform);
if (itr != m_builderSettings.end())
{
return &m_builderSettings[platform];
return &itr->second;
}
return nullptr;
}
const PlatformNameList BuilderSettingManager::GetPlatformList()
const PlatformNameList BuilderSettingManager::GetPlatformList() const
{
PlatformNameList platforms;
@@ -161,7 +171,7 @@ namespace ImageProcessingAtom
return platforms;
}
const AZStd::map <FileMask, AZStd::unordered_set<PresetName>>& BuilderSettingManager::GetPresetFilterMap()
const AZStd::map <FileMask, AZStd::unordered_set<PresetName>>& BuilderSettingManager::GetPresetFilterMap() const
{
AZStd::lock_guard<AZStd::recursive_mutex> lock(m_presetMapLock);
return m_presetFilterMap;
@@ -188,7 +198,6 @@ namespace ImageProcessingAtom
m_presetFilterMap.clear();
m_builderSettings.clear();
m_presets.clear();
m_defaultPresetByFileMask.clear();
}
StringOutcome BuilderSettingManager::LoadConfig()
@@ -198,44 +207,53 @@ namespace ImageProcessingAtom
auto fileIoBase = AZ::IO::FileIOBase::GetInstance();
if (fileIoBase == nullptr)
{
return AZ::Failure(AZStd::string("File IO instance needs to be initialized to resolve ImageProcessing builder file aliases"));
return AZ::Failure(
AZStd::string("File IO instance needs to be initialized to resolve ImageProcessing builder file aliases"));
}
// Construct the default setting path
AZ::IO::FixedMaxPath defaultConfigFolder;
if (auto engineRoot = fileIoBase->ResolvePath("@engroot@"); engineRoot.has_value())
{
defaultConfigFolder = *engineRoot;
defaultConfigFolder /= s_defaultConfigRelativeFolder;
m_defaultConfigFolder = *engineRoot;
m_defaultConfigFolder /= s_defaultConfigRelativeFolder;
}
AZ::IO::FixedMaxPath projectConfigFolder;
if (auto sourceGameRoot = fileIoBase->ResolvePath("@projectroot@"); sourceGameRoot.has_value())
{
projectConfigFolder = *sourceGameRoot;
projectConfigFolder /= s_projectConfigRelativeFolder;
m_projectConfigFolder = *sourceGameRoot;
m_projectConfigFolder /= s_projectConfigRelativeFolder;
}
AZStd::lock_guard<AZStd::recursive_mutex> lock(m_presetMapLock);
ClearSettings();
outcome = LoadSettings((projectConfigFolder / s_builderSettingFileName).Native());
if (!outcome.IsSuccess())
{
outcome = LoadSettings((defaultConfigFolder / s_builderSettingFileName).Native());
}
outcome = LoadSettings();
if (outcome.IsSuccess())
{
// Load presets in default folder first, then load from project folder.
// The same presets which loaded last will overwrite previous loaded one.
LoadPresets(defaultConfigFolder.Native());
LoadPresets(projectConfigFolder.Native());
LoadPresets(m_defaultConfigFolder.Native());
LoadPresets(m_projectConfigFolder.Native());
}
// Regenerate file mask mapping after all presets loaded
RegenerateMappings();
// Collect extra file masks from preset files
CollectFileMasksFromPresets();
if (QCoreApplication::instance())
{
m_fileWatcher.reset(new QFileSystemWatcher);
// track preset files
// Note, the QT signal would only works for AP but not AssetBuilder
// We use file time stamp to track preset file change in builder's CreateJob
for (auto& preset : m_presets)
{
m_fileWatcher.data()->addPath(QString(preset.second.m_presetFilePath.c_str()));
}
m_fileWatcher.data()->addPath(QString(m_defaultConfigFolder.c_str()));
m_fileWatcher.data()->addPath(QString(m_projectConfigFolder.c_str()));
QObject::connect(m_fileWatcher.data(), &QFileSystemWatcher::fileChanged, this, &BuilderSettingManager::OnFileChanged);
QObject::connect(m_fileWatcher.data(), &QFileSystemWatcher::directoryChanged, this, &BuilderSettingManager::OnFolderChanged);
}
return outcome;
@@ -243,36 +261,84 @@ namespace ImageProcessingAtom
void BuilderSettingManager::LoadPresets(AZStd::string_view presetFolder)
{
AZStd::lock_guard<AZStd::recursive_mutex> lock(m_presetMapLock);
QDirIterator it(presetFolder.data(), QStringList() << "*.preset", QDir::Files, QDirIterator::NoIteratorFlags);
while (it.hasNext())
{
QString filePath = it.next();
QFileInfo fileInfo = it.fileInfo();
LoadPreset(filePath.toUtf8().data());
}
}
MultiplatformPresetSettings preset;
auto result = AZ::JsonSerializationUtils::LoadObjectFromFile(preset, filePath.toUtf8().data());
if (!result.IsSuccess())
bool BuilderSettingManager::LoadPreset(const AZStd::string& filePath)
{
QFileInfo fileInfo (filePath.c_str());
if (!fileInfo.exists())
{
return false;
}
MultiplatformPresetSettings preset;
auto result = AZ::JsonSerializationUtils::LoadObjectFromFile(preset, filePath);
if (!result.IsSuccess())
{
AZ_Warning(LogWindow, false, "Failed to load preset file %s. Error: %s",
filePath.c_str(), result.GetError().c_str());
return false;
}
PresetName presetName(fileInfo.baseName().toUtf8().data());
AZ_Warning(LogWindow, presetName == preset.GetPresetName(), "Preset file name '%s' is not"
" same as preset name '%s'. Using preset file name as preset name",
filePath.c_str(), preset.GetPresetName().GetCStr());
preset.SetPresetName(presetName);
m_presets[presetName] = PresetEntry{preset, filePath.c_str(), fileInfo.lastModified()};
return true;
}
void BuilderSettingManager::ReloadPreset(const PresetName& presetName)
{
// Find the preset file from project or default config folder
AZStd::string presetFileName = AZStd::string::format("%s.%s", presetName.GetCStr(), s_presetFileExtension);
AZ::IO::FixedMaxPath filePath = m_projectConfigFolder/presetFileName;
QFileInfo fileInfo (filePath.c_str());
if (!fileInfo.exists())
{
filePath = (m_defaultConfigFolder/presetFileName).c_str();
fileInfo = QFileInfo(filePath.c_str());
}
AZStd::lock_guard<AZStd::recursive_mutex> lock(m_presetMapLock);
//Skip the loading if the file wasn't chagned
if (fileInfo.exists())
{
if (m_presets.find(presetName) != m_presets.end())
{
AZ_Warning("Image Processing", false, "Failed to load preset file %s. Error: %s",
filePath.toUtf8().data(), result.GetError().c_str());
if (m_presets[presetName].m_lastModifiedTime == fileInfo.lastModified()
&& m_presets[presetName].m_presetFilePath == filePath.c_str())
{
return;
}
}
}
PresetName presetName(fileInfo.baseName().toUtf8().data());
// remove preset
m_presets.erase(presetName);
AZ_Warning("Image Processing", presetName == preset.GetPresetName(), "Preset file name '%s' is not"
" same as preset name '%s'. Using preset file name as preset name",
filePath.toUtf8().data(), preset.GetPresetName().GetCStr());
preset.SetPresetName(presetName);
m_presets[presetName] = PresetEntry{preset, filePath.toUtf8().data()};
if (fileInfo.exists())
{
LoadPreset(filePath.c_str());
}
}
StringOutcome BuilderSettingManager::LoadConfigFromFolder(AZStd::string_view configFolder)
{
AZStd::lock_guard<AZStd::recursive_mutex> lock(m_presetMapLock);
// Load builder settings
AZStd::string settingFilePath = AZStd::string::format("%.*s%s", aznumeric_cast<int>(configFolder.size()),
configFolder.data(), s_builderSettingFileName);
@@ -282,12 +348,108 @@ namespace ImageProcessingAtom
if (result.IsSuccess())
{
LoadPresets(configFolder);
RegenerateMappings();
}
return result;
}
void BuilderSettingManager::ReportDeprecatedSettings()
{
// reported deprecated attributes in image builder settings
if (!m_analysisFingerprint.empty())
{
AZ_Warning(LogWindow, false, "'AnalysisFingerprint' is deprecated and it should be removed from file [%s]", s_builderSettingFileName);
}
if (!m_defaultPresetByFileMask.empty())
{
AZ_Warning(LogWindow, false, "'DefaultPresetsByFileMask' is deprecated and it should be removed from file [%s]. Use PresetsByFileMask instead", s_builderSettingFileName);
}
}
StringOutcome BuilderSettingManager::LoadSettings()
{
// If the project image build setting file exist, it will merge image builder settings from project folder to the settings from default config folder.
bool needMerge = false;
AZStd::string projectSettingFile{ (m_projectConfigFolder / s_builderSettingFileName).Native() };
if (AZ::IO::SystemFile::Exists(projectSettingFile.c_str()))
{
needMerge = true;
}
AZ::Outcome<void, AZStd::string> outcome;
AZStd::string defaultSettingFile{ (m_defaultConfigFolder / s_builderSettingFileName).Native() };
if (needMerge)
{
auto outcome1 = AZ::JsonSerializationUtils::ReadJsonFile(defaultSettingFile);
auto outcome2 = AZ::JsonSerializationUtils::ReadJsonFile(projectSettingFile);
// return error if it failed to load default settings
if (!outcome1.IsSuccess())
{
return STRING_OUTCOME_ERROR(outcome1.GetError());
}
// if project config was loaded successfully, apply merge patch
rapidjson::Document& originDoc = outcome1.GetValue();
if (outcome2.IsSuccess())
{
const rapidjson::Document& patchDoc = outcome2.GetValue();
AZ::JsonSerializationResult::ResultCode result =
AZ::JsonSerialization::ApplyPatch(originDoc, originDoc.GetAllocator(), patchDoc, AZ::JsonMergeApproach::JsonMergePatch);
if (result.GetProcessing() == AZ::JsonSerializationResult::Processing::Completed)
{
AZStd::vector<char> outBuffer;
AZ::IO::ByteContainerStream<AZStd::vector<char>> outStream{ &outBuffer };
AZ::JsonSerializationUtils::WriteJsonStream(originDoc, outStream);
outStream.Seek(0, AZ::IO::GenericStream::ST_SEEK_BEGIN);
outcome = AZ::JsonSerializationUtils::LoadObjectFromStream(*this, outStream);
if (!outcome.IsSuccess())
{
return STRING_OUTCOME_ERROR(outcome.GetError());
}
ReportDeprecatedSettings();
// Generate config file fingerprint
outStream.Seek(0, AZ::IO::GenericStream::ST_SEEK_BEGIN);
AZ::u64 hash = AssetBuilderSDK::GetHashFromIOStream(outStream);
m_analysisFingerprint = AZStd::string::format("%llX", hash);
}
else
{
needMerge = false;
AZ_Warning(LogWindow, false, "Failed to fully merge data into image builder settings. Skipping project build setting file [%s]", projectSettingFile.c_str());
}
}
else
{
AZ_Warning(LogWindow, false, "Failed to load project setting file [%s]. Skipping", projectSettingFile.c_str());
}
}
if (!needMerge)
{
outcome = AZ::JsonSerializationUtils::LoadObjectFromFile(*this, defaultSettingFile);
if (!outcome.IsSuccess())
{
return STRING_OUTCOME_ERROR(outcome.GetError());
}
ReportDeprecatedSettings();
// Generate config file fingerprint
AZ::u64 hash = AssetBuilderSDK::GetFileHash(defaultSettingFile.c_str());
m_analysisFingerprint = AZStd::string::format("%llX", hash);
}
return STRING_OUTCOME_SUCCESS;
}
StringOutcome BuilderSettingManager::LoadSettings(AZStd::string_view filepath)
{
AZStd::lock_guard<AZStd::recursive_mutex> lock(m_presetMapLock);
@@ -336,13 +498,13 @@ namespace ImageProcessingAtom
return m_analysisFingerprint;
}
void BuilderSettingManager::RegenerateMappings()
void BuilderSettingManager::CollectFileMasksFromPresets()
{
AZStd::lock_guard<AZStd::recursive_mutex> lock(m_presetMapLock);
AZStd::string noFilter = AZStd::string();
m_presetFilterMap.clear();
AZStd::string extraString;
for (const auto& presetIter : m_presets)
{
@@ -357,22 +519,31 @@ namespace ImageProcessingAtom
{
if (filemask.empty() || filemask[0] != FileMaskDelimiter)
{
AZ_Warning("Image Processing", false, "File mask '%s' is invalid. It must start with '%c'.", filemask.c_str(), FileMaskDelimiter);
AZ_Warning(LogWindow, false, "File mask '%s' is invalid. It must start with '%c'.", filemask.c_str(), FileMaskDelimiter);
continue;
}
else if (filemask.size() < 2)
{
AZ_Warning("Image Processing", false, "File mask '%s' is invalid. The '%c' must be followed by at least one other character.", filemask.c_str());
AZ_Warning(LogWindow, false, "File mask '%s' is invalid. The '%c' must be followed by at least one other character.", filemask.c_str());
continue;
}
else if (filemask.find(FileMaskDelimiter, 1) != AZStd::string::npos)
{
AZ_Warning("Image Processing", false, "File mask '%s' is invalid. It must contain only a single '%c' character.", filemask.c_str(), FileMaskDelimiter);
AZ_Warning(LogWindow, false, "File mask '%s' is invalid. It must contain only a single '%c' character.", filemask.c_str(), FileMaskDelimiter);
continue;
}
extraString += (filemask + preset.m_name.GetCStr());
m_presetFilterMap[filemask].insert(preset.m_name);
}
}
if (!extraString.empty())
{
AZ::u64 hash = AZStd::hash<AZStd::string>{}(extraString);
m_analysisFingerprint += AZStd::string::format("%llX", hash);
}
}
void BuilderSettingManager::MetafilePathFromImagePath(AZStd::string_view imagePath, AZStd::string& metafilePath)
@@ -419,38 +590,15 @@ namespace ImageProcessingAtom
return m_presets.find(presetName) != m_presets.end();
}
PresetName BuilderSettingManager::GetSuggestedPreset(AZStd::string_view imageFilePath, IImageObjectPtr imageFromFile)
PresetName BuilderSettingManager::GetSuggestedPreset(AZStd::string_view imageFilePath) const
{
PresetName emptyPreset;
//load the image to get its size for later use
IImageObjectPtr image = imageFromFile;
//if the input image is empty we will try to load it from the path
if (imageFromFile == nullptr)
{
image = IImageObjectPtr(LoadImageFromFile(imageFilePath));
}
if (image == nullptr)
{
return emptyPreset;
}
//get file mask of this image file
AZStd::string fileMask = GetFileMask(imageFilePath);
PresetName outPreset = emptyPreset;
//check default presets for some file masks
if (m_defaultPresetByFileMask.find(fileMask) != m_defaultPresetByFileMask.end())
{
outPreset = m_defaultPresetByFileMask[fileMask];
if (!IsValidPreset(outPreset))
{
outPreset = emptyPreset;
}
}
//use the preset filter map to find
if (outPreset.IsEmpty() && !fileMask.empty())
{
@@ -461,54 +609,21 @@ namespace ImageProcessingAtom
}
}
const PresetSettings* presetInfo = nullptr;
if (!outPreset.IsEmpty())
{
presetInfo = GetPreset(outPreset);
//special case for cubemap
if (presetInfo && presetInfo->m_cubemapSetting)
{
// If it's not a latitude-longitude map or it doesn't match any cubemap layouts then reset its preset
if (!IsValidLatLongMap(image) && CubemapLayout::GetCubemapLayoutInfo(image) == nullptr)
{
outPreset = emptyPreset;
}
}
}
if (outPreset == emptyPreset)
{
if (image->GetAlphaContent() == EAlphaContent::eAlphaContent_Absent)
{
outPreset = m_defaultPreset;
}
else
{
outPreset = m_defaultPresetAlpha;
}
outPreset = m_defaultPreset;
}
//get the pixel format for selected preset
presetInfo = GetPreset(outPreset);
return outPreset;
}
if (presetInfo)
{
//valid whether image size work with pixel format
if (CPixelFormats::GetInstance().IsImageSizeValid(presetInfo->m_pixelFormat,
image->GetWidth(0), image->GetHeight(0), false))
{
return outPreset;
}
else
{
AZ_Warning("Image Processing", false, "Image dimensions are not compatible with preset '%s'. The default preset will be used.", presetInfo->m_name.GetCStr());
}
}
//uncompressed one which could be used for almost everything
return m_defaultPresetNonePOT;
AZStd::vector<AZStd::string> BuilderSettingManager::GetPossiblePresetPaths(const PresetName& presetName) const
{
AZStd::vector<AZStd::string> paths;
AZStd::string presetFile = AZStd::string::format("%s.preset", presetName.GetCStr());
paths.push_back((m_defaultConfigFolder / presetFile).c_str());
paths.push_back((m_projectConfigFolder / presetFile).c_str());
return paths;
}
bool BuilderSettingManager::DoesSupportPlatform(AZStd::string_view platformId)
@@ -526,18 +641,50 @@ namespace ImageProcessingAtom
AZStd::string filePath;
if (!AzFramework::StringFunc::Path::Join(outputFolder.data(), fileName.c_str(), filePath))
{
AZ_Warning("Image Processing", false, "Failed to construct path with folder '%.*s' and file: '%s' to save preset",
AZ_Warning(LogWindow, false, "Failed to construct path with folder '%.*s' and file: '%s' to save preset",
aznumeric_cast<int>(outputFolder.size()), outputFolder.data(), filePath.c_str());
continue;
}
auto result = AZ::JsonSerializationUtils::SaveObjectToFile(&presetEntry.m_multiPreset, filePath);
if (!result.IsSuccess())
{
AZ_Warning("Image Processing", false, "Failed to save preset '%s' to file '%s'. Error: %s",
AZ_Warning(LogWindow, false, "Failed to save preset '%s' to file '%s'. Error: %s",
presetEntry.m_multiPreset.GetDefaultPreset().m_name.GetCStr(), filePath.c_str(), result.GetError().c_str());
}
}
}
void BuilderSettingManager::OnFileChanged(const QString &path)
{
// handles preset file change
// Note: this signal only works with AP but not AssetBuilder
AZ_TracePrintf(LogWindow, "File changed %s\n", path.toUtf8().data());
QFileInfo info(path);
// skip if the file is not a preset file
// Note: for .settings file change it's handled when restart AP.
if (info.suffix() != s_presetFileExtension)
{
return;
}
ReloadPreset(PresetName(info.baseName().toUtf8().data()));
}
void BuilderSettingManager::OnFolderChanged([[maybe_unused]] const QString &path)
{
// handles new file added or removed
// Note: this signal only works with AP but not AssetBuilder
AZ_TracePrintf(LogWindow, "folder changed %s\n", path.toUtf8().data());
AZStd::lock_guard<AZStd::recursive_mutex> lock(m_presetMapLock);
m_presets.clear();
LoadPresets(m_defaultConfigFolder.Native());
LoadPresets(m_projectConfigFolder.Native());
for (auto& preset : m_presets)
{
m_fileWatcher.data()->addPath(QString(preset.second.m_presetFilePath.c_str()));
}
}
} // namespace ImageProcessingAtom