diff --git a/Code/Editor/Animation/AnimationBipedBoneNames.cpp b/Code/Editor/Animation/AnimationBipedBoneNames.cpp index d9f1b845ca..72502fe61d 100644 --- a/Code/Editor/Animation/AnimationBipedBoneNames.cpp +++ b/Code/Editor/Animation/AnimationBipedBoneNames.cpp @@ -10,24 +10,21 @@ #include "AnimationBipedBoneNames.h" -namespace EditorAnimationBones +namespace EditorAnimationBones::Biped { - namespace Biped - { - const char* Pelvis = "Bip01 Pelvis"; - const char* Head = "Bip01 Head"; - const char* Weapon = "weapon_bone"; + const char* Pelvis = "Bip01 Pelvis"; + const char* Head = "Bip01 Head"; + const char* Weapon = "weapon_bone"; - const char* LeftEye = "eye_bone_left"; - const char* RightEye = "eye_bone_right"; + const char* LeftEye = "eye_bone_left"; + const char* RightEye = "eye_bone_right"; - const char* Spine[5] = { "Bip01 Spine", "Bip01 Spine1", "Bip01 Spine2", "Bip01 Spine3", "Bip01 Spine4" }; - const char* Neck[2] = { "Bip01 Neck", "Bip01 Neck1" }; + const char* Spine[5] = { "Bip01 Spine", "Bip01 Spine1", "Bip01 Spine2", "Bip01 Spine3", "Bip01 Spine4" }; + const char* Neck[2] = { "Bip01 Neck", "Bip01 Neck1" }; - const char* LeftHeel = "Bip01 L Heel"; - const char* LeftToe[2] = { "Bip01 L Toe0", "Bip01 L Toe1" }; + const char* LeftHeel = "Bip01 L Heel"; + const char* LeftToe[2] = { "Bip01 L Toe0", "Bip01 L Toe1" }; - const char* RightHeel = "Bip01 R Heel"; - const char* RightToe[2] = { "Bip01 R Toe0", "Bip01 R Toe1" }; - } -} + const char* RightHeel = "Bip01 R Heel"; + const char* RightToe[2] = { "Bip01 R Toe0", "Bip01 R Toe1" }; +} // namespace EditorAnimationBones::Biped diff --git a/Code/Editor/AssetImporter/AssetImporterManager/AssetImporterManager.cpp b/Code/Editor/AssetImporter/AssetImporterManager/AssetImporterManager.cpp index b721b8759e..baa287e47f 100644 --- a/Code/Editor/AssetImporter/AssetImporterManager/AssetImporterManager.cpp +++ b/Code/Editor/AssetImporter/AssetImporterManager/AssetImporterManager.cpp @@ -140,7 +140,7 @@ bool AssetImporterManager::OnBrowseFiles() bool encounteredCrate = false; QStringList invalidFiles; - for (QString path : fileDialog.selectedFiles()) + for (const QString& path : fileDialog.selectedFiles()) { QString fileName = GetFileName(path); QFileInfo info(path); diff --git a/Code/Editor/Controls/ReflectedPropertyControl/ReflectedPropertyCtrl.cpp b/Code/Editor/Controls/ReflectedPropertyControl/ReflectedPropertyCtrl.cpp index 1151137bfc..c5bf6a4d81 100644 --- a/Code/Editor/Controls/ReflectedPropertyControl/ReflectedPropertyCtrl.cpp +++ b/Code/Editor/Controls/ReflectedPropertyControl/ReflectedPropertyCtrl.cpp @@ -671,7 +671,7 @@ AzToolsFramework::PropertyRowWidget* ReflectedPropertyControl::FindPropertyRowWi return nullptr; } const AzToolsFramework::ReflectedPropertyEditor::WidgetList& widgets = m_editor->GetWidgets(); - for (auto instance : widgets) + for (const auto& instance : widgets) { if (instance.second->label() == item->GetPropertyName()) { diff --git a/Code/Editor/Dialogs/PythonScriptsDialog.cpp b/Code/Editor/Dialogs/PythonScriptsDialog.cpp index 35047947ac..06b1acdbc8 100644 --- a/Code/Editor/Dialogs/PythonScriptsDialog.cpp +++ b/Code/Editor/Dialogs/PythonScriptsDialog.cpp @@ -40,10 +40,10 @@ AZ_POP_DISABLE_DLL_EXPORT_MEMBER_WARNING namespace { // File name extension for python files - const QString s_kPythonFileNameSpec = "*.py"; + const QString s_kPythonFileNameSpec("*.py"); // Tree root element name - const QString s_kRootElementName = "Python Scripts"; + const QString s_kRootElementName("Python Scripts"); } ////////////////////////////////////////////////////////////////////////// diff --git a/Code/Editor/Objects/BaseObject.cpp b/Code/Editor/Objects/BaseObject.cpp index e5f121694a..b721dba56b 100644 --- a/Code/Editor/Objects/BaseObject.cpp +++ b/Code/Editor/Objects/BaseObject.cpp @@ -1649,7 +1649,7 @@ QString CBaseObject::GetTypeName() const } QString name; - name.append(className.mid(0, className.length() - subClassName.length())); + name.append(className.midRef(0, className.length() - subClassName.length())); return name; } diff --git a/Code/Editor/Objects/EntityObject.cpp b/Code/Editor/Objects/EntityObject.cpp index b2dfc04102..ae847da160 100644 --- a/Code/Editor/Objects/EntityObject.cpp +++ b/Code/Editor/Objects/EntityObject.cpp @@ -592,11 +592,11 @@ void CEntityObject::AdjustLightProperties(CVarBlockPtr& properties, const char* if (IVariable* pCastShadowVarLegacy = FindVariableInSubBlock(properties, pSubBlockVar, "bCastShadow")) { pCastShadowVarLegacy->SetFlags(pCastShadowVarLegacy->GetFlags() | IVariable::UI_INVISIBLE); - - if (pCastShadowVarLegacy->GetDisplayValue()[0] != '0') + const QString zeroPrefix("0"); + if (!pCastShadowVarLegacy->GetDisplayValue().startsWith(zeroPrefix)) { bCastShadowLegacy = true; - pCastShadowVarLegacy->SetDisplayValue("0"); + pCastShadowVarLegacy->SetDisplayValue(zeroPrefix); } } diff --git a/Code/Editor/Objects/ObjectManager.cpp b/Code/Editor/Objects/ObjectManager.cpp index ae627f4b9d..df3371a31c 100644 --- a/Code/Editor/Objects/ObjectManager.cpp +++ b/Code/Editor/Objects/ObjectManager.cpp @@ -828,7 +828,7 @@ void CObjectManager::ShowLastHiddenObject() { uint64 mostRecentID = CBaseObject::s_invalidHiddenID; CBaseObject* mostRecentObject = nullptr; - for (auto it : m_objects) + for (const auto& it : m_objects) { CBaseObject* obj = it.second; diff --git a/Code/Editor/Plugins/ComponentEntityEditorPlugin/UI/Outliner/OutlinerListModel.cpp b/Code/Editor/Plugins/ComponentEntityEditorPlugin/UI/Outliner/OutlinerListModel.cpp index c25248d4f6..08d79adcf5 100644 --- a/Code/Editor/Plugins/ComponentEntityEditorPlugin/UI/Outliner/OutlinerListModel.cpp +++ b/Code/Editor/Plugins/ComponentEntityEditorPlugin/UI/Outliner/OutlinerListModel.cpp @@ -2640,7 +2640,7 @@ QSize OutlinerItemDelegate::sizeHint(const QStyleOptionViewItem& option, const Q m_cachedBoundingRectOfTallCharacter = QRect(); }; - QTimer::singleShot(0, resetFunction); + QTimer::singleShot(0, this, resetFunction); } // And add 8 to it gives the outliner roughly the visible spacing we're looking for. diff --git a/Code/Editor/Plugins/ComponentEntityEditorPlugin/UI/Outliner/OutlinerWidget.cpp b/Code/Editor/Plugins/ComponentEntityEditorPlugin/UI/Outliner/OutlinerWidget.cpp index 803deb3509..af58ac4b6b 100644 --- a/Code/Editor/Plugins/ComponentEntityEditorPlugin/UI/Outliner/OutlinerWidget.cpp +++ b/Code/Editor/Plugins/ComponentEntityEditorPlugin/UI/Outliner/OutlinerWidget.cpp @@ -121,6 +121,18 @@ namespace SortEntityChildrenRecursively(childId, comparer); } } + + QModelIndex nextIndexForTree(bool direction, OutlinerTreeView *tree, QModelIndex current) + { + if (direction) + { + return tree->indexAbove(current); + } + else + { + return tree->indexBelow(current); + } + } } OutlinerWidget::OutlinerWidget(QWidget* pParent, Qt::WindowFlags flags) @@ -891,9 +903,7 @@ void OutlinerWidget::DoSelectSliceRootNextToSelection(bool isTraversalUpwards) return; } - AZStd::function getNextIdxFunction = - AZStd::bind(isTraversalUpwards ? &QTreeView::indexAbove : &QTreeView::indexBelow, treeView, AZStd::placeholders::_1); - QModelIndex nextIdx = getNextIdxFunction(currentIdx); + QModelIndex nextIdx = nextIndexForTree(isTraversalUpwards,treeView,currentIdx); bool foundSliceRoot = false; while (nextIdx.isValid() && !foundSliceRoot) @@ -904,7 +914,7 @@ void OutlinerWidget::DoSelectSliceRootNextToSelection(bool isTraversalUpwards) AzToolsFramework::ToolsApplicationRequestBus::BroadcastResult( foundSliceRoot, &AzToolsFramework::ToolsApplicationRequests::IsSliceRootEntity, currentEntityId); - nextIdx = getNextIdxFunction(currentIdx); + nextIdx = nextIndexForTree(isTraversalUpwards, treeView, currentIdx); } if (foundSliceRoot) @@ -934,13 +944,10 @@ void OutlinerWidget::DoSelectEdgeSliceRoot(bool shouldSelectTopMostSlice) } QModelIndex currentIdx; - AZStd::function getNextIdxFunction; + if (shouldSelectTopMostSlice) { currentIdx = itemModel->index(0, OutlinerListModel::ColumnName); - - getNextIdxFunction = - AZStd::bind(&QTreeView::indexBelow, treeView, AZStd::placeholders::_1); } else { @@ -949,9 +956,6 @@ void OutlinerWidget::DoSelectEdgeSliceRoot(bool shouldSelectTopMostSlice) { currentIdx = itemModel->index(itemModel->rowCount(currentIdx) - 1, OutlinerListModel::ColumnName, currentIdx); } - - getNextIdxFunction = - AZStd::bind(&QTreeView::indexAbove, treeView, AZStd::placeholders::_1); } QModelIndex nextIdx = currentIdx; @@ -964,7 +968,7 @@ void OutlinerWidget::DoSelectEdgeSliceRoot(bool shouldSelectTopMostSlice) AzToolsFramework::ToolsApplicationRequestBus::BroadcastResult( foundSliceRoot, &AzToolsFramework::ToolsApplicationRequests::IsSliceRootEntity, currentEntityId); - nextIdx = getNextIdxFunction(currentIdx); + nextIdx = nextIndexForTree(shouldSelectTopMostSlice,treeView,currentIdx); } while (nextIdx.isValid() && !foundSliceRoot); if (foundSliceRoot) @@ -1416,7 +1420,10 @@ void OutlinerWidget::SortContent() } m_entitiesToSort.clear(); - auto comparer = AZStd::bind(&CompareEntitiesForSorting, AZStd::placeholders::_1, AZStd::placeholders::_2, m_sortMode); + auto comparer = [sortMode = m_sortMode](AZ::EntityId left, AZ::EntityId right) -> bool + { + return CompareEntitiesForSorting(left, right, sortMode); + }; for (const AZ::EntityId& entityId : parentsToSort) { SortEntityChildren(entityId, comparer); @@ -1433,7 +1440,10 @@ void OutlinerWidget::OnSortModeChanged(EntityOutliner::DisplaySortMode sortMode) if (sortMode != EntityOutliner::DisplaySortMode::Manually) { AZ_PROFILE_FUNCTION(AzToolsFramework); - auto comparer = AZStd::bind(&CompareEntitiesForSorting, AZStd::placeholders::_1, AZStd::placeholders::_2, sortMode); + auto comparer = [sortMode = m_sortMode](AZ::EntityId left, AZ::EntityId right) -> bool + { + return CompareEntitiesForSorting(left, right, sortMode); + }; SortEntityChildrenRecursively(AZ::EntityId(), comparer); } diff --git a/Code/Framework/AzCore/AzCore/Asset/AssetCommon.cpp b/Code/Framework/AzCore/AzCore/Asset/AssetCommon.cpp index fdc3053b5c..bf3ad20768 100644 --- a/Code/Framework/AzCore/AzCore/Asset/AssetCommon.cpp +++ b/Code/Framework/AzCore/AzCore/Asset/AssetCommon.cpp @@ -14,453 +14,450 @@ #include #include -namespace AZ +namespace AZ::Data { - namespace Data + AssetFilterInfo::AssetFilterInfo(const AssetId& id, const AssetType& assetType, AssetLoadBehavior loadBehavior) + : m_assetId(id) + , m_assetType(assetType) + , m_loadBehavior(loadBehavior) { - AssetFilterInfo::AssetFilterInfo(const AssetId& id, const AssetType& assetType, AssetLoadBehavior loadBehavior) - : m_assetId(id) - , m_assetType(assetType) - , m_loadBehavior(loadBehavior) + } + + AssetFilterInfo::AssetFilterInfo(const Asset& asset) + : m_assetId(asset.GetId()) + , m_assetType(asset.GetType()) + , m_loadBehavior(asset.GetAutoLoadBehavior()) + { + } + + + AssetId AssetId::CreateString(AZStd::string_view input) + { + size_t separatorIdx = input.find(':'); + if (separatorIdx == AZStd::string_view::npos) { + return AssetId(); } - AssetFilterInfo::AssetFilterInfo(const Asset& asset) - : m_assetId(asset.GetId()) - , m_assetType(asset.GetType()) - , m_loadBehavior(asset.GetAutoLoadBehavior()) + AssetId assetId; + assetId.m_guid = Uuid::CreateString(input.data(), separatorIdx); + if (assetId.m_guid.IsNull()) { + return AssetId(); } + assetId.m_subId = strtoul(&input[separatorIdx + 1], nullptr, 16); - AssetId AssetId::CreateString(AZStd::string_view input) + return assetId; + } + + void AssetId::Reflect(AZ::ReflectContext* context) + { + if (SerializeContext* serializeContext = azrtti_cast(context)) { - size_t separatorIdx = input.find(':'); - if (separatorIdx == AZStd::string_view::npos) - { - return AssetId(); - } - - AssetId assetId; - assetId.m_guid = Uuid::CreateString(input.data(), separatorIdx); - if (assetId.m_guid.IsNull()) - { - return AssetId(); - } - - assetId.m_subId = strtoul(&input[separatorIdx + 1], nullptr, 16); - - return assetId; + serializeContext->Class() + ->Version(1) + ->Field("guid", &Data::AssetId::m_guid) + ->Field("subId", &Data::AssetId::m_subId) + ; } - void AssetId::Reflect(AZ::ReflectContext* context) + if (BehaviorContext* behaviorContext = azrtti_cast(context)) { - if (SerializeContext* serializeContext = azrtti_cast(context)) - { - serializeContext->Class() - ->Version(1) - ->Field("guid", &Data::AssetId::m_guid) - ->Field("subId", &Data::AssetId::m_subId) - ; - } + behaviorContext->Class() + ->Attribute(AZ::Script::Attributes::Scope, AZ::Script::Attributes::ScopeFlags::Common) + ->Attribute(AZ::Script::Attributes::Category, "Asset") + ->Attribute(AZ::Script::Attributes::Module, "asset") + ->Constructor() + ->Constructor() + ->Method("CreateString", &Data::AssetId::CreateString) + ->Method("IsValid", &Data::AssetId::IsValid) + ->Attribute(AZ::Script::Attributes::Alias, "is_valid") + ->Method("ToString", [](const Data::AssetId* self) { return self->ToString(); }) + ->Attribute(AZ::Script::Attributes::Alias, "to_string") + ->Method("IsEqual", [](const Data::AssetId& self, const Data::AssetId& other) { return self == other; }) + ->Attribute(AZ::Script::Attributes::Alias, "is_equal") + ->Attribute(AZ::Script::Attributes::Operator, AZ::Script::Attributes::OperatorType::Equal) + ; - if (BehaviorContext* behaviorContext = azrtti_cast(context)) - { - behaviorContext->Class() - ->Attribute(AZ::Script::Attributes::Scope, AZ::Script::Attributes::ScopeFlags::Common) - ->Attribute(AZ::Script::Attributes::Category, "Asset") - ->Attribute(AZ::Script::Attributes::Module, "asset") - ->Constructor() - ->Constructor() - ->Method("CreateString", &Data::AssetId::CreateString) - ->Method("IsValid", &Data::AssetId::IsValid) - ->Attribute(AZ::Script::Attributes::Alias, "is_valid") - ->Method("ToString", [](const Data::AssetId* self) { return self->ToString(); }) - ->Attribute(AZ::Script::Attributes::Alias, "to_string") - ->Method("IsEqual", [](const Data::AssetId& self, const Data::AssetId& other) { return self == other; }) - ->Attribute(AZ::Script::Attributes::Alias, "is_equal") - ->Attribute(AZ::Script::Attributes::Operator, AZ::Script::Attributes::OperatorType::Equal) - ; + behaviorContext->Class() + ->Attribute(AZ::Script::Attributes::Scope, AZ::Script::Attributes::ScopeFlags::Common) + ->Attribute(AZ::Script::Attributes::Category, "Asset") + ->Attribute(AZ::Script::Attributes::Module, "asset") + ->Property("assetId", BehaviorValueGetter(&Data::AssetInfo::m_assetId), nullptr) + ->Property("assetType", BehaviorValueGetter(&Data::AssetInfo::m_assetType), nullptr) + ->Property("sizeBytes", BehaviorValueGetter(&Data::AssetInfo::m_sizeBytes), nullptr) + ->Property("relativePath", BehaviorValueGetter(&Data::AssetInfo::m_relativePath), nullptr) + ; + } + } - behaviorContext->Class() - ->Attribute(AZ::Script::Attributes::Scope, AZ::Script::Attributes::ScopeFlags::Common) - ->Attribute(AZ::Script::Attributes::Category, "Asset") - ->Attribute(AZ::Script::Attributes::Module, "asset") - ->Property("assetId", BehaviorValueGetter(&Data::AssetInfo::m_assetId), nullptr) - ->Property("assetType", BehaviorValueGetter(&Data::AssetInfo::m_assetType), nullptr) - ->Property("sizeBytes", BehaviorValueGetter(&Data::AssetInfo::m_sizeBytes), nullptr) - ->Property("relativePath", BehaviorValueGetter(&Data::AssetInfo::m_relativePath), nullptr) - ; - } + namespace AssetInternal + { + Asset FindOrCreateAsset(const AssetId& id, const AssetType& type, AssetLoadBehavior assetReferenceLoadBehavior) + { + return AssetManager::Instance().FindOrCreateAsset(id, type, assetReferenceLoadBehavior); } - namespace AssetInternal + Asset GetAsset(const AssetId& id, const AssetType& type, AssetLoadBehavior assetReferenceLoadBehavior, + const AssetLoadParameters& loadParams) { - Asset FindOrCreateAsset(const AssetId& id, const AssetType& type, AssetLoadBehavior assetReferenceLoadBehavior) - { - return AssetManager::Instance().FindOrCreateAsset(id, type, assetReferenceLoadBehavior); - } - - Asset GetAsset(const AssetId& id, const AssetType& type, AssetLoadBehavior assetReferenceLoadBehavior, - const AssetLoadParameters& loadParams) - { - return AssetManager::Instance().GetAsset(id, type, assetReferenceLoadBehavior, loadParams); - } - - AssetData::AssetStatus BlockUntilLoadComplete(const Asset& asset) - { - return AssetManager::Instance().BlockUntilLoadComplete(asset); - } - - void UpdateAssetInfo(AssetId& id, AZStd::string& assetHint) - { - // it is possible that the assetID given is legacy / old and we have a new assetId we can use instead for it. - // in that case, upgrade the AssetID to the new one, so that future saves are in the new format. - // this function should only be invoked if the feature is turned on in the asset manager as it can be (slightly) expensive - - if ((!AssetManager::IsReady()) || (!AssetManager::Instance().GetAssetInfoUpgradingEnabled())) - { - return; - } - - AZ::Data::AssetInfo assetInfo; - AZ::Data::AssetCatalogRequestBus::BroadcastResult(assetInfo, &AZ::Data::AssetCatalogRequests::GetAssetInfoById, id); - if (assetInfo.m_assetId.IsValid()) - { - id = assetInfo.m_assetId; - if (!assetInfo.m_relativePath.empty()) - { - assetHint = assetInfo.m_relativePath; - } - } - } - - bool ReloadAsset(AssetData* assetData, AssetLoadBehavior assetReferenceLoadBehavior) - { - AssetManager::Instance().ReloadAsset(assetData->GetId(), assetReferenceLoadBehavior); - return true; - } - - bool SaveAsset(AssetData* assetData, AssetLoadBehavior assetReferenceLoadBehavior) - { - AssetManager::Instance().SaveAsset({ assetData, assetReferenceLoadBehavior }); - return true; - } - - Asset GetAssetData(const AssetId& id, AssetLoadBehavior assetReferenceLoadBehavior) - { - if (AssetManager::IsReady()) - { - AZStd::lock_guard assetLock(AssetManager::Instance().m_assetMutex); - auto it = AssetManager::Instance().m_assets.find(id); - if (it != AssetManager::Instance().m_assets.end()) - { - return { it->second, assetReferenceLoadBehavior }; - } - } - return {}; - } - - AssetId ResolveAssetId(const AssetId& id) - { - AZ::Data::AssetInfo assetInfo; - AZ::Data::AssetCatalogRequestBus::BroadcastResult(assetInfo, &AZ::Data::AssetCatalogRequests::GetAssetInfoById, id); - if (assetInfo.m_assetId.IsValid()) - { - return assetInfo.m_assetId; - } - else - { - return id; - } - - } + return AssetManager::Instance().GetAsset(id, type, assetReferenceLoadBehavior, loadParams); } - AssetData::~AssetData() + AssetData::AssetStatus BlockUntilLoadComplete(const Asset& asset) { - UnregisterWithHandler(); + return AssetManager::Instance().BlockUntilLoadComplete(asset); } - void AssetData::Reflect(AZ::ReflectContext* context) + void UpdateAssetInfo(AssetId& id, AZStd::string& assetHint) { - if (SerializeContext* serializeContext = azrtti_cast(context)) + // it is possible that the assetID given is legacy / old and we have a new assetId we can use instead for it. + // in that case, upgrade the AssetID to the new one, so that future saves are in the new format. + // this function should only be invoked if the feature is turned on in the asset manager as it can be (slightly) expensive + + if ((!AssetManager::IsReady()) || (!AssetManager::Instance().GetAssetInfoUpgradingEnabled())) { - serializeContext->Class() - ->Version(1) - ; - } - - if (BehaviorContext* behaviorContext = azrtti_cast(context)) - { - behaviorContext->Class("AssetData") - ->Attribute(AZ::Script::Attributes::Scope, AZ::Script::Attributes::ScopeFlags::Common) - ->Attribute(AZ::Script::Attributes::Category, "Asset") - ->Attribute(AZ::Script::Attributes::Module, "asset") - ->Method("IsReady", &AssetData::IsReady) - ->Attribute(AZ::Script::Attributes::Alias, "is_ready") - ->Method("IsError", &AssetData::IsError) - ->Attribute(AZ::Script::Attributes::Alias, "is_error") - ->Method("IsLoading", &AssetData::IsLoading) - ->Attribute(AZ::Script::Attributes::Alias, "is_loading") - ->Method("GetId", &AssetData::GetId) - ->Attribute(AZ::Script::Attributes::Alias, "get_id") - ->Method("GetUseCount", &AssetData::GetUseCount) - ->Attribute(AZ::Script::Attributes::Alias, "get_use_count") - ; - } - } - - void AssetData::Acquire() - { - AZ_Assert(m_useCount >= 0, "AssetData has been deleted"); - - AcquireWeak(); - ++m_useCount; - } - - void AssetData::Release() - { - AZ_Assert(m_useCount > 0, "Usecount is already 0!"); - - if (m_useCount.fetch_sub(1) == 1) - { - if (AssetManager::IsReady()) - { - AssetManager::Instance().OnAssetUnused(this); - } - else - { - AZ_Assert(false, "Attempting to release asset after AssetManager has been destroyed!"); - } - } - - ReleaseWeak(); - } - - void AssetData::AcquireWeak() - { - AZ_Assert(m_useCount >= 0, "AssetData has been deleted"); - ++m_weakUseCount; - } - - void AssetData::ReleaseWeak() - { - AZ_Assert(m_weakUseCount > 0, "WeakUseCount is already 0"); - - AssetId assetId = m_assetId; - int creationToken = m_creationToken; - AssetType assetType = GetType(); - bool removeFromHash = IsRegisterReadonlyAndShareable(); - // default creation token implies that the asset was not created by the asset manager and therefore it cannot be in the asset map. - removeFromHash = creationToken == s_defaultCreationToken ? false : removeFromHash; - - if (m_weakUseCount.fetch_sub(1) == 1) - { - if (AssetManager::IsReady()) - { - AssetManager::Instance().ReleaseAsset(this, assetId, assetType, removeFromHash, creationToken); - } - else - { - AZ_Assert(false, "Attempting to release asset after AssetManager has been destroyed!"); - } - } - } - - bool AssetData::IsLoading(bool includeQueued) const - { - auto curStatus = GetStatus(); - return(curStatus == AssetStatus::Loading || curStatus == AssetStatus::LoadedPreReady || curStatus==AssetStatus::StreamReady || - (includeQueued && curStatus == AssetStatus::Queued)); - } - - void AssetData::RegisterWithHandler(AssetHandler* handler) - { - if (!handler) - { - AZ_Error("AssetData", false, "No handler to register with"); return; } - m_registeredHandler = handler; - } - void AssetData::UnregisterWithHandler() - { - if (m_registeredHandler) + AZ::Data::AssetInfo assetInfo; + AZ::Data::AssetCatalogRequestBus::BroadcastResult(assetInfo, &AZ::Data::AssetCatalogRequests::GetAssetInfoById, id); + if (assetInfo.m_assetId.IsValid()) { - m_registeredHandler = nullptr; - } - } - - bool AssetData::GetFlag(const AssetDataFlags& checkFlag) const - { - return m_flags[aznumeric_cast(checkFlag)]; - } - - void AssetData::SetFlag(const AssetDataFlags& checkFlag, bool setValue) - { - m_flags.set(aznumeric_cast(checkFlag), setValue); - } - - bool AssetData::GetRequeue() const - { - return GetFlag(AssetDataFlags::Requeue); - } - void AssetData::SetRequeue(bool requeue) - { - SetFlag(AssetDataFlags::Requeue, requeue); - } - - void AssetBusCallbacks::SetCallbacks(const AssetReadyCB& readyCB, const AssetMovedCB& movedCB, const AssetReloadedCB& reloadedCB, - const AssetSavedCB& savedCB, const AssetUnloadedCB& unloadedCB, const AssetErrorCB& errorCB, const AssetCanceledCB& cancelCB) - { - m_onAssetReadyCB = readyCB; - m_onAssetMovedCB = movedCB; - m_onAssetReloadedCB = reloadedCB; - m_onAssetSavedCB = savedCB; - m_onAssetUnloadedCB = unloadedCB; - m_onAssetErrorCB = errorCB; - m_onAssetCanceledCB = cancelCB; - } - - void AssetBusCallbacks::ClearCallbacks() - { - SetCallbacks(AssetBusCallbacks::AssetReadyCB(), - AssetBusCallbacks::AssetMovedCB(), - AssetBusCallbacks::AssetReloadedCB(), - AssetBusCallbacks::AssetSavedCB(), - AssetBusCallbacks::AssetUnloadedCB(), - AssetBusCallbacks::AssetErrorCB(), - AssetBusCallbacks::AssetCanceledCB()); - } - - - void AssetBusCallbacks::SetOnAssetReadyCallback(const AssetReadyCB& readyCB) - { - m_onAssetReadyCB = readyCB; - } - - void AssetBusCallbacks::SetOnAssetMovedCallback(const AssetMovedCB& movedCB) - { - m_onAssetMovedCB = movedCB; - } - - void AssetBusCallbacks::SetOnAssetReloadedCallback(const AssetReloadedCB& reloadedCB) - { - m_onAssetReloadedCB = reloadedCB; - } - - void AssetBusCallbacks::SetOnAssetSavedCallback(const AssetSavedCB& savedCB) - { - m_onAssetSavedCB = savedCB; - } - - void AssetBusCallbacks::SetOnAssetUnloadedCallback(const AssetUnloadedCB& unloadedCB) - { - m_onAssetUnloadedCB = unloadedCB; - } - - void AssetBusCallbacks::SetOnAssetErrorCallback(const AssetErrorCB& errorCB) - { - m_onAssetErrorCB = errorCB; - } - - void AssetBusCallbacks::SetOnAssetCanceledCallback(const AssetCanceledCB& cancelCB) - { - m_onAssetCanceledCB = cancelCB; - } - - void AssetBusCallbacks::OnAssetReady(Asset asset) - { - if (m_onAssetReadyCB) - { - m_onAssetReadyCB(asset, *this); - } - } - - void AssetBusCallbacks::OnAssetMoved(Asset asset, void* oldDataPointer) - { - if (m_onAssetMovedCB) - { - m_onAssetMovedCB(asset, oldDataPointer, *this); - } - } - - void AssetBusCallbacks::OnAssetReloaded(Asset asset) - { - if (m_onAssetReloadedCB) - { - m_onAssetReloadedCB(asset, *this); - } - } - - void AssetBusCallbacks::OnAssetSaved(Asset asset, bool isSuccessful) - { - if (m_onAssetSavedCB) - { - m_onAssetSavedCB(asset, isSuccessful, *this); - } - } - - void AssetBusCallbacks::OnAssetUnloaded(const AssetId assetId, const AssetType assetType) - { - if (m_onAssetUnloadedCB) - { - m_onAssetUnloadedCB(assetId, assetType, *this); - } - } - - void AssetBusCallbacks::OnAssetError(Asset asset) - { - if (m_onAssetErrorCB) - { - m_onAssetErrorCB(asset, *this); - } - } - - void AssetBusCallbacks::OnAssetCanceled(const AssetId assetId) - { - if (m_onAssetCanceledCB) - { - m_onAssetCanceledCB(assetId, *this); - } - } - - /*static*/ bool AssetFilterNoAssetLoading([[maybe_unused]] const AssetFilterInfo& filterInfo) - { - return false; - } - namespace ProductDependencyInfo - { - AZ::Data::AssetLoadBehavior LoadBehaviorFromFlags(const ProductDependencyFlags& dependencyFlags) - { - AZ::u8 loadBehaviorValue = 0; - for (AZ::u8 thisFlag = aznumeric_cast(ProductDependencyFlagBits::LoadBehaviorLow); - thisFlag <= aznumeric_cast(ProductDependencyFlagBits::LoadBehaviorHigh); ++thisFlag) + id = assetInfo.m_assetId; + if (!assetInfo.m_relativePath.empty()) { - if (dependencyFlags[thisFlag]) - { - loadBehaviorValue |= (1 << thisFlag); - } + assetHint = assetInfo.m_relativePath; } - return static_cast(loadBehaviorValue); - } - - ProductDependencyFlags CreateFlags(AZ::Data::AssetLoadBehavior autoLoadBehavior) - { - AZ::Data::ProductDependencyInfo::ProductDependencyFlags returnFlags; - AZ::u8 loadBehavior = aznumeric_caster(autoLoadBehavior); - for (AZ::u8 thisFlag = aznumeric_cast(ProductDependencyFlagBits::LoadBehaviorLow); - thisFlag <= aznumeric_cast(ProductDependencyFlagBits::LoadBehaviorHigh); ++thisFlag) - { - if (loadBehavior & (1 << thisFlag)) - { - returnFlags[thisFlag] = true; - } - } - return returnFlags; } } - } // namespace Data -} // namespace AZ + + bool ReloadAsset(AssetData* assetData, AssetLoadBehavior assetReferenceLoadBehavior) + { + AssetManager::Instance().ReloadAsset(assetData->GetId(), assetReferenceLoadBehavior); + return true; + } + + bool SaveAsset(AssetData* assetData, AssetLoadBehavior assetReferenceLoadBehavior) + { + AssetManager::Instance().SaveAsset({ assetData, assetReferenceLoadBehavior }); + return true; + } + + Asset GetAssetData(const AssetId& id, AssetLoadBehavior assetReferenceLoadBehavior) + { + if (AssetManager::IsReady()) + { + AZStd::lock_guard assetLock(AssetManager::Instance().m_assetMutex); + auto it = AssetManager::Instance().m_assets.find(id); + if (it != AssetManager::Instance().m_assets.end()) + { + return { it->second, assetReferenceLoadBehavior }; + } + } + return {}; + } + + AssetId ResolveAssetId(const AssetId& id) + { + AZ::Data::AssetInfo assetInfo; + AZ::Data::AssetCatalogRequestBus::BroadcastResult(assetInfo, &AZ::Data::AssetCatalogRequests::GetAssetInfoById, id); + if (assetInfo.m_assetId.IsValid()) + { + return assetInfo.m_assetId; + } + else + { + return id; + } + + } + } + + AssetData::~AssetData() + { + UnregisterWithHandler(); + } + + void AssetData::Reflect(AZ::ReflectContext* context) + { + if (SerializeContext* serializeContext = azrtti_cast(context)) + { + serializeContext->Class() + ->Version(1) + ; + } + + if (BehaviorContext* behaviorContext = azrtti_cast(context)) + { + behaviorContext->Class("AssetData") + ->Attribute(AZ::Script::Attributes::Scope, AZ::Script::Attributes::ScopeFlags::Common) + ->Attribute(AZ::Script::Attributes::Category, "Asset") + ->Attribute(AZ::Script::Attributes::Module, "asset") + ->Method("IsReady", &AssetData::IsReady) + ->Attribute(AZ::Script::Attributes::Alias, "is_ready") + ->Method("IsError", &AssetData::IsError) + ->Attribute(AZ::Script::Attributes::Alias, "is_error") + ->Method("IsLoading", &AssetData::IsLoading) + ->Attribute(AZ::Script::Attributes::Alias, "is_loading") + ->Method("GetId", &AssetData::GetId) + ->Attribute(AZ::Script::Attributes::Alias, "get_id") + ->Method("GetUseCount", &AssetData::GetUseCount) + ->Attribute(AZ::Script::Attributes::Alias, "get_use_count") + ; + } + } + + void AssetData::Acquire() + { + AZ_Assert(m_useCount >= 0, "AssetData has been deleted"); + + AcquireWeak(); + ++m_useCount; + } + + void AssetData::Release() + { + AZ_Assert(m_useCount > 0, "Usecount is already 0!"); + + if (m_useCount.fetch_sub(1) == 1) + { + if (AssetManager::IsReady()) + { + AssetManager::Instance().OnAssetUnused(this); + } + else + { + AZ_Assert(false, "Attempting to release asset after AssetManager has been destroyed!"); + } + } + + ReleaseWeak(); + } + + void AssetData::AcquireWeak() + { + AZ_Assert(m_useCount >= 0, "AssetData has been deleted"); + ++m_weakUseCount; + } + + void AssetData::ReleaseWeak() + { + AZ_Assert(m_weakUseCount > 0, "WeakUseCount is already 0"); + + AssetId assetId = m_assetId; + int creationToken = m_creationToken; + AssetType assetType = GetType(); + bool removeFromHash = IsRegisterReadonlyAndShareable(); + // default creation token implies that the asset was not created by the asset manager and therefore it cannot be in the asset map. + removeFromHash = creationToken == s_defaultCreationToken ? false : removeFromHash; + + if (m_weakUseCount.fetch_sub(1) == 1) + { + if (AssetManager::IsReady()) + { + AssetManager::Instance().ReleaseAsset(this, assetId, assetType, removeFromHash, creationToken); + } + else + { + AZ_Assert(false, "Attempting to release asset after AssetManager has been destroyed!"); + } + } + } + + bool AssetData::IsLoading(bool includeQueued) const + { + auto curStatus = GetStatus(); + return(curStatus == AssetStatus::Loading || curStatus == AssetStatus::LoadedPreReady || curStatus==AssetStatus::StreamReady || + (includeQueued && curStatus == AssetStatus::Queued)); + } + + void AssetData::RegisterWithHandler(AssetHandler* handler) + { + if (!handler) + { + AZ_Error("AssetData", false, "No handler to register with"); + return; + } + m_registeredHandler = handler; + } + + void AssetData::UnregisterWithHandler() + { + if (m_registeredHandler) + { + m_registeredHandler = nullptr; + } + } + + bool AssetData::GetFlag(const AssetDataFlags& checkFlag) const + { + return m_flags[aznumeric_cast(checkFlag)]; + } + + void AssetData::SetFlag(const AssetDataFlags& checkFlag, bool setValue) + { + m_flags.set(aznumeric_cast(checkFlag), setValue); + } + + bool AssetData::GetRequeue() const + { + return GetFlag(AssetDataFlags::Requeue); + } + void AssetData::SetRequeue(bool requeue) + { + SetFlag(AssetDataFlags::Requeue, requeue); + } + + void AssetBusCallbacks::SetCallbacks(const AssetReadyCB& readyCB, const AssetMovedCB& movedCB, const AssetReloadedCB& reloadedCB, + const AssetSavedCB& savedCB, const AssetUnloadedCB& unloadedCB, const AssetErrorCB& errorCB, const AssetCanceledCB& cancelCB) + { + m_onAssetReadyCB = readyCB; + m_onAssetMovedCB = movedCB; + m_onAssetReloadedCB = reloadedCB; + m_onAssetSavedCB = savedCB; + m_onAssetUnloadedCB = unloadedCB; + m_onAssetErrorCB = errorCB; + m_onAssetCanceledCB = cancelCB; + } + + void AssetBusCallbacks::ClearCallbacks() + { + SetCallbacks(AssetBusCallbacks::AssetReadyCB(), + AssetBusCallbacks::AssetMovedCB(), + AssetBusCallbacks::AssetReloadedCB(), + AssetBusCallbacks::AssetSavedCB(), + AssetBusCallbacks::AssetUnloadedCB(), + AssetBusCallbacks::AssetErrorCB(), + AssetBusCallbacks::AssetCanceledCB()); + } + + + void AssetBusCallbacks::SetOnAssetReadyCallback(const AssetReadyCB& readyCB) + { + m_onAssetReadyCB = readyCB; + } + + void AssetBusCallbacks::SetOnAssetMovedCallback(const AssetMovedCB& movedCB) + { + m_onAssetMovedCB = movedCB; + } + + void AssetBusCallbacks::SetOnAssetReloadedCallback(const AssetReloadedCB& reloadedCB) + { + m_onAssetReloadedCB = reloadedCB; + } + + void AssetBusCallbacks::SetOnAssetSavedCallback(const AssetSavedCB& savedCB) + { + m_onAssetSavedCB = savedCB; + } + + void AssetBusCallbacks::SetOnAssetUnloadedCallback(const AssetUnloadedCB& unloadedCB) + { + m_onAssetUnloadedCB = unloadedCB; + } + + void AssetBusCallbacks::SetOnAssetErrorCallback(const AssetErrorCB& errorCB) + { + m_onAssetErrorCB = errorCB; + } + + void AssetBusCallbacks::SetOnAssetCanceledCallback(const AssetCanceledCB& cancelCB) + { + m_onAssetCanceledCB = cancelCB; + } + + void AssetBusCallbacks::OnAssetReady(Asset asset) + { + if (m_onAssetReadyCB) + { + m_onAssetReadyCB(asset, *this); + } + } + + void AssetBusCallbacks::OnAssetMoved(Asset asset, void* oldDataPointer) + { + if (m_onAssetMovedCB) + { + m_onAssetMovedCB(asset, oldDataPointer, *this); + } + } + + void AssetBusCallbacks::OnAssetReloaded(Asset asset) + { + if (m_onAssetReloadedCB) + { + m_onAssetReloadedCB(asset, *this); + } + } + + void AssetBusCallbacks::OnAssetSaved(Asset asset, bool isSuccessful) + { + if (m_onAssetSavedCB) + { + m_onAssetSavedCB(asset, isSuccessful, *this); + } + } + + void AssetBusCallbacks::OnAssetUnloaded(const AssetId assetId, const AssetType assetType) + { + if (m_onAssetUnloadedCB) + { + m_onAssetUnloadedCB(assetId, assetType, *this); + } + } + + void AssetBusCallbacks::OnAssetError(Asset asset) + { + if (m_onAssetErrorCB) + { + m_onAssetErrorCB(asset, *this); + } + } + + void AssetBusCallbacks::OnAssetCanceled(const AssetId assetId) + { + if (m_onAssetCanceledCB) + { + m_onAssetCanceledCB(assetId, *this); + } + } + + /*static*/ bool AssetFilterNoAssetLoading([[maybe_unused]] const AssetFilterInfo& filterInfo) + { + return false; + } + namespace ProductDependencyInfo + { + AZ::Data::AssetLoadBehavior LoadBehaviorFromFlags(const ProductDependencyFlags& dependencyFlags) + { + AZ::u8 loadBehaviorValue = 0; + for (AZ::u8 thisFlag = aznumeric_cast(ProductDependencyFlagBits::LoadBehaviorLow); + thisFlag <= aznumeric_cast(ProductDependencyFlagBits::LoadBehaviorHigh); ++thisFlag) + { + if (dependencyFlags[thisFlag]) + { + loadBehaviorValue |= (1 << thisFlag); + } + } + return static_cast(loadBehaviorValue); + } + + ProductDependencyFlags CreateFlags(AZ::Data::AssetLoadBehavior autoLoadBehavior) + { + AZ::Data::ProductDependencyInfo::ProductDependencyFlags returnFlags; + AZ::u8 loadBehavior = aznumeric_caster(autoLoadBehavior); + for (AZ::u8 thisFlag = aznumeric_cast(ProductDependencyFlagBits::LoadBehaviorLow); + thisFlag <= aznumeric_cast(ProductDependencyFlagBits::LoadBehaviorHigh); ++thisFlag) + { + if (loadBehavior & (1 << thisFlag)) + { + returnFlags[thisFlag] = true; + } + } + return returnFlags; + } + } +} // namespace AZ::Data diff --git a/Code/Framework/AzCore/AzCore/Asset/AssetContainer.cpp b/Code/Framework/AzCore/AzCore/Asset/AssetContainer.cpp index ce15c7bc4e..7c5fd182c2 100644 --- a/Code/Framework/AzCore/AzCore/Asset/AssetContainer.cpp +++ b/Code/Framework/AzCore/AzCore/Asset/AssetContainer.cpp @@ -11,466 +11,452 @@ #include #include -namespace AZ +namespace AZ::Data { - namespace Data + AssetContainer::AssetContainer(Asset rootAsset, const AssetLoadParameters& loadParams) { - AssetContainer::AssetContainer(Asset rootAsset, const AssetLoadParameters& loadParams) - { - m_rootAsset = AssetInternal::WeakAsset(rootAsset); - m_containerAssetId = m_rootAsset.GetId(); + m_rootAsset = AssetInternal::WeakAsset(rootAsset); + m_containerAssetId = m_rootAsset.GetId(); - AddDependentAssets(rootAsset, loadParams); + AddDependentAssets(rootAsset, loadParams); + } + + AssetContainer::~AssetContainer() + { + // Validate that if the AssetManager is performing normal processing duties, the AssetContainer is only destroyed once all + // dependent asset loads have completed. + if (AssetManager::IsReady() && !AssetManager::Instance().ShouldCancelAllActiveJobs()) + { + AZ_Assert(m_waitingCount == 0, "Container destroyed while dependent assets are still loading. The dependent assets may " + "end up in a perpetual loading state if there is no top-level container signalling the completion of the full load."); } - AssetContainer::~AssetContainer() + AssetBus::MultiHandler::BusDisconnect(); + AssetLoadBus::MultiHandler::BusDisconnect(); + } + + void AssetContainer::AddDependentAssets(Asset rootAsset, const AssetLoadParameters& loadParams) + { + AssetId rootAssetId = rootAsset.GetId(); + AssetType rootAssetType = rootAsset.GetType(); + + // Every asset we're going to be waiting on a load for - the root and all valid dependencies + AZStd::vector waitingList; + waitingList.push_back(rootAssetId); + + // Every asset dependency that we're aware of, whether or not it gets filtered out by the asset filter callback. + // This will be used at the point that asset references get serialized in to see whether or not we've received any + // unexpected assets that didn't appear in our asset catalog dependency list that need to be loaded anyways. + AZStd::vector handledAssetDependencyList; + + // Cached AssetInfo to save another lookup inside Assetmanager + AZStd::vector dependencyInfoList; + Outcome, AZStd::string> getDependenciesResult = Failure(AZStd::string()); + + // Track preloads in an additional list - they're in our waiting/dependencyInfo lists as well, but preloads require us to + // suppress emitting "AssetReady" until everything we care about in this context is ready + PreloadAssetListType preloadDependencies; + if (loadParams.m_dependencyRules == AssetDependencyLoadRules::UseLoadBehavior) { - // Validate that if the AssetManager is performing normal processing duties, the AssetContainer is only destroyed once all - // dependent asset loads have completed. - if (AssetManager::IsReady() && !AssetManager::Instance().ShouldCancelAllActiveJobs()) - { - AZ_Assert(m_waitingCount == 0, "Container destroyed while dependent assets are still loading. The dependent assets may " - "end up in a perpetual loading state if there is no top-level container signalling the completion of the full load."); - } - - AssetBus::MultiHandler::BusDisconnect(); - AssetLoadBus::MultiHandler::BusDisconnect(); - } - - void AssetContainer::AddDependentAssets(Asset rootAsset, const AssetLoadParameters& loadParams) - { - AssetId rootAssetId = rootAsset.GetId(); - AssetType rootAssetType = rootAsset.GetType(); - - // Every asset we're going to be waiting on a load for - the root and all valid dependencies - AZStd::vector waitingList; - waitingList.push_back(rootAssetId); - - // Every asset dependency that we're aware of, whether or not it gets filtered out by the asset filter callback. - // This will be used at the point that asset references get serialized in to see whether or not we've received any - // unexpected assets that didn't appear in our asset catalog dependency list that need to be loaded anyways. - AZStd::vector handledAssetDependencyList; - - // Cached AssetInfo to save another lookup inside Assetmanager - AZStd::vector dependencyInfoList; - Outcome, AZStd::string> getDependenciesResult = Failure(AZStd::string()); - - // Track preloads in an additional list - they're in our waiting/dependencyInfo lists as well, but preloads require us to - // suppress emitting "AssetReady" until everything we care about in this context is ready - PreloadAssetListType preloadDependencies; - if (loadParams.m_dependencyRules == AssetDependencyLoadRules::UseLoadBehavior) - { - AZStd::unordered_set noloadDependencies; - AssetCatalogRequestBus::BroadcastResult(getDependenciesResult, &AssetCatalogRequestBus::Events::GetLoadBehaviorProductDependencies, - rootAssetId, noloadDependencies, preloadDependencies); - if (!noloadDependencies.empty()) - { - AZStd::lock_guard dependencyLock(m_dependencyMutex); - m_unloadedDependencies.insert(noloadDependencies.begin(), noloadDependencies.end()); - } - } - else if (loadParams.m_dependencyRules == AssetDependencyLoadRules::LoadAll) - { - AssetCatalogRequestBus::BroadcastResult(getDependenciesResult, &AssetCatalogRequestBus::Events::GetAllProductDependencies, rootAssetId); - } - // Do as much validation of dependencies as we can before the AddWaitingAssets and GetAsset calls for dependencies below - if (getDependenciesResult.IsSuccess()) - { - for (const auto& thisAsset : getDependenciesResult.GetValue()) - { - AssetInfo assetInfo; - AssetCatalogRequestBus::BroadcastResult(assetInfo, &AssetCatalogRequestBus::Events::GetAssetInfoById, thisAsset.m_assetId); - - // No matter whether or not the asset dependency is valid, loaded, or filtered out, mark it as successfully handled. - // When we encounter the asset reference during serialization, we will know that it should intentionally be skipped. - // Otherwise, it would be treated as a missing dependency and assert. - handledAssetDependencyList.emplace_back(thisAsset.m_assetId); - - if (!assetInfo.m_assetId.IsValid()) - { - // Handlers may just not currently be around for a given asset type so we only warn here - AZ_Warning("AssetContainer", false, "Asset %s (%s) references/depends on asset %s which does not exist in the catalog and cannot be loaded.", - rootAsset.GetHint().c_str(), - rootAssetId.ToString().c_str(), - thisAsset.m_assetId.ToString().c_str()); - m_invalidDependencies++; - continue; - } - if (assetInfo.m_assetId == rootAssetId) - { - // Circular dependencies in our graph need to be raised as errors as they could cause problems elsewhere - AZ_Error("AssetContainer", false, "Circular dependency found under asset %s", rootAssetId.ToString().c_str()); - m_invalidDependencies++; - continue; - } - if (!AssetManager::Instance().GetHandler(assetInfo.m_assetType)) - { - // Handlers may just not currently be around for a given asset type so we only warn here - m_invalidDependencies++; - continue; - } - if (loadParams.m_assetLoadFilterCB) - { - if (!loadParams.m_assetLoadFilterCB({thisAsset.m_assetId, assetInfo.m_assetType, - AZ::Data::ProductDependencyInfo::LoadBehaviorFromFlags(thisAsset.m_flags) })) - { - continue; - } - } - dependencyInfoList.push_back(assetInfo); - } - } - for (auto& thisInfo : dependencyInfoList) - { - waitingList.push_back(thisInfo.m_assetId); - } - - // Add waiting assets ahead of time to hear signals for any which may already be loading - AddWaitingAssets(waitingList); - SetupPreloadLists(move(preloadDependencies), rootAssetId); - - auto loadParamsCopyWithNoLoadingFilter = loadParams; - - // All asset dependencies below the root asset should be provided by the asset catalog, and therefore should *not* - // get triggered to load when the asset reference is serialized in. However, it's useful to detect, warn, and handle - // the case where the asset dependencies are NOT set up correctly. - loadParamsCopyWithNoLoadingFilter.m_assetLoadFilterCB = [handledAssetDependencyList](const AssetFilterInfo& filterInfo) - { - // NoLoad dependencies should always get filtered out and not loaded. - if (filterInfo.m_loadBehavior == AZ::Data::AssetLoadBehavior::NoLoad) - { - return false; - } - - // In the normal case, the dependent asset appears in the handled asset list, and we should return false so that - // the asset isn't attempted to be loaded, since the asset will already be triggered to get loaded or was possibly - // already filtered out by the load filter callback. - // In the error case, the asset dependencies haven't been produced by the builder correctly, so assets - // have shown up that the asset container hasn't triggered to load and isn't listening for. Assert that this case - // has happened so that the builder for this asset type can be fixed. - // Ideally we would proceed forward and load them by returning "true", but the triggered load would use this lambda - // function as the asset load filter for that load as well, which isn't correct. If we ever want to support that - // behavior, we would need to rework the way filters work as well as the code in AssetSerializer.cpp to pass down - // the loadParams.m_assetLoadFilterCB that was passed into the AddDependentAssets() methods to use as the dependent - // asset filter instead of this lambda function. - AZ_UNUSED(handledAssetDependencyList); // Prevent unused warning in release builds - AZ_Assert(AZStd::find(handledAssetDependencyList.begin(), handledAssetDependencyList.end(), filterInfo.m_assetId) != - handledAssetDependencyList.end(), - "Dependent Asset ID (%s) is expected to load, but the Asset Catalog has no dependency recorded. " - "Examine the asset builder for the asset relying on this to ensure it is generating the correct dependencies.", - filterInfo.m_assetId.ToString().c_str()); - - // The dependent asset should have already been created and at least queued to load prior to reaching this point. - // The asset serializer needs to get a successful result from FindAsset(), or else our asset reference will fail - // to point to the asset data once it is loaded. - if (!Data::AssetManager::Instance().FindAsset(filterInfo.m_assetId, AZ::Data::AssetLoadBehavior::Default)) - { - AZ_Assert(!Data::AssetManager::Instance().FindAsset(filterInfo.m_assetId, AZ::Data::AssetLoadBehavior::Default), - "Dependent Asset ID (%s) can't be found in the AssetManager, which means the asset referencing it has probably " - "started loading before the dependent asset has been queued to load. Verify that the asset dependencies have " - "been created correctly for the parent asset.", - filterInfo.m_assetId.ToString().c_str()); - } - - return false; - }; - - // This will contain the list of dependent assets that have been created (or found) and queued to load. - // We also keep a copy of the AssetInfo structure as a small optimization to avoid a redundant lookup in GetAssetInternal. - AZStd::vector>> dependencyAssets; - - // Make sure all the dependencies are created first before we try to load them. - // Since we've set the load filter to not load dependencies, we need to ensure all the assets are created beforehand - // so the dependencies can be hooked up as soon as each asset gets serialized in, even if they start getting serialized - // while we're still in the middle of triggering all of the asset loads below. - for (auto& thisInfo : dependencyInfoList) - { - auto dependentAsset = AssetManager::Instance().FindOrCreateAsset( - thisInfo.m_assetId, thisInfo.m_assetType, AZ::Data::AssetLoadBehavior::Default); - - if (!dependentAsset || !dependentAsset.GetId().IsValid()) - { - AZ_Warning("AssetContainer", false, "Dependency Asset %s (%s) was not found\n", - thisInfo.m_assetId.ToString().c_str(), thisInfo.m_relativePath.c_str()); - RemoveWaitingAsset(thisInfo.m_assetId); - continue; - } - dependencyAssets.emplace_back(thisInfo, AZStd::move(dependentAsset)); - } - - // Queue the loading of all of the dependent assets before loading the root asset. - for (auto& [dependentAssetInfo, dependentAsset] : dependencyAssets) - { - // Queue each asset to load. - auto queuedDependentAsset = AssetManager::Instance().GetAssetInternal( - dependentAsset.GetId(), dependentAsset.GetType(), - AZ::Data::AssetLoadBehavior::Default, loadParamsCopyWithNoLoadingFilter, - dependentAssetInfo, HasPreloads(dependentAsset.GetId())); - - // Verify that the returned asset reference matches the one that we found or created and queued to load. - AZ_Assert(dependentAsset == queuedDependentAsset, "GetAssetInternal returned an unexpected asset reference for Asset %s", - dependentAsset.GetId().ToString().c_str()); - } - - // Add all of the queued dependent assets as dependencies + AZStd::unordered_set noloadDependencies; + AssetCatalogRequestBus::BroadcastResult(getDependenciesResult, &AssetCatalogRequestBus::Events::GetLoadBehaviorProductDependencies, + rootAssetId, noloadDependencies, preloadDependencies); + if (!noloadDependencies.empty()) { AZStd::lock_guard dependencyLock(m_dependencyMutex); - for (auto& [dependentAssetInfo, dependentAsset] : dependencyAssets) + m_unloadedDependencies.insert(noloadDependencies.begin(), noloadDependencies.end()); + } + } + else if (loadParams.m_dependencyRules == AssetDependencyLoadRules::LoadAll) + { + AssetCatalogRequestBus::BroadcastResult(getDependenciesResult, &AssetCatalogRequestBus::Events::GetAllProductDependencies, rootAssetId); + } + // Do as much validation of dependencies as we can before the AddWaitingAssets and GetAsset calls for dependencies below + if (getDependenciesResult.IsSuccess()) + { + for (const auto& thisAsset : getDependenciesResult.GetValue()) + { + AssetInfo assetInfo; + AssetCatalogRequestBus::BroadcastResult(assetInfo, &AssetCatalogRequestBus::Events::GetAssetInfoById, thisAsset.m_assetId); + + // No matter whether or not the asset dependency is valid, loaded, or filtered out, mark it as successfully handled. + // When we encounter the asset reference during serialization, we will know that it should intentionally be skipped. + // Otherwise, it would be treated as a missing dependency and assert. + handledAssetDependencyList.emplace_back(thisAsset.m_assetId); + + if (!assetInfo.m_assetId.IsValid()) { - AddDependency(AZStd::move(dependentAsset)); + // Handlers may just not currently be around for a given asset type so we only warn here + AZ_Warning("AssetContainer", false, "Asset %s (%s) references/depends on asset %s which does not exist in the catalog and cannot be loaded.", + rootAsset.GetHint().c_str(), + rootAssetId.ToString().c_str(), + thisAsset.m_assetId.ToString().c_str()); + m_invalidDependencies++; + continue; } + if (assetInfo.m_assetId == rootAssetId) + { + // Circular dependencies in our graph need to be raised as errors as they could cause problems elsewhere + AZ_Error("AssetContainer", false, "Circular dependency found under asset %s", rootAssetId.ToString().c_str()); + m_invalidDependencies++; + continue; + } + if (!AssetManager::Instance().GetHandler(assetInfo.m_assetType)) + { + // Handlers may just not currently be around for a given asset type so we only warn here + m_invalidDependencies++; + continue; + } + if (loadParams.m_assetLoadFilterCB) + { + if (!loadParams.m_assetLoadFilterCB({thisAsset.m_assetId, assetInfo.m_assetType, + AZ::Data::ProductDependencyInfo::LoadBehaviorFromFlags(thisAsset.m_flags) })) + { + continue; + } + } + dependencyInfoList.push_back(assetInfo); + } + } + for (auto& thisInfo : dependencyInfoList) + { + waitingList.push_back(thisInfo.m_assetId); + } + + // Add waiting assets ahead of time to hear signals for any which may already be loading + AddWaitingAssets(waitingList); + SetupPreloadLists(move(preloadDependencies), rootAssetId); + + auto loadParamsCopyWithNoLoadingFilter = loadParams; + + // All asset dependencies below the root asset should be provided by the asset catalog, and therefore should *not* + // get triggered to load when the asset reference is serialized in. However, it's useful to detect, warn, and handle + // the case where the asset dependencies are NOT set up correctly. + loadParamsCopyWithNoLoadingFilter.m_assetLoadFilterCB = [handledAssetDependencyList](const AssetFilterInfo& filterInfo) + { + // NoLoad dependencies should always get filtered out and not loaded. + if (filterInfo.m_loadBehavior == AZ::Data::AssetLoadBehavior::NoLoad) + { + return false; } - // Finally, after creating and queueing the dependent assets, queue the root asset. This is saved until last to ensure that - // it doesn't have any chance of serializing in until after all the dependent assets have been queued for loading and have - // been added to the list of dependencies. - auto thisAsset = AssetManager::Instance().GetAssetInternal(rootAssetId, rootAssetType, rootAsset.GetAutoLoadBehavior(), - loadParamsCopyWithNoLoadingFilter, AssetInfo(), HasPreloads(rootAssetId)); + // In the normal case, the dependent asset appears in the handled asset list, and we should return false so that + // the asset isn't attempted to be loaded, since the asset will already be triggered to get loaded or was possibly + // already filtered out by the load filter callback. + // In the error case, the asset dependencies haven't been produced by the builder correctly, so assets + // have shown up that the asset container hasn't triggered to load and isn't listening for. Assert that this case + // has happened so that the builder for this asset type can be fixed. + // Ideally we would proceed forward and load them by returning "true", but the triggered load would use this lambda + // function as the asset load filter for that load as well, which isn't correct. If we ever want to support that + // behavior, we would need to rework the way filters work as well as the code in AssetSerializer.cpp to pass down + // the loadParams.m_assetLoadFilterCB that was passed into the AddDependentAssets() methods to use as the dependent + // asset filter instead of this lambda function. + AZ_UNUSED(handledAssetDependencyList); // Prevent unused warning in release builds + AZ_Assert(AZStd::find(handledAssetDependencyList.begin(), handledAssetDependencyList.end(), filterInfo.m_assetId) != + handledAssetDependencyList.end(), + "Dependent Asset ID (%s) is expected to load, but the Asset Catalog has no dependency recorded. " + "Examine the asset builder for the asset relying on this to ensure it is generating the correct dependencies.", + filterInfo.m_assetId.ToString().c_str()); - if (!thisAsset) + // The dependent asset should have already been created and at least queued to load prior to reaching this point. + // The asset serializer needs to get a successful result from FindAsset(), or else our asset reference will fail + // to point to the asset data once it is loaded. + if (!Data::AssetManager::Instance().FindAsset(filterInfo.m_assetId, AZ::Data::AssetLoadBehavior::Default)) { - AZ_Assert(false, "Root asset with id %s failed to load, asset container is invalid.", - rootAssetId.ToString().c_str()); - ClearWaitingAssets(); - // initComplete remains false, because we have failed to initialize successfully. + AZ_Assert(!Data::AssetManager::Instance().FindAsset(filterInfo.m_assetId, AZ::Data::AssetLoadBehavior::Default), + "Dependent Asset ID (%s) can't be found in the AssetManager, which means the asset referencing it has probably " + "started loading before the dependent asset has been queued to load. Verify that the asset dependencies have " + "been created correctly for the parent asset.", + filterInfo.m_assetId.ToString().c_str()); + } + + return false; + }; + + // This will contain the list of dependent assets that have been created (or found) and queued to load. + // We also keep a copy of the AssetInfo structure as a small optimization to avoid a redundant lookup in GetAssetInternal. + AZStd::vector>> dependencyAssets; + + // Make sure all the dependencies are created first before we try to load them. + // Since we've set the load filter to not load dependencies, we need to ensure all the assets are created beforehand + // so the dependencies can be hooked up as soon as each asset gets serialized in, even if they start getting serialized + // while we're still in the middle of triggering all of the asset loads below. + for (auto& thisInfo : dependencyInfoList) + { + auto dependentAsset = AssetManager::Instance().FindOrCreateAsset( + thisInfo.m_assetId, thisInfo.m_assetType, AZ::Data::AssetLoadBehavior::Default); + + if (!dependentAsset || !dependentAsset.GetId().IsValid()) + { + AZ_Warning("AssetContainer", false, "Dependency Asset %s (%s) was not found\n", + thisInfo.m_assetId.ToString().c_str(), thisInfo.m_relativePath.c_str()); + RemoveWaitingAsset(thisInfo.m_assetId); + continue; + } + dependencyAssets.emplace_back(thisInfo, AZStd::move(dependentAsset)); + } + + // Queue the loading of all of the dependent assets before loading the root asset. + for (auto& [dependentAssetInfo, dependentAsset] : dependencyAssets) + { + // Queue each asset to load. + auto queuedDependentAsset = AssetManager::Instance().GetAssetInternal( + dependentAsset.GetId(), dependentAsset.GetType(), + AZ::Data::AssetLoadBehavior::Default, loadParamsCopyWithNoLoadingFilter, + dependentAssetInfo, HasPreloads(dependentAsset.GetId())); + + // Verify that the returned asset reference matches the one that we found or created and queued to load. + AZ_Assert(dependentAsset == queuedDependentAsset, "GetAssetInternal returned an unexpected asset reference for Asset %s", + dependentAsset.GetId().ToString().c_str()); + } + + // Add all of the queued dependent assets as dependencies + { + AZStd::lock_guard dependencyLock(m_dependencyMutex); + for (auto& [dependentAssetInfo, dependentAsset] : dependencyAssets) + { + AddDependency(AZStd::move(dependentAsset)); + } + } + + // Finally, after creating and queueing the dependent assets, queue the root asset. This is saved until last to ensure that + // it doesn't have any chance of serializing in until after all the dependent assets have been queued for loading and have + // been added to the list of dependencies. + auto thisAsset = AssetManager::Instance().GetAssetInternal(rootAssetId, rootAssetType, rootAsset.GetAutoLoadBehavior(), + loadParamsCopyWithNoLoadingFilter, AssetInfo(), HasPreloads(rootAssetId)); + + if (!thisAsset) + { + AZ_Assert(false, "Root asset with id %s failed to load, asset container is invalid.", + rootAssetId.ToString().c_str()); + ClearWaitingAssets(); + // initComplete remains false, because we have failed to initialize successfully. + return; + } + + m_initComplete = true; + + // *After* setting initComplete to true, check to see if the assets are already ready. + // This check needs to wait until after setting initComplete because if they *are* ready, we want the final call to + // RemoveWaitingAsset to trigger the OnAssetContainerReady/Canceled event. If we call CheckReady() *before* setting + // initComplete, if all the assets are ready, the event will never get triggered. + CheckReady(); + } + + bool AssetContainer::IsReady() const + { + return (m_rootAsset && m_waitingCount == 0); + } + + bool AssetContainer::IsLoading() const + { + return (m_rootAsset || m_waitingCount); + } + + bool AssetContainer::IsValid() const + { + return (m_containerAssetId.IsValid() && m_initComplete && m_rootAsset); + } + + void AssetContainer::CheckReady() + { + if (!m_dependencies.empty()) + { + for (auto& [assetId, dependentAsset] : m_dependencies) + { + if (dependentAsset->IsReady() || dependentAsset->IsError()) + { + HandleReadyAsset(dependentAsset); + } + } + } + if (auto asset = m_rootAsset.GetStrongReference(); asset.IsReady() || asset.IsError()) + { + HandleReadyAsset(asset); + } + } + + Asset AssetContainer::GetRootAsset() + { + return m_rootAsset.GetStrongReference(); + } + + AssetId AssetContainer::GetContainerAssetId() + { + return m_containerAssetId; + } + + void AssetContainer::ClearRootAsset() + { + AssetId rootId = m_rootAsset.GetId(); + + { + AZStd::lock_guard preloadGuard(m_preloadMutex); + + // Erase the entry in the preloadWaitList for the root asset if one exists. + m_preloadWaitList.erase(rootId); + + // It's possible that the root asset has preload dependencies, so make sure to check the preload list and remove + // the entry for the root asset if it has one. + auto rootAssetPreloadIter = m_preloadList.find(rootId); + if (rootAssetPreloadIter != m_preloadList.end()) + { + // Since the root asset has a preload list, that means the preload wait list will also have references to the + // root asset. (The preload wait list is a list of assets waiting on a preload asset to finish) Clear those + // out as well. + auto waitAssetSet = rootAssetPreloadIter->second; + for (auto& waitId : waitAssetSet) + { + auto waitAssetIter = m_preloadWaitList.find(waitId); + if (waitAssetIter != m_preloadWaitList.end()) + { + waitAssetIter->second.erase(rootId); + } + } + + m_preloadList.erase(rootAssetPreloadIter); + } + } + + // Clear out the root asset before removing it from the waiting list to ensure that we trigger an "OnAssetContainerCanceled" + // event instead of "OnAssetContainerReady". + m_rootAsset = {}; + RemoveWaitingAsset(rootId); + + } + + void AssetContainer::AddDependency(const Asset& newDependency) + { + m_dependencies[newDependency->GetId()] = newDependency; + } + void AssetContainer::AddDependency(Asset&& newDependency) + { + m_dependencies[newDependency->GetId()] = AZStd::move(newDependency); + } + + void AssetContainer::OnAssetReady(Asset asset) + { + HandleReadyAsset(asset); + } + + void AssetContainer::OnAssetError(Asset asset) + { + AZ_Warning("AssetContainer", false, "Error loading asset %s", asset->GetId().ToString().c_str()); + HandleReadyAsset(asset); + } + + void AssetContainer::HandleReadyAsset(Asset asset) + { + RemoveFromAllWaitingPreloads(asset->GetId()); + RemoveWaitingAsset(asset->GetId()); + } + + void AssetContainer::OnAssetDataLoaded(Asset asset) + { + // Remove only from this asset's waiting list. Anything else should + // listen for OnAssetReady as the true signal. This is essentially removing the + // "marker" we placed in SetupPreloads that we need to wait for our own data + RemoveFromWaitingPreloads(asset->GetId(), asset->GetId()); + } + + void AssetContainer::RemoveFromWaitingPreloads(const AssetId& waiterId, const AssetId& preloadID) + { + { + AZStd::lock_guard preloadGuard(m_preloadMutex); + + auto remainingPreloadIter = m_preloadList.find(waiterId); + if (remainingPreloadIter == m_preloadList.end()) + { + // If we got here without an entry on the preload list, it probably means this asset was triggered to load multiple + // times, some with dependencies and some without. To ensure that we don't disturb the loads that expect the + // dependencies, just silently return and don't treat the asset as finished loading. We'll rely on the other load + // to send an OnAssetReady() whenever its expected dependencies are met. return; } - - m_initComplete = true; - - // *After* setting initComplete to true, check to see if the assets are already ready. - // This check needs to wait until after setting initComplete because if they *are* ready, we want the final call to - // RemoveWaitingAsset to trigger the OnAssetContainerReady/Canceled event. If we call CheckReady() *before* setting - // initComplete, if all the assets are ready, the event will never get triggered. - CheckReady(); - } - - bool AssetContainer::IsReady() const - { - return (m_rootAsset && m_waitingCount == 0); - } - - bool AssetContainer::IsLoading() const - { - return (m_rootAsset || m_waitingCount); - } - - bool AssetContainer::IsValid() const - { - return (m_containerAssetId.IsValid() && m_initComplete && m_rootAsset); - } - - void AssetContainer::CheckReady() - { - if (!m_dependencies.empty()) + if (!remainingPreloadIter->second.erase(preloadID)) { - for (auto& [assetId, dependentAsset] : m_dependencies) - { - if (dependentAsset->IsReady() || dependentAsset->IsError()) - { - HandleReadyAsset(dependentAsset); - } - } + AZ_Warning("AssetContainer", !m_initComplete, "Couldn't remove %s from waiting list of %s", preloadID.ToString().c_str(), waiterId.ToString().c_str()); + return; } - if (auto asset = m_rootAsset.GetStrongReference(); asset.IsReady() || asset.IsError()) + if (!remainingPreloadIter->second.empty()) { - HandleReadyAsset(asset); + return; } } + auto thisAsset = GetAssetData(waiterId); + AssetManager::Instance().ValidateAndPostLoad(thisAsset, true, false, nullptr); + } - Asset AssetContainer::GetRootAsset() + void AssetContainer::RemoveFromAllWaitingPreloads(const AssetId& thisId) + { + AZStd::unordered_set checkList; { - return m_rootAsset.GetStrongReference(); - } - - AssetId AssetContainer::GetContainerAssetId() - { - return m_containerAssetId; - } - - void AssetContainer::ClearRootAsset() - { - AssetId rootId = m_rootAsset.GetId(); - - { - AZStd::lock_guard preloadGuard(m_preloadMutex); - - // Erase the entry in the preloadWaitList for the root asset if one exists. - m_preloadWaitList.erase(rootId); - - // It's possible that the root asset has preload dependencies, so make sure to check the preload list and remove - // the entry for the root asset if it has one. - auto rootAssetPreloadIter = m_preloadList.find(rootId); - if (rootAssetPreloadIter != m_preloadList.end()) - { - // Since the root asset has a preload list, that means the preload wait list will also have references to the - // root asset. (The preload wait list is a list of assets waiting on a preload asset to finish) Clear those - // out as well. - auto waitAssetSet = rootAssetPreloadIter->second; - for (auto& waitId : waitAssetSet) - { - auto waitAssetIter = m_preloadWaitList.find(waitId); - if (waitAssetIter != m_preloadWaitList.end()) - { - waitAssetIter->second.erase(rootId); - } - } - - m_preloadList.erase(rootAssetPreloadIter); - } - } - - // Clear out the root asset before removing it from the waiting list to ensure that we trigger an "OnAssetContainerCanceled" - // event instead of "OnAssetContainerReady". - m_rootAsset = {}; - RemoveWaitingAsset(rootId); - - } - - void AssetContainer::AddDependency(const Asset& newDependency) - { - m_dependencies[newDependency->GetId()] = newDependency; - } - void AssetContainer::AddDependency(Asset&& newDependency) - { - m_dependencies[newDependency->GetId()] = AZStd::move(newDependency); - } - - void AssetContainer::OnAssetReady(Asset asset) - { - HandleReadyAsset(asset); - } - - void AssetContainer::OnAssetError(Asset asset) - { - AZ_Warning("AssetContainer", false, "Error loading asset %s", asset->GetId().ToString().c_str()); - HandleReadyAsset(asset); - } - - void AssetContainer::HandleReadyAsset(Asset asset) - { - RemoveFromAllWaitingPreloads(asset->GetId()); - RemoveWaitingAsset(asset->GetId()); - } - - void AssetContainer::OnAssetDataLoaded(Asset asset) - { - // Remove only from this asset's waiting list. Anything else should - // listen for OnAssetReady as the true signal. This is essentially removing the - // "marker" we placed in SetupPreloads that we need to wait for our own data - RemoveFromWaitingPreloads(asset->GetId(), asset->GetId()); - } - - void AssetContainer::RemoveFromWaitingPreloads(const AssetId& waiterId, const AssetId& preloadID) - { - { - AZStd::lock_guard preloadGuard(m_preloadMutex); - - auto remainingPreloadIter = m_preloadList.find(waiterId); - if (remainingPreloadIter == m_preloadList.end()) - { - // If we got here without an entry on the preload list, it probably means this asset was triggered to load multiple - // times, some with dependencies and some without. To ensure that we don't disturb the loads that expect the - // dependencies, just silently return and don't treat the asset as finished loading. We'll rely on the other load - // to send an OnAssetReady() whenever its expected dependencies are met. - return; - } - if (!remainingPreloadIter->second.erase(preloadID)) - { - AZ_Warning("AssetContainer", !m_initComplete, "Couldn't remove %s from waiting list of %s", preloadID.ToString().c_str(), waiterId.ToString().c_str()); - return; - } - if (!remainingPreloadIter->second.empty()) - { - return; - } - } - auto thisAsset = GetAssetData(waiterId); - AssetManager::Instance().ValidateAndPostLoad(thisAsset, true, false, nullptr); - } - - void AssetContainer::RemoveFromAllWaitingPreloads(const AssetId& thisId) - { - AZStd::unordered_set checkList; - { - AZStd::lock_guard preloadGuard(m_preloadMutex); - - auto waitingList = m_preloadWaitList.find(thisId); - if (waitingList != m_preloadWaitList.end()) - { - checkList = move(waitingList->second); - m_preloadWaitList.erase(waitingList); - } - } - for (auto& thisDepId : checkList) - { - if (thisDepId != thisId) - { - RemoveFromWaitingPreloads(thisDepId, thisId); - } - } - } - - void AssetContainer::ClearWaitingAssets() - { - AZStd::lock_guard lock(m_readyMutex); - m_waitingCount = 0; - for (auto& thisAsset : m_waitingAssets) - { - AssetBus::MultiHandler::BusDisconnect(thisAsset); - } - m_waitingAssets.clear(); - } - - void AssetContainer::ListWaitingAssets() const - { -#if defined(AZ_ENABLE_TRACING) - AZStd::lock_guard lock(m_readyMutex); - AZ_TracePrintf("AssetContainer", "Waiting on assets:\n"); - for (auto& thisAsset : m_waitingAssets) - { - AZ_TracePrintf("AssetContainer", " %s\n",thisAsset.ToString().c_str()); - } -#endif - } - - void AssetContainer::ListWaitingPreloads([[maybe_unused]] const AssetId& assetId) const - { -#if defined(AZ_ENABLE_TRACING) AZStd::lock_guard preloadGuard(m_preloadMutex); - auto preloadEntry = m_preloadList.find(assetId); - if (preloadEntry != m_preloadList.end()) + + auto waitingList = m_preloadWaitList.find(thisId); + if (waitingList != m_preloadWaitList.end()) { - AZ_TracePrintf("AssetContainer", "%s waiting on preloads : \n",assetId.ToString().c_str()); - for (auto& thisId : preloadEntry->second) - { - AZ_TracePrintf("AssetContainer", " %s\n",thisId.ToString().c_str()); - } + checkList = move(waitingList->second); + m_preloadWaitList.erase(waitingList); } - else + } + for (auto& thisDepId : checkList) + { + if (thisDepId != thisId) { - AZ_TracePrintf("AssetContainer", "%s isn't waiting on any preloads:\n", assetId.ToString().c_str()); + RemoveFromWaitingPreloads(thisDepId, thisId); } + } + } + + void AssetContainer::ClearWaitingAssets() + { + AZStd::lock_guard lock(m_readyMutex); + m_waitingCount = 0; + for (auto& thisAsset : m_waitingAssets) + { + AssetBus::MultiHandler::BusDisconnect(thisAsset); + } + m_waitingAssets.clear(); + } + + void AssetContainer::ListWaitingAssets() const + { +#if defined(AZ_ENABLE_TRACING) + AZStd::lock_guard lock(m_readyMutex); + AZ_TracePrintf("AssetContainer", "Waiting on assets:\n"); + for (auto& thisAsset : m_waitingAssets) + { + AZ_TracePrintf("AssetContainer", " %s\n",thisAsset.ToString().c_str()); + } #endif - } + } - void AssetContainer::AddWaitingAssets(const AZStd::vector& assetList) + void AssetContainer::ListWaitingPreloads([[maybe_unused]] const AssetId& assetId) const + { +#if defined(AZ_ENABLE_TRACING) + AZStd::lock_guard preloadGuard(m_preloadMutex); + auto preloadEntry = m_preloadList.find(assetId); + if (preloadEntry != m_preloadList.end()) { - AZStd::lock_guard lock(m_readyMutex); - for (auto& thisAsset : assetList) + AZ_TracePrintf("AssetContainer", "%s waiting on preloads : \n",assetId.ToString().c_str()); + for (auto& thisId : preloadEntry->second) { - if (m_waitingAssets.insert(thisAsset).second) - { - ++m_waitingCount; - AssetBus::MultiHandler::BusConnect(thisAsset); - AssetLoadBus::MultiHandler::BusConnect(thisAsset); - } + AZ_TracePrintf("AssetContainer", " %s\n",thisId.ToString().c_str()); } } - - void AssetContainer::AddWaitingAsset(const AssetId& thisAsset) + else + { + AZ_TracePrintf("AssetContainer", "%s isn't waiting on any preloads:\n", assetId.ToString().c_str()); + } +#endif + } + + void AssetContainer::AddWaitingAssets(const AZStd::vector& assetList) + { + AZStd::lock_guard lock(m_readyMutex); + for (auto& thisAsset : assetList) { - AZStd::lock_guard lock(m_readyMutex); if (m_waitingAssets.insert(thisAsset).second) { ++m_waitingCount; @@ -478,196 +464,207 @@ namespace AZ AssetLoadBus::MultiHandler::BusConnect(thisAsset); } } + } - void AssetContainer::RemoveWaitingAsset(const AssetId& thisAsset) + void AssetContainer::AddWaitingAsset(const AssetId& thisAsset) + { + AZStd::lock_guard lock(m_readyMutex); + if (m_waitingAssets.insert(thisAsset).second) { - bool allReady{ false }; - { - bool disconnectEbus = false; + ++m_waitingCount; + AssetBus::MultiHandler::BusConnect(thisAsset); + AssetLoadBus::MultiHandler::BusConnect(thisAsset); + } + } - { // Intentionally limiting lock scope - AZStd::lock_guard lock(m_readyMutex); - // If we're trying to remove something already removed, just ignore it - if (m_waitingAssets.erase(thisAsset)) - { - m_waitingCount -= 1; - disconnectEbus = true; + void AssetContainer::RemoveWaitingAsset(const AssetId& thisAsset) + { + bool allReady{ false }; + { + bool disconnectEbus = false; - } - if (m_waitingAssets.empty()) - { - allReady = true; - } - } - - if(disconnectEbus) + { // Intentionally limiting lock scope + AZStd::lock_guard lock(m_readyMutex); + // If we're trying to remove something already removed, just ignore it + if (m_waitingAssets.erase(thisAsset)) { - AssetBus::MultiHandler::BusDisconnect(thisAsset); - AssetLoadBus::MultiHandler::BusDisconnect(thisAsset); + m_waitingCount -= 1; + disconnectEbus = true; + + } + if (m_waitingAssets.empty()) + { + allReady = true; } } - // If there are no assets left to be loaded, trigger the final AssetContainer notification (ready or canceled). - // We guard against prematurely sending it (m_initComplete) because it's possible for assets to get removed from our waiting - // list *while* we're still building up the list, so the list would appear to be empty too soon. - // We also guard against sending it multiple times (m_finalNotificationSent), because in some error conditions, it may be - // possible to try to remove the same asset multiple times, which if it's the last asset, it could trigger multiple - // notifications. - if (allReady && m_initComplete && !m_finalNotificationSent) + if(disconnectEbus) { - m_finalNotificationSent = true; - if (m_rootAsset) - { - AssetManagerBus::Broadcast(&AssetManagerBus::Events::OnAssetContainerReady, this); - } - else - { - AssetManagerBus::Broadcast(&AssetManagerBus::Events::OnAssetContainerCanceled, this); - } + AssetBus::MultiHandler::BusDisconnect(thisAsset); + AssetLoadBus::MultiHandler::BusDisconnect(thisAsset); } } - AssetContainer::operator bool() const + // If there are no assets left to be loaded, trigger the final AssetContainer notification (ready or canceled). + // We guard against prematurely sending it (m_initComplete) because it's possible for assets to get removed from our waiting + // list *while* we're still building up the list, so the list would appear to be empty too soon. + // We also guard against sending it multiple times (m_finalNotificationSent), because in some error conditions, it may be + // possible to try to remove the same asset multiple times, which if it's the last asset, it could trigger multiple + // notifications. + if (allReady && m_initComplete && !m_finalNotificationSent) { - return m_rootAsset ? true : false; - } - - const AssetContainer::DependencyList& AssetContainer::GetDependencies() const - { - return m_dependencies; - } - - const AZStd::unordered_set& AssetContainer::GetUnloadedDependencies() const - { - return m_unloadedDependencies; - } - - void AssetContainer::SetupPreloadLists(PreloadAssetListType&& preloadList, const AssetId& rootAssetId) - { - if (!preloadList.empty()) + m_finalNotificationSent = true; + if (m_rootAsset) { - // This method can be entered as additional NoLoad dependency groups are loaded - the container could - // be in the middle of loading so we need to grab both mutexes. - AZStd::scoped_lock lock(m_readyMutex, m_preloadMutex); + AssetManagerBus::Broadcast(&AssetManagerBus::Events::OnAssetContainerReady, this); + } + else + { + AssetManagerBus::Broadcast(&AssetManagerBus::Events::OnAssetContainerCanceled, this); + } + } + } - for (auto thisListPair = preloadList.begin(); thisListPair != preloadList.end();) + AssetContainer::operator bool() const + { + return m_rootAsset ? true : false; + } + + const AssetContainer::DependencyList& AssetContainer::GetDependencies() const + { + return m_dependencies; + } + + const AZStd::unordered_set& AssetContainer::GetUnloadedDependencies() const + { + return m_unloadedDependencies; + } + + void AssetContainer::SetupPreloadLists(PreloadAssetListType&& preloadList, const AssetId& rootAssetId) + { + if (!preloadList.empty()) + { + // This method can be entered as additional NoLoad dependency groups are loaded - the container could + // be in the middle of loading so we need to grab both mutexes. + AZStd::scoped_lock lock(m_readyMutex, m_preloadMutex); + + for (auto thisListPair = preloadList.begin(); thisListPair != preloadList.end();) + { + // We only should add ourselves if we have another valid preload we're waiting on + bool foundAsset{ false }; + // It's possible this set of preload dependencies was culled out by lack of asset handler + // Or filtering rules. This is not an error, we should just remove it from the list of + // Preloads we're waiting on + if (!m_waitingAssets.count(thisListPair->first)) { - // We only should add ourselves if we have another valid preload we're waiting on - bool foundAsset{ false }; - // It's possible this set of preload dependencies was culled out by lack of asset handler - // Or filtering rules. This is not an error, we should just remove it from the list of - // Preloads we're waiting on - if (!m_waitingAssets.count(thisListPair->first)) + thisListPair = preloadList.erase(thisListPair); + continue; + } + for (auto thisAsset = thisListPair->second.begin(); thisAsset != thisListPair->second.end();) + { + // These are data errors. We'll emit the error but carry on. The container + // will load the assets but won't/can't create a circular preload dependency chain + if (*thisAsset == rootAssetId) { - thisListPair = preloadList.erase(thisListPair); + AZ_Error("AssetContainer", false, "Circular preload dependency found - %s has a preload" + "dependency back to root %s\n", + thisListPair->first.ToString().c_str(), + rootAssetId.ToString().c_str()); + thisAsset = thisListPair->second.erase(thisAsset); continue; } - for (auto thisAsset = thisListPair->second.begin(); thisAsset != thisListPair->second.end();) + else if (*thisAsset == thisListPair->first) { - // These are data errors. We'll emit the error but carry on. The container - // will load the assets but won't/can't create a circular preload dependency chain - if (*thisAsset == rootAssetId) - { - AZ_Error("AssetContainer", false, "Circular preload dependency found - %s has a preload" - "dependency back to root %s\n", - thisListPair->first.ToString().c_str(), - rootAssetId.ToString().c_str()); - thisAsset = thisListPair->second.erase(thisAsset); - continue; - } - else if (*thisAsset == thisListPair->first) - { - AZ_Error("AssetContainer", false, "Circular preload dependency found - Root asset %s has a preload" - "dependency on %s which depends back back to itself\n", - rootAssetId.ToString().c_str(), - thisListPair->first.ToString().c_str()); - thisAsset = thisListPair->second.erase(thisAsset); - continue; - } - else if (m_preloadWaitList.count(thisListPair->first) && m_preloadWaitList[thisListPair->first].count(*thisAsset)) - { - AZ_Error("AssetContainer", false, "Circular dependency found - Root asset %s has a preload" - "dependency on %s which has a circular dependency with %s\n", - rootAssetId.ToString().c_str(), - thisListPair->first.ToString().c_str(), - thisAsset->ToString().c_str()); - thisAsset = thisListPair->second.erase(thisAsset); - continue; - } - else if (m_waitingAssets.count(*thisAsset)) - { - foundAsset = true; - m_preloadWaitList[*thisAsset].insert(thisListPair->first); - ++thisAsset; - } - else - { - // This particular preload dependency of this asset was culled - // similar to the case above this can be due to no established asset handler - // or filtering rules. We'll just erase the entry because we're not loading this - thisAsset = thisListPair->second.erase(thisAsset); - continue; - } + AZ_Error("AssetContainer", false, "Circular preload dependency found - Root asset %s has a preload" + "dependency on %s which depends back back to itself\n", + rootAssetId.ToString().c_str(), + thisListPair->first.ToString().c_str()); + thisAsset = thisListPair->second.erase(thisAsset); + continue; } - if (foundAsset) + else if (m_preloadWaitList.count(thisListPair->first) && m_preloadWaitList[thisListPair->first].count(*thisAsset)) { - // We've established that this asset has at least one preload dependency it needs to wait on - // so we additionally add the waiting asset as its own preload so all of our "waiting assets" - // are managed in the same list. We can't consider this asset to be "ready" until all - // of its preloads are ready and it has been loaded. It will request an OnAssetDataLoaded - // notification from AssetManager rather than an OnAssetReady because of these additional dependencies. - thisListPair->second.insert(thisListPair->first); - m_preloadWaitList[thisListPair->first].insert(thisListPair->first); + AZ_Error("AssetContainer", false, "Circular dependency found - Root asset %s has a preload" + "dependency on %s which has a circular dependency with %s\n", + rootAssetId.ToString().c_str(), + thisListPair->first.ToString().c_str(), + thisAsset->ToString().c_str()); + thisAsset = thisListPair->second.erase(thisAsset); + continue; + } + else if (m_waitingAssets.count(*thisAsset)) + { + foundAsset = true; + m_preloadWaitList[*thisAsset].insert(thisListPair->first); + ++thisAsset; + } + else + { + // This particular preload dependency of this asset was culled + // similar to the case above this can be due to no established asset handler + // or filtering rules. We'll just erase the entry because we're not loading this + thisAsset = thisListPair->second.erase(thisAsset); + continue; } - ++thisListPair; } - for(auto& thisList : preloadList) + if (foundAsset) { - // Only save the entry to the final preload list if it has at least one dependent asset still remaining after - // the checks above. - if (!thisList.second.empty()) - { - m_preloadList[thisList.first].insert(thisList.second.begin(), thisList.second.end()); - } + // We've established that this asset has at least one preload dependency it needs to wait on + // so we additionally add the waiting asset as its own preload so all of our "waiting assets" + // are managed in the same list. We can't consider this asset to be "ready" until all + // of its preloads are ready and it has been loaded. It will request an OnAssetDataLoaded + // notification from AssetManager rather than an OnAssetReady because of these additional dependencies. + thisListPair->second.insert(thisListPair->first); + m_preloadWaitList[thisListPair->first].insert(thisListPair->first); + } + ++thisListPair; + } + for(auto& thisList : preloadList) + { + // Only save the entry to the final preload list if it has at least one dependent asset still remaining after + // the checks above. + if (!thisList.second.empty()) + { + m_preloadList[thisList.first].insert(thisList.second.begin(), thisList.second.end()); } } } + } - bool AssetContainer::HasPreloads(const AssetId& assetId) const + bool AssetContainer::HasPreloads(const AssetId& assetId) const + { + AZStd::lock_guard preloadGuard(m_preloadMutex); + auto preloadEntry = m_preloadList.find(assetId); + if (preloadEntry != m_preloadList.end()) { - AZStd::lock_guard preloadGuard(m_preloadMutex); - auto preloadEntry = m_preloadList.find(assetId); - if (preloadEntry != m_preloadList.end()) - { - return !preloadEntry->second.empty(); - } - return false; + return !preloadEntry->second.empty(); } + return false; + } - Asset AssetContainer::GetAssetData(const AssetId& assetId) const + Asset AssetContainer::GetAssetData(const AssetId& assetId) const + { + AZStd::lock_guard dependenciesGuard(m_dependencyMutex); + if (auto rootAsset = m_rootAsset.GetStrongReference(); rootAsset.GetId() == assetId) { - AZStd::lock_guard dependenciesGuard(m_dependencyMutex); - if (auto rootAsset = m_rootAsset.GetStrongReference(); rootAsset.GetId() == assetId) - { - return rootAsset; - } - auto dependencyIter = m_dependencies.find(assetId); - if (dependencyIter != m_dependencies.end()) - { - return dependencyIter->second; - } - AZ_Warning("AssetContainer", false, "Asset %s not found in container", assetId.ToString().c_str()); - return {}; + return rootAsset; } + auto dependencyIter = m_dependencies.find(assetId); + if (dependencyIter != m_dependencies.end()) + { + return dependencyIter->second; + } + AZ_Warning("AssetContainer", false, "Asset %s not found in container", assetId.ToString().c_str()); + return {}; + } - int AssetContainer::GetNumWaitingDependencies() const - { - return m_waitingCount.load(); - } + int AssetContainer::GetNumWaitingDependencies() const + { + return m_waitingCount.load(); + } - int AssetContainer::GetInvalidDependencies() const - { - return m_invalidDependencies.load(); - } - } // namespace Data -} // namespace AZ + int AssetContainer::GetInvalidDependencies() const + { + return m_invalidDependencies.load(); + } +} // namespace AZ::Data diff --git a/Code/Framework/AzCore/AzCore/Asset/AssetJsonSerializer.cpp b/Code/Framework/AzCore/AzCore/Asset/AssetJsonSerializer.cpp index 3fa3b39ca5..0d28036646 100644 --- a/Code/Framework/AzCore/AzCore/Asset/AssetJsonSerializer.cpp +++ b/Code/Framework/AzCore/AzCore/Asset/AssetJsonSerializer.cpp @@ -12,207 +12,204 @@ #include #include -namespace AZ +namespace AZ::Data { - namespace Data + AZ_CLASS_ALLOCATOR_IMPL(AssetJsonSerializer, SystemAllocator, 0); + + JsonSerializationResult::Result AssetJsonSerializer::Load(void* outputValue, const Uuid& /*outputValueTypeId*/, + const rapidjson::Value& inputValue, JsonDeserializerContext& context) { - AZ_CLASS_ALLOCATOR_IMPL(AssetJsonSerializer, SystemAllocator, 0); + namespace JSR = JsonSerializationResult; - JsonSerializationResult::Result AssetJsonSerializer::Load(void* outputValue, const Uuid& /*outputValueTypeId*/, - const rapidjson::Value& inputValue, JsonDeserializerContext& context) + switch (inputValue.GetType()) { - namespace JSR = JsonSerializationResult; + case rapidjson::kObjectType: + return LoadAsset(outputValue, inputValue, context); + case rapidjson::kArrayType: // fall through + case rapidjson::kNullType: // fall through + case rapidjson::kStringType: // fall through + case rapidjson::kFalseType: // fall through + case rapidjson::kTrueType: // fall through + case rapidjson::kNumberType: + return context.Report(JSR::Tasks::ReadField, JSR::Outcomes::Unsupported, + "Unsupported type. Asset can only be read from an object."); - switch (inputValue.GetType()) + default: + return context.Report(JSR::Tasks::ReadField, JSR::Outcomes::Unknown, "Unknown json type encountered for Asset."); + } + } + + JsonSerializationResult::Result AssetJsonSerializer::Store(rapidjson::Value& outputValue, const void* inputValue, + const void* defaultValue, const Uuid& /*valueTypeId*/, JsonSerializerContext& context) + { + namespace JSR = JsonSerializationResult; + + const Asset* instance = reinterpret_cast*>(inputValue); + const Asset* defaultInstance = reinterpret_cast*>(defaultValue); + + JSR::ResultCode result(JSR::Tasks::WriteValue); + { + ScopedContextPath subPathId(context, "m_assetId"); + const auto* id = &instance->GetId(); + const auto* defaultId = defaultInstance ? &defaultInstance->GetId() : nullptr; + rapidjson::Value assetIdValue; + result = ContinueStoring(assetIdValue, id, defaultId, azrtti_typeid(), context); + if (result.GetOutcome() == JSR::Outcomes::Success || result.GetOutcome() == JSR::Outcomes::PartialDefaults) { - case rapidjson::kObjectType: - return LoadAsset(outputValue, inputValue, context); - case rapidjson::kArrayType: // fall through - case rapidjson::kNullType: // fall through - case rapidjson::kStringType: // fall through - case rapidjson::kFalseType: // fall through - case rapidjson::kTrueType: // fall through - case rapidjson::kNumberType: - return context.Report(JSR::Tasks::ReadField, JSR::Outcomes::Unsupported, - "Unsupported type. Asset can only be read from an object."); - - default: - return context.Report(JSR::Tasks::ReadField, JSR::Outcomes::Unknown, "Unknown json type encountered for Asset."); + if (!outputValue.IsObject()) + { + outputValue.SetObject(); + } + outputValue.AddMember(rapidjson::StringRef("assetId"), AZStd::move(assetIdValue), context.GetJsonAllocator()); } } - JsonSerializationResult::Result AssetJsonSerializer::Store(rapidjson::Value& outputValue, const void* inputValue, - const void* defaultValue, const Uuid& /*valueTypeId*/, JsonSerializerContext& context) { - namespace JSR = JsonSerializationResult; + const AZ::Data::AssetLoadBehavior autoLoadBehavior = instance->GetAutoLoadBehavior(); + const AZ::Data::AssetLoadBehavior defaultAutoLoadBehavior = defaultInstance ? + defaultInstance->GetAutoLoadBehavior() : AZ::Data::AssetLoadBehavior::Default; - const Asset* instance = reinterpret_cast*>(inputValue); - const Asset* defaultInstance = reinterpret_cast*>(defaultValue); - - JSR::ResultCode result(JSR::Tasks::WriteValue); - { - ScopedContextPath subPathId(context, "m_assetId"); - const auto* id = &instance->GetId(); - const auto* defaultId = defaultInstance ? &defaultInstance->GetId() : nullptr; - rapidjson::Value assetIdValue; - result = ContinueStoring(assetIdValue, id, defaultId, azrtti_typeid(), context); - if (result.GetOutcome() == JSR::Outcomes::Success || result.GetOutcome() == JSR::Outcomes::PartialDefaults) - { - if (!outputValue.IsObject()) - { - outputValue.SetObject(); - } - outputValue.AddMember(rapidjson::StringRef("assetId"), AZStd::move(assetIdValue), context.GetJsonAllocator()); - } - } - - { - const AZ::Data::AssetLoadBehavior autoLoadBehavior = instance->GetAutoLoadBehavior(); - const AZ::Data::AssetLoadBehavior defaultAutoLoadBehavior = defaultInstance ? - defaultInstance->GetAutoLoadBehavior() : AZ::Data::AssetLoadBehavior::Default; - - result.Combine( - ContinueStoringToJsonObjectField(outputValue, "loadBehavior", - &autoLoadBehavior, &defaultAutoLoadBehavior, - azrtti_typeid(), context)); - } - - { - ScopedContextPath subPathHint(context, "m_assetHint"); - const AZStd::string* hint = &instance->GetHint(); - const AZStd::string defaultHint; - rapidjson::Value assetHintValue; - JSR::ResultCode resultHint = ContinueStoring(assetHintValue, hint, &defaultHint, azrtti_typeid(), context); - if (resultHint.GetOutcome() == JSR::Outcomes::Success || resultHint.GetOutcome() == JSR::Outcomes::PartialDefaults) - { - if (!outputValue.IsObject()) - { - outputValue.SetObject(); - } - outputValue.AddMember(rapidjson::StringRef("assetHint"), AZStd::move(assetHintValue), context.GetJsonAllocator()); - } - result.Combine(resultHint); - } - - return context.Report(result, - result.GetProcessing() == JSR::Processing::Completed ? "Successfully stored Asset." : "Failed to store Asset."); + result.Combine( + ContinueStoringToJsonObjectField(outputValue, "loadBehavior", + &autoLoadBehavior, &defaultAutoLoadBehavior, + azrtti_typeid(), context)); } - JsonSerializationResult::Result AssetJsonSerializer::LoadAsset(void* outputValue, const rapidjson::Value& inputValue, - JsonDeserializerContext& context) { - namespace JSR = JsonSerializationResult; - - Asset* instance = reinterpret_cast*>(outputValue); - AssetId id; - JSR::ResultCode result(JSR::Tasks::ReadField); - - SerializedAssetTracker* assetTracker = - context.GetMetadata().Find(); - + ScopedContextPath subPathHint(context, "m_assetHint"); + const AZStd::string* hint = &instance->GetHint(); + const AZStd::string defaultHint; + rapidjson::Value assetHintValue; + JSR::ResultCode resultHint = ContinueStoring(assetHintValue, hint, &defaultHint, azrtti_typeid(), context); + if (resultHint.GetOutcome() == JSR::Outcomes::Success || resultHint.GetOutcome() == JSR::Outcomes::PartialDefaults) { - Data::AssetLoadBehavior loadBehavior = instance->GetAutoLoadBehavior(); - - result = - ContinueLoadingFromJsonObjectField(&loadBehavior, - azrtti_typeid(), - inputValue, "loadBehavior", context); - - instance->SetAutoLoadBehavior(loadBehavior); + if (!outputValue.IsObject()) + { + outputValue.SetObject(); + } + outputValue.AddMember(rapidjson::StringRef("assetHint"), AZStd::move(assetHintValue), context.GetJsonAllocator()); } + result.Combine(resultHint); + } - auto it = inputValue.FindMember("assetId"); - if (it != inputValue.MemberEnd()) + return context.Report(result, + result.GetProcessing() == JSR::Processing::Completed ? "Successfully stored Asset." : "Failed to store Asset."); + } + + JsonSerializationResult::Result AssetJsonSerializer::LoadAsset(void* outputValue, const rapidjson::Value& inputValue, + JsonDeserializerContext& context) + { + namespace JSR = JsonSerializationResult; + + Asset* instance = reinterpret_cast*>(outputValue); + AssetId id; + JSR::ResultCode result(JSR::Tasks::ReadField); + + SerializedAssetTracker* assetTracker = + context.GetMetadata().Find(); + + { + Data::AssetLoadBehavior loadBehavior = instance->GetAutoLoadBehavior(); + + result = + ContinueLoadingFromJsonObjectField(&loadBehavior, + azrtti_typeid(), + inputValue, "loadBehavior", context); + + instance->SetAutoLoadBehavior(loadBehavior); + } + + auto it = inputValue.FindMember("assetId"); + if (it != inputValue.MemberEnd()) + { + ScopedContextPath subPath(context, "assetId"); + result.Combine(ContinueLoading(&id, azrtti_typeid(), it->value, context)); + if (!id.m_guid.IsNull()) { - ScopedContextPath subPath(context, "assetId"); - result.Combine(ContinueLoading(&id, azrtti_typeid(), it->value, context)); - if (!id.m_guid.IsNull()) + *instance = AssetManager::Instance().FindOrCreateAsset(id, instance->GetType(), instance->GetAutoLoadBehavior()); + if (!instance->GetId().IsValid()) { - *instance = AssetManager::Instance().FindOrCreateAsset(id, instance->GetType(), instance->GetAutoLoadBehavior()); - if (!instance->GetId().IsValid()) - { - // If the asset failed to be created, FindOrCreateAsset returns an asset instance with a null - // id. To preserve the asset id in the source json, reset the asset to an empty one, but with - // the right id. - const auto loadBehavior = instance->GetAutoLoadBehavior(); - *instance = Asset(id, instance->GetType()); - instance->SetAutoLoadBehavior(loadBehavior); - } + // If the asset failed to be created, FindOrCreateAsset returns an asset instance with a null + // id. To preserve the asset id in the source json, reset the asset to an empty one, but with + // the right id. + const auto loadBehavior = instance->GetAutoLoadBehavior(); + *instance = Asset(id, instance->GetType()); + instance->SetAutoLoadBehavior(loadBehavior); + } - result.Combine(context.Report(result, "Successfully created Asset with id.")); - } - else if (result.GetProcessing() == JSR::Processing::Completed) - { - result.Combine(context.Report(JSR::Tasks::ReadField, JSR::Outcomes::DefaultsUsed, - "Null Asset created.")); - } - else - { - result.Combine(context.Report(result, "Failed to retrieve asset id for Asset.")); - } + result.Combine(context.Report(result, "Successfully created Asset with id.")); + } + else if (result.GetProcessing() == JSR::Processing::Completed) + { + result.Combine(context.Report(JSR::Tasks::ReadField, JSR::Outcomes::DefaultsUsed, + "Null Asset created.")); } else { - result.Combine(context.Report(JSR::Tasks::ReadField, JSR::Outcomes::DefaultsUsed, - "The asset id is missing, so there's not enough information to create an Asset.")); - } - - it = inputValue.FindMember("assetHint"); - if (it != inputValue.MemberEnd()) - { - ScopedContextPath subPath(context, "assetHint"); - AZStd::string hint; - result.Combine(ContinueLoading(&hint, azrtti_typeid(), it->value, context)); - instance->SetHint(AZStd::move(hint)); - } - else - { - result.Combine(context.Report(JSR::Tasks::ReadField, JSR::Outcomes::DefaultsUsed, - "The asset hint is missing for Asset, so it will be left empty.")); - } - - if (assetTracker) - { - assetTracker->FixUpAsset(*instance); - assetTracker->AddAsset(*instance); - } - - bool success = result.GetOutcome() <= JSR::Outcomes::PartialSkip; - bool defaulted = result.GetOutcome() == JSR::Outcomes::DefaultsUsed || result.GetOutcome() == JSR::Outcomes::PartialDefaults; - AZStd::string_view message = - success ? "Successfully loaded information and created instance of Asset." : - defaulted ? "A default id was provided for Asset, so no instance could be created." : - "Not enough information was available to create an instance of Asset or data was corrupted."; - return context.Report(result, message); - } - - void SerializedAssetTracker::SetAssetFixUp(AssetFixUp assetFixUpCallback) - { - m_assetFixUpCallback = AZStd::move(assetFixUpCallback); - } - - void SerializedAssetTracker::FixUpAsset(Asset& asset) - { - if (m_assetFixUpCallback) - { - m_assetFixUpCallback(asset); + result.Combine(context.Report(result, "Failed to retrieve asset id for Asset.")); } } - - void SerializedAssetTracker::AddAsset(Asset asset) + else { - m_serializedAssets.emplace_back(asset); + result.Combine(context.Report(JSR::Tasks::ReadField, JSR::Outcomes::DefaultsUsed, + "The asset id is missing, so there's not enough information to create an Asset.")); } - const AZStd::vector>& SerializedAssetTracker::GetTrackedAssets() const + it = inputValue.FindMember("assetHint"); + if (it != inputValue.MemberEnd()) { - return m_serializedAssets; + ScopedContextPath subPath(context, "assetHint"); + AZStd::string hint; + result.Combine(ContinueLoading(&hint, azrtti_typeid(), it->value, context)); + instance->SetHint(AZStd::move(hint)); + } + else + { + result.Combine(context.Report(JSR::Tasks::ReadField, JSR::Outcomes::DefaultsUsed, + "The asset hint is missing for Asset, so it will be left empty.")); } - AZStd::vector>& SerializedAssetTracker::GetTrackedAssets() + if (assetTracker) { - return m_serializedAssets; + assetTracker->FixUpAsset(*instance); + assetTracker->AddAsset(*instance); } - } // namespace Data -} // namespace AZ + bool success = result.GetOutcome() <= JSR::Outcomes::PartialSkip; + bool defaulted = result.GetOutcome() == JSR::Outcomes::DefaultsUsed || result.GetOutcome() == JSR::Outcomes::PartialDefaults; + AZStd::string_view message = + success ? "Successfully loaded information and created instance of Asset." : + defaulted ? "A default id was provided for Asset, so no instance could be created." : + "Not enough information was available to create an instance of Asset or data was corrupted."; + return context.Report(result, message); + } + + void SerializedAssetTracker::SetAssetFixUp(AssetFixUp assetFixUpCallback) + { + m_assetFixUpCallback = AZStd::move(assetFixUpCallback); + } + + void SerializedAssetTracker::FixUpAsset(Asset& asset) + { + if (m_assetFixUpCallback) + { + m_assetFixUpCallback(asset); + } + } + + void SerializedAssetTracker::AddAsset(Asset asset) + { + m_serializedAssets.emplace_back(asset); + } + + const AZStd::vector>& SerializedAssetTracker::GetTrackedAssets() const + { + return m_serializedAssets; + } + + AZStd::vector>& SerializedAssetTracker::GetTrackedAssets() + { + return m_serializedAssets; + } + +} // namespace AZ::Data diff --git a/Code/Framework/AzCore/AzCore/Asset/AssetManager.cpp b/Code/Framework/AzCore/AzCore/Asset/AssetManager.cpp index b03e1affdc..e419666a32 100644 --- a/Code/Framework/AzCore/AzCore/Asset/AssetManager.cpp +++ b/Code/Framework/AzCore/AzCore/Asset/AssetManager.cpp @@ -27,2169 +27,2166 @@ #include #include -namespace AZ +namespace AZ::Data { - namespace Data + AZ_CVAR(bool, cl_assetLoadWarningEnable, false, nullptr, AZ::ConsoleFunctorFlags::Null, + "Enable warnings that show when AssetHandler::LoadAssetData has exceeded the time set in cl_assetLoadWarningMsThreshold."); + AZ_CVAR(uint32_t, cl_assetLoadWarningMsThreshold, 100, nullptr, AZ::ConsoleFunctorFlags::Null, + "Number of milliseconds that AssetHandler::LoadAssetData can execute for before printing a warning."); + AZ_CVAR(int, cl_assetLoadDelay, 0, nullptr, AZ::ConsoleFunctorFlags::Null, + "Number of milliseconds to artifically delay an asset load."); + AZ_CVAR(bool, cl_assetLoadError, false, nullptr, AZ::ConsoleFunctorFlags::Null, + "Enable failure of all asset loads."); + + static constexpr char kAssetDBInstanceVarName[] = "AssetDatabaseInstance"; + + /* + * This is the base class for Async AssetDatabase jobs + */ + class AssetDatabaseAsyncJob + : public AssetDatabaseJob + , public Job { - AZ_CVAR(bool, cl_assetLoadWarningEnable, false, nullptr, AZ::ConsoleFunctorFlags::Null, - "Enable warnings that show when AssetHandler::LoadAssetData has exceeded the time set in cl_assetLoadWarningMsThreshold."); - AZ_CVAR(uint32_t, cl_assetLoadWarningMsThreshold, 100, nullptr, AZ::ConsoleFunctorFlags::Null, - "Number of milliseconds that AssetHandler::LoadAssetData can execute for before printing a warning."); - AZ_CVAR(int, cl_assetLoadDelay, 0, nullptr, AZ::ConsoleFunctorFlags::Null, - "Number of milliseconds to artifically delay an asset load."); - AZ_CVAR(bool, cl_assetLoadError, false, nullptr, AZ::ConsoleFunctorFlags::Null, - "Enable failure of all asset loads."); - - static constexpr char kAssetDBInstanceVarName[] = "AssetDatabaseInstance"; - - /* - * This is the base class for Async AssetDatabase jobs - */ - class AssetDatabaseAsyncJob - : public AssetDatabaseJob - , public Job + public: + AssetDatabaseAsyncJob(JobContext* jobContext, bool deleteWhenDone, AssetManager* owner, const Asset& asset, AssetHandler* assetHandler) + : AssetDatabaseJob(owner, asset, assetHandler) + , Job(deleteWhenDone, jobContext) { - public: - AssetDatabaseAsyncJob(JobContext* jobContext, bool deleteWhenDone, AssetManager* owner, const Asset& asset, AssetHandler* assetHandler) - : AssetDatabaseJob(owner, asset, assetHandler) - , Job(deleteWhenDone, jobContext) - { - } + } - ~AssetDatabaseAsyncJob() override - { - } - }; - - /** - * Internally allows threads blocking on asset loads to be notified on load completion. - */ - class BlockingAssetLoadEvents - : public EBusTraits + ~AssetDatabaseAsyncJob() override { - public: - ////////////////////////////////////////////////////////////////////////// - // EBusTraits overrides - static const AZ::EBusAddressPolicy AddressPolicy = AZ::EBusAddressPolicy::ById; - using BusIdType = AssetId; - using MutexType = AZStd::recursive_mutex; - - template - struct AssetJobConnectionPolicy - : public EBusConnectionPolicy - { - static void Connect(typename Bus::BusPtr& busPtr, typename Bus::Context& context, typename Bus::HandlerNode& handler, typename Bus::Context::ConnectLockGuard& connectLock, const typename Bus::BusIdType& id = 0) - { - typename Bus::BusIdType actualId = AssetInternal::ResolveAssetId(id); - EBusConnectionPolicy::Connect(busPtr, context, handler, connectLock, actualId); - - // If the asset is loaded or failed already, deliver the status update immediately - // Note that we check IsReady here, ReadyPreNotify must be tested because there is - // a small gap between ReadyPreNotify and Ready where the callback could be missed. - // Also note because the Asset<> reference isn't passed around anywhere, it doesn't matter - // what the AssetLoadBehavior is set to, as it will never make it back to any callers. - Asset assetData(AssetInternal::GetAssetData(actualId, AssetLoadBehavior::Default)); - if (assetData) - { - if (assetData->IsReady() || assetData->IsError()) - { - connectLock.unlock(); - handler->OnLoadComplete(); - } - } - } - }; - - template - using ConnectionPolicy = AssetJobConnectionPolicy; - - virtual void OnLoadComplete() = 0; - virtual void OnLoadCanceled(AssetId assetId) = 0; - }; - - using BlockingAssetLoadBus = EBus; - - /* - * This class processes async AssetDatabase load jobs - */ - class LoadAssetJob - : public AssetDatabaseAsyncJob - { - public: - AZ_CLASS_ALLOCATOR(LoadAssetJob, ThreadPoolAllocator, 0); - - LoadAssetJob(AssetManager* owner, const Asset& asset, - AZStd::shared_ptr dataStream, bool isReload, AZ::IO::IStreamerTypes::RequestStatus requestState, - AssetHandler* handler, const AssetLoadParameters& loadParams, bool signalLoaded) - : AssetDatabaseAsyncJob(JobContext::GetGlobalContext(), true, owner, asset, handler) - , m_dataStream(dataStream) - , m_isReload(isReload) - , m_requestState(requestState) - , m_loadParams(loadParams) - , m_signalLoaded(signalLoaded) - { - AZ_Assert(m_dataStream, "Data stream pointer received through the callback from AZ::IO::Streamer is invalid."); - - AZ_Assert((m_requestState == AZ::IO::IStreamerTypes::RequestStatus::Completed) - || (m_requestState == AZ::IO::IStreamerTypes::RequestStatus::Canceled) - || (m_requestState == AZ::IO::IStreamerTypes::RequestStatus::Failed), - "AssetManager::LoadAssetJob was called with an unexpected streamer state: %i", m_requestState); - } - - ~LoadAssetJob() override - { - } - - void Process() override - { - Asset asset = m_asset.GetStrongReference(); - - // Verify that we didn't somehow get here after the Asset Manager has finished shutting down. - AZ_Assert(AssetManager::IsReady(), "Asset Manager shutdown didn't clean up pending asset loads properly."); - if (!AssetManager::IsReady()) - { - return; - } - - bool shouldCancel = m_owner->ShouldCancelAllActiveJobs() - || !asset // No outstanding references, so cancel the load - || m_requestState == AZ::IO::IStreamerTypes::RequestStatus::Canceled; - - if (shouldCancel) - { - BlockingAssetLoadBus::Event(m_asset.GetId(), &BlockingAssetLoadBus::Events::OnLoadCanceled, m_asset.GetId()); - AssetManagerBus::Broadcast(&AssetManagerBus::Events::OnAssetCanceled, m_asset.GetId()); - } - else - { - - AZ_PROFILE_SCOPE(AzCore, "AZ::Data::LoadAssetJob::Process: %s", - asset.GetHint().c_str()); - - AZ_ASSET_ATTACH_TO_SCOPE(this); - - if (m_owner->ValidateAndRegisterAssetLoading(asset)) - { - LoadAndSignal(asset); - } - } - } - - void LoadAndSignal(Asset& asset) - { - const bool loadSucceeded = LoadData(); - - if (m_signalLoaded && loadSucceeded) - { - AZ_Assert(!m_isReload, "OnAssetDataLoaded signal isn't supported for asset reloads."); - // This asset has preload dependencies, we need to evaluate whether they're all ready before calling PostLoad - AssetLoadBus::Event(asset.GetId(), &AssetLoadBus::Events::OnAssetDataLoaded, asset); - } - else - { - // As long as we don't need to signal preload dependencies, just finish the load whether or not it was successful. - m_owner->PostLoad(asset, loadSucceeded, m_isReload, m_assetHandler); - } - } - - bool LoadData() - { - Asset asset = m_asset.GetStrongReference(); - - if(cl_assetLoadDelay > 0) - { - AZ_PROFILE_SCOPE(AzCore, "LoadData suspended"); - AZStd::this_thread::sleep_for(AZStd::chrono::milliseconds(cl_assetLoadDelay)); - } - - AZ_ASSET_NAMED_SCOPE(asset.GetHint().c_str()); - bool loadedSuccessfully = false; - - if (!cl_assetLoadError && m_requestState == AZ::IO::IStreamerTypes::RequestStatus::Completed) - { - if (m_dataStream->IsFullyLoaded()) - { - AssetHandler::LoadResult result = - m_assetHandler->LoadAssetDataFromStream(asset, m_dataStream, m_loadParams.m_assetLoadFilterCB); - loadedSuccessfully = (result == AssetHandler::LoadResult::LoadComplete); - } - } - - return loadedSuccessfully; - } - - private: - AZStd::shared_ptr m_dataStream; - AssetLoadParameters m_loadParams{}; - AZ::IO::IStreamerTypes::RequestStatus m_requestState{ AZ::IO::IStreamerTypes::RequestStatus::Pending}; - bool m_isReload{ false }; - bool m_signalLoaded{ false }; - }; - - - /** - * Utility class to wait when a blocking load is requested for an asset that's already loading asynchronously. - * Uses the BlockingAssetLoadBus to detect completion, and a semaphore to signal it. - */ - - class WaitForAsset - : public BlockingAssetLoadBus::Handler - { - public: - AZ_CLASS_ALLOCATOR(WaitForAsset, ThreadPoolAllocator, 0); - - - WaitForAsset(const Asset& assetToWaitFor, bool shouldDispatchEvents) - : m_assetData(assetToWaitFor) - , m_shouldDispatchEvents(shouldDispatchEvents) - { - // Track all blocking requests with the AssetManager. This enables load jobs to potentially get routed - // to the thread that's currently blocking waiting on the load job to complete. - AssetManager::Instance().AddBlockingRequest(m_assetData.GetId(), this); - } - - ~WaitForAsset() override - { - // Stop tracking the blocking request, which will ensure that load jobs won't be provided to this instance - // for processing. - AssetManager::Instance().RemoveBlockingRequest(m_assetData.GetId(), this); - - // It shouldn't be possible to destroy a blocking load request before the load job that it's blocked on - // has been processed, so assert if it ever happens, but make sure to process it just in case. - if (m_loadJob) - { - // (If a valid case is ever found where this can occur, it should be safe to remove the assert) - AZ_Assert(false, "Blocking load request is being deleted before it could process the blocking load."); - ProcessLoadJob(); - } - } - - // Provides a blocked load with a LoadJob to process while it's blocking. - // Returns true if it can be queued, false if it can't. - bool QueueAssetLoadJob(LoadAssetJob* loadJob) - { - if(m_shouldDispatchEvents) - { - // Any load job that is going to be dispatching events should not accept additional work since dispatching events - // can lead to more code that's blocking on an asset load which prevents us from finishing the dispatch - // and doing the assigned work. - // Specifically, if dispatching leads to a second block call, the load job will be assigned to the first block call, - // which will never be completed until the second block call is finished. If both blocks are on the same asset, - // we end up deadlocked. - return false; - } - - AZStd::scoped_lock mutexLock(m_loadJobMutex); - - AZ_Assert(!m_loadJob, "Trying to process multiple load jobs for the same asset with the same blocking handler."); - if (!m_loadJob) - { - m_loadJob = loadJob; - m_waitEvent.release(); - return true; - } - - return false; - } - - void OnLoadComplete() override - { - Finish(); - } - - void OnLoadCanceled([[maybe_unused]] const AssetId assetId) override - { - Finish(); - } - - void WaitUntilReady() - { - BusConnect(m_assetData.GetId()); - - Wait(); - - BusDisconnect(m_assetData.GetId()); - } - - protected: - void Wait() - { - AZ_PROFILE_SCOPE(AzCore, "WaitForAsset - %s", m_assetData.GetHint().c_str()); - - // Continue to loop until the load completes. (Most of the time in the loop will be spent in a thread-blocking state) - while (!m_loadCompleted) - { - if (m_shouldDispatchEvents) - { - // The event will wake up either when the load finishes, a load job is queued for processing, or every - // N milliseconds to see if it should dispatch events. - constexpr int MaxWaitBetweenDispatchMs = 1; - while (!m_waitEvent.try_acquire_for(AZStd::chrono::milliseconds(MaxWaitBetweenDispatchMs))) - { - AssetManager::Instance().DispatchEvents(); - } - } - else - { - - // Don't wake up until a load job is queued for processing or the load is entirely finished. - m_waitEvent.acquire(); - } - - // Check to see if any load jobs have been provided for this thread to process. - // (Load jobs will attempt to reuse blocked threads before spinning off new job threads) - ProcessLoadJob(); - } - - // Pump the AssetBus function queue once more after the load has completed in case additional - // functions have been queued between the last call to DispatchEvents and the completion - // of the current load job - if (m_shouldDispatchEvents) - { - AssetManager::Instance().DispatchEvents(); - } - } - - void Finish() - { - AZ_PROFILE_FUNCTION(AzCore); - m_loadCompleted = true; - m_waitEvent.release(); - } - - bool ProcessLoadJob() - { - AZStd::scoped_lock mutexLock(m_loadJobMutex); - bool jobProcessed = false; - - if (m_loadJob) - { - m_loadJob->Process(); - if (m_loadJob->IsAutoDelete()) - { - delete m_loadJob; - } - m_loadJob = nullptr; - jobProcessed = true; - } - - return jobProcessed; - } - - Asset m_assetData; - AZStd::binary_semaphore m_waitEvent; - const bool m_shouldDispatchEvents{ false }; - LoadAssetJob* m_loadJob{ nullptr }; - AZStd::mutex m_loadJobMutex; - AZStd::atomic_bool m_loadCompleted{ false }; - }; - - - /* - * This class processes async AssetDatabase save jobs - */ - class SaveAssetJob - : public AssetDatabaseAsyncJob - { - public: - AZ_CLASS_ALLOCATOR(SaveAssetJob, ThreadPoolAllocator, 0); - - SaveAssetJob(JobContext* jobContext, AssetManager* owner, const Asset& asset, AssetHandler* assetHandler) - : AssetDatabaseAsyncJob(jobContext, true, owner, asset, assetHandler) - { - } - - ~SaveAssetJob() override - { - } - - void Process() override - { - SaveAsset(); - } - - void SaveAsset() - { - auto asset = m_asset.GetStrongReference(); - AZ_PROFILE_FUNCTION(AzCore); - bool isSaved = false; - AssetStreamInfo saveInfo = m_owner->GetSaveStreamInfoForAsset(asset.GetId(), asset.GetType()); - if (saveInfo.IsValid()) - { - IO::FileIOStream stream(saveInfo.m_streamName.c_str(), saveInfo.m_streamFlags); - stream.Seek(saveInfo.m_dataOffset, IO::GenericStream::SeekMode::ST_SEEK_BEGIN); - isSaved = m_assetHandler->SaveAssetData(asset, &stream); - } - // queue broadcast message for delivery on game thread - AssetBus::QueueEvent(asset.GetId(), &AssetBus::Events::OnAssetSaved, asset, isSaved); - } - }; + } + }; + /** + * Internally allows threads blocking on asset loads to be notified on load completion. + */ + class BlockingAssetLoadEvents + : public EBusTraits + { + public: ////////////////////////////////////////////////////////////////////////// - // Globals - EnvironmentVariable AssetManager::s_assetDB = nullptr; - ////////////////////////////////////////////////////////////////////////// + // EBusTraits overrides + static const AZ::EBusAddressPolicy AddressPolicy = AZ::EBusAddressPolicy::ById; + using BusIdType = AssetId; + using MutexType = AZStd::recursive_mutex; - //========================================================================= - // AssetDatabaseJob - // [4/3/2014] - //========================================================================= - AssetDatabaseJob::AssetDatabaseJob(AssetManager* owner, const Asset& asset, AssetHandler* assetHandler) + template + struct AssetJobConnectionPolicy + : public EBusConnectionPolicy { - m_owner = owner; - m_asset = AssetInternal::WeakAsset(asset); - m_assetHandler = assetHandler; - owner->AddJob(this); - } - - //========================================================================= - // ~AssetDatabaseJob - // [4/3/2014] - //========================================================================= - AssetDatabaseJob::~AssetDatabaseJob() - { - // Make sure that the asset reference is cleared out prior to removing the job registration. - // It's possible that clearing this reference will trigger the cleanup code for the asset, so if we wait for the - // destructor to clear it *after* the RemoveJob call, then HasActiveJobsOrStreamerRequests() will be able to return - // false even though the job is still executing asset-related code. - m_asset = {}; - m_owner->RemoveJob(this); - } - - //========================================================================= - // Create - // [6/12/2012] - //========================================================================= - bool AssetManager::Create(const Descriptor& desc) - { - AZ_Assert(!s_assetDB || !s_assetDB.Get(), "AssetManager already created!"); - - if (!s_assetDB) + static void Connect(typename Bus::BusPtr& busPtr, typename Bus::Context& context, typename Bus::HandlerNode& handler, typename Bus::Context::ConnectLockGuard& connectLock, const typename Bus::BusIdType& id = 0) { - s_assetDB = Environment::CreateVariable(kAssetDBInstanceVarName); - } - if (!s_assetDB.Get()) - { - s_assetDB.Set(aznew AssetManager(desc)); - } + typename Bus::BusIdType actualId = AssetInternal::ResolveAssetId(id); + EBusConnectionPolicy::Connect(busPtr, context, handler, connectLock, actualId); - return true; - } - - //========================================================================= - // Destroy - // [6/12/2012] - //========================================================================= - void AssetManager::Destroy() - { - AZ_Assert(s_assetDB, "AssetManager not created!"); - delete (*s_assetDB); - *s_assetDB = nullptr; - } - - //========================================================================= - // IsReady - //========================================================================= - bool AssetManager::IsReady() - { - if (!s_assetDB) - { - s_assetDB = Environment::FindVariable(kAssetDBInstanceVarName); - } - - return s_assetDB && *s_assetDB; - } - - //========================================================================= - // Instance - //========================================================================= - AssetManager& AssetManager::Instance() - { - if (!s_assetDB) - { - s_assetDB = Environment::FindVariable(kAssetDBInstanceVarName); - } - - AZ_Assert(s_assetDB && *s_assetDB, "AssetManager not created!"); - return *(*s_assetDB); - } - - bool AssetManager::SetInstance(AssetManager* assetManager) - { - if (!s_assetDB) - { - s_assetDB = Environment::CreateVariable(kAssetDBInstanceVarName); - } - - // The old instance needs to be null or else it will leak on the assignment. - AZ_Assert(!(*s_assetDB), - "AssetManager::SetInstance was called without first destroying the old instance and setting it to nullptr. " - "This will cause the previous AssetManager instance to leak." ); - - (*s_assetDB) = assetManager; - return true; - } - - //========================================================================= - // AssetDatabase - // [6/12/2012] - //========================================================================= - AssetManager::AssetManager(const AssetManager::Descriptor& desc) - : m_mainThreadId(AZStd::this_thread::get_id()) - , m_debugAssetEvents(AZ::Interface::Get()) - { - (void)desc; - - AssetManagerBus::Handler::BusConnect(); - } - - //========================================================================= - // ~AssetManager - // [6/12/2012] - //========================================================================= - AssetManager::~AssetManager() - { - PrepareShutDown(); - - // Acquire the asset lock to make sure nobody else is trying to do anything fancy with assets - AZStd::scoped_lock assetLock(m_assetMutex); - - while (!m_handlers.empty()) - { - AssetHandlerMap::iterator it = m_handlers.begin(); - AssetHandler* handler = it->second; - UnregisterHandler(handler); - delete handler; - } - - AssetManagerBus::Handler::BusDisconnect(); - } - - //========================================================================= - // DispatchEvents - // [04/02/2014] - //========================================================================= - void AssetManager::DispatchEvents() - { - AZ_PROFILE_FUNCTION(AzCore); - AssetManagerNotificationBus::Broadcast(&AssetManagerNotificationBus::Events::OnAssetEventsDispatchBegin); - while (AssetBus::QueuedEventCount()) - { - AssetBus::ExecuteQueuedEvents(); - } - AssetManagerNotificationBus::Broadcast(&AssetManagerNotificationBus::Events::OnAssetEventsDispatchEnd); - } - - //========================================================================= - void AssetManager::SetAssetInfoUpgradingEnabled(bool enable) - { - m_assetInfoUpgradingEnabled = enable; - } - - bool AssetManager::GetAssetInfoUpgradingEnabled() const - { -#if defined(_RELEASE) - // in release ("FINAL") builds, we never do this. - return false; -#else - return m_assetInfoUpgradingEnabled; -#endif - } - - bool AssetManager::ShouldCancelAllActiveJobs() const - { - return m_cancelAllActiveJobs; - } - - void AssetManager::SetParallelDependentLoadingEnabled(bool enable) - { - m_enableParallelDependentLoading = enable; - } - - bool AssetManager::GetParallelDependentLoadingEnabled() const - { - return m_enableParallelDependentLoading; - } - - void AssetManager::PrepareShutDown() - { - m_cancelAllActiveJobs = true; - - // We want to ensure that no active load jobs are in flight and - // therefore we need to wait till all jobs have completed. Please note that jobs get deleted automatically once they complete. - WaitForActiveJobsAndStreamerRequestsToFinish(); - - m_ownedAssetContainerLookup.clear(); - m_ownedAssetContainers.clear(); - m_assetContainers.clear(); - - // Ensure that there are no queued events on the AssetBus - DispatchEvents(); - } - - void AssetManager::WaitForActiveJobsAndStreamerRequestsToFinish() - { - while (HasActiveJobsOrStreamerRequests()) - { - DispatchEvents(); - AZStd::this_thread::yield(); - } - } - - //========================================================================= - // RegisterHandler - // [7/9/2014] - //========================================================================= - void AssetManager::RegisterHandler(AssetHandler* handler, const AssetType& assetType) - { - AZ_Error("AssetDatabase", handler != nullptr, "Attempting to register a null asset handler!"); - if (handler) - { - if (m_handlers.insert(AZStd::make_pair(assetType, handler)).second) + // If the asset is loaded or failed already, deliver the status update immediately + // Note that we check IsReady here, ReadyPreNotify must be tested because there is + // a small gap between ReadyPreNotify and Ready where the callback could be missed. + // Also note because the Asset<> reference isn't passed around anywhere, it doesn't matter + // what the AssetLoadBehavior is set to, as it will never make it back to any callers. + Asset assetData(AssetInternal::GetAssetData(actualId, AssetLoadBehavior::Default)); + if (assetData) { - handler->m_nHandledTypes++; - } - else - { - AZ_Error("AssetDatabase", false, "Asset type %s already has a handler registered! New registration ignored!", assetType.ToString().c_str()); - } - } - } - - //========================================================================= - // UnregisterHandler - // [7/9/2014] - //========================================================================= - void AssetManager::UnregisterHandler(AssetHandler* handler) - { - AZ_Error("AssetDatabase", handler != nullptr, "Attempting to unregister a null asset handler!"); - if (handler) - { - for (AssetHandlerMap::iterator it = m_handlers.begin(); it != m_handlers.end(); /*++it*/) - { - if (it->second == handler) + if (assetData->IsReady() || assetData->IsError()) { - // When unregistering asset handlers, it's possible that there are still some load jobs that have "finished" but - // haven't destroyed themselves yet by the time the asset handler gets unregistered. LoadAssetJob contains a weak - // asset reference that doesn't clear until the job is destroyed, which happens *after* the OnAssetReady - // notification is triggered. If the thread gets swapped out between the OnAssetReady and the job destruction, - // the job will still be holding onto an asset reference for this asset handler, and it will trigger the - // error below. To ensure that this case doesn't happen, we will instead call - // WaitForActiveJobsAndStreamerRequestsToFinish() to make sure that any in-process jobs have completely cleaned - // themselves up before proceeding forward. - // One example of this pattern occurs in unit tests, where the test loads an asset, validates it, destroys the - // asset, and unregisters the handler, all in rapid succession. This would extremely infrequently - // (~1 per 5000 runs) trigger the error case if we didn't wait for the jobs to finish here. - WaitForActiveJobsAndStreamerRequestsToFinish(); - - { - // this scope is used to control the scope of the lock. - AZStd::lock_guard assetLock(m_assetMutex); - for (const auto &assetEntry : m_assets) - { - // is the handler that handles this type, this handler we're removing? - if (assetEntry.second->m_registeredHandler == handler) - { - AZ_Error("AssetManager", false, "Asset handler for %s is being removed, when assetid %s is still loaded!\n", - assetEntry.second->GetType().ToString().c_str(), - assetEntry.second->GetId().ToString().c_str()); // this will write the name IF AVAILABLE - assetEntry.second->UnregisterWithHandler(); - } - } - } - it = m_handlers.erase(it); - handler->m_nHandledTypes--; - } - else - { - ++it; + connectLock.unlock(); + handler->OnLoadComplete(); } } } + }; + + template + using ConnectionPolicy = AssetJobConnectionPolicy; + + virtual void OnLoadComplete() = 0; + virtual void OnLoadCanceled(AssetId assetId) = 0; + }; + + using BlockingAssetLoadBus = EBus; + + /* + * This class processes async AssetDatabase load jobs + */ + class LoadAssetJob + : public AssetDatabaseAsyncJob + { + public: + AZ_CLASS_ALLOCATOR(LoadAssetJob, ThreadPoolAllocator, 0); + + LoadAssetJob(AssetManager* owner, const Asset& asset, + AZStd::shared_ptr dataStream, bool isReload, AZ::IO::IStreamerTypes::RequestStatus requestState, + AssetHandler* handler, const AssetLoadParameters& loadParams, bool signalLoaded) + : AssetDatabaseAsyncJob(JobContext::GetGlobalContext(), true, owner, asset, handler) + , m_dataStream(dataStream) + , m_isReload(isReload) + , m_requestState(requestState) + , m_loadParams(loadParams) + , m_signalLoaded(signalLoaded) + { + AZ_Assert(m_dataStream, "Data stream pointer received through the callback from AZ::IO::Streamer is invalid."); + + AZ_Assert((m_requestState == AZ::IO::IStreamerTypes::RequestStatus::Completed) + || (m_requestState == AZ::IO::IStreamerTypes::RequestStatus::Canceled) + || (m_requestState == AZ::IO::IStreamerTypes::RequestStatus::Failed), + "AssetManager::LoadAssetJob was called with an unexpected streamer state: %i", m_requestState); } - //========================================================================= - // RegisterCatalog - // [8/27/2012] - //========================================================================= - void AssetManager::RegisterCatalog(AssetCatalog* catalog, const AssetType& assetType) + ~LoadAssetJob() override { - AZ_Error("AssetDatabase", catalog != nullptr, "Attempting to register a null catalog!"); - if (catalog) - { - AZStd::scoped_lock l(m_catalogMutex); - if (m_catalogs.insert(AZStd::make_pair(assetType, catalog)).second == false) - { - AZ_Error("AssetDatabase", false, "Asset type %s already has a catalog registered! New registration ignored!", assetType.ToString().c_str()); - } - } } - //========================================================================= - // UnregisterCatalog - // [8/27/2012] - //========================================================================= - void AssetManager::UnregisterCatalog(AssetCatalog* catalog) + void Process() override { - AZ_Error("AssetDatabase", catalog != nullptr, "Attempting to unregister a null catalog!"); - if (catalog) - { - AZStd::scoped_lock l(m_catalogMutex); - for (AssetCatalogMap::iterator iter = m_catalogs.begin(); iter != m_catalogs.end(); ) - { - if (iter->second == catalog) - { - iter = m_catalogs.erase(iter); - } - else - { - ++iter; - } + Asset asset = m_asset.GetStrongReference(); - } - } - } - - //========================================================================= - // GetHandledAssetTypes - // [6/27/2016] - //========================================================================= - void AssetManager::GetHandledAssetTypes(AssetCatalog* catalog, AZStd::vector& assetTypes) - { - for (AssetCatalogMap::iterator iter = m_catalogs.begin(); iter != m_catalogs.end(); iter++) - { - if (iter->second == catalog) - { - assetTypes.push_back(iter->first); - } - } - } - - void AssetManager::SuspendAssetRelease() - { - ++m_suspendAssetRelease; - } - - void AssetManager::ResumeAssetRelease() - { - if(--m_suspendAssetRelease != 0) + // Verify that we didn't somehow get here after the Asset Manager has finished shutting down. + AZ_Assert(AssetManager::IsReady(), "Asset Manager shutdown didn't clean up pending asset loads properly."); + if (!AssetManager::IsReady()) { return; } - AZStd::scoped_lock assetLock(m_assetMutex); - // First, release any containers that were loading this asset - for (auto asset = m_assets.begin();asset != m_assets.end();) + bool shouldCancel = m_owner->ShouldCancelAllActiveJobs() + || !asset // No outstanding references, so cancel the load + || m_requestState == AZ::IO::IStreamerTypes::RequestStatus::Canceled; + + if (shouldCancel) { - if (asset->second->m_useCount == 0) + BlockingAssetLoadBus::Event(m_asset.GetId(), &BlockingAssetLoadBus::Events::OnLoadCanceled, m_asset.GetId()); + AssetManagerBus::Broadcast(&AssetManagerBus::Events::OnAssetCanceled, m_asset.GetId()); + } + else + { + + AZ_PROFILE_SCOPE(AzCore, "AZ::Data::LoadAssetJob::Process: %s", + asset.GetHint().c_str()); + + AZ_ASSET_ATTACH_TO_SCOPE(this); + + if (m_owner->ValidateAndRegisterAssetLoading(asset)) { - auto releaseAsset = asset->second; - ++asset; - ReleaseAssetContainersForAsset(releaseAsset); + LoadAndSignal(asset); + } + } + } + + void LoadAndSignal(Asset& asset) + { + const bool loadSucceeded = LoadData(); + + if (m_signalLoaded && loadSucceeded) + { + AZ_Assert(!m_isReload, "OnAssetDataLoaded signal isn't supported for asset reloads."); + // This asset has preload dependencies, we need to evaluate whether they're all ready before calling PostLoad + AssetLoadBus::Event(asset.GetId(), &AssetLoadBus::Events::OnAssetDataLoaded, asset); + } + else + { + // As long as we don't need to signal preload dependencies, just finish the load whether or not it was successful. + m_owner->PostLoad(asset, loadSucceeded, m_isReload, m_assetHandler); + } + } + + bool LoadData() + { + Asset asset = m_asset.GetStrongReference(); + + if(cl_assetLoadDelay > 0) + { + AZ_PROFILE_SCOPE(AzCore, "LoadData suspended"); + AZStd::this_thread::sleep_for(AZStd::chrono::milliseconds(cl_assetLoadDelay)); + } + + AZ_ASSET_NAMED_SCOPE(asset.GetHint().c_str()); + bool loadedSuccessfully = false; + + if (!cl_assetLoadError && m_requestState == AZ::IO::IStreamerTypes::RequestStatus::Completed) + { + if (m_dataStream->IsFullyLoaded()) + { + AssetHandler::LoadResult result = + m_assetHandler->LoadAssetDataFromStream(asset, m_dataStream, m_loadParams.m_assetLoadFilterCB); + loadedSuccessfully = (result == AssetHandler::LoadResult::LoadComplete); + } + } + + return loadedSuccessfully; + } + + private: + AZStd::shared_ptr m_dataStream; + AssetLoadParameters m_loadParams{}; + AZ::IO::IStreamerTypes::RequestStatus m_requestState{ AZ::IO::IStreamerTypes::RequestStatus::Pending}; + bool m_isReload{ false }; + bool m_signalLoaded{ false }; + }; + + + /** + * Utility class to wait when a blocking load is requested for an asset that's already loading asynchronously. + * Uses the BlockingAssetLoadBus to detect completion, and a semaphore to signal it. + */ + + class WaitForAsset + : public BlockingAssetLoadBus::Handler + { + public: + AZ_CLASS_ALLOCATOR(WaitForAsset, ThreadPoolAllocator, 0); + + + WaitForAsset(const Asset& assetToWaitFor, bool shouldDispatchEvents) + : m_assetData(assetToWaitFor) + , m_shouldDispatchEvents(shouldDispatchEvents) + { + // Track all blocking requests with the AssetManager. This enables load jobs to potentially get routed + // to the thread that's currently blocking waiting on the load job to complete. + AssetManager::Instance().AddBlockingRequest(m_assetData.GetId(), this); + } + + ~WaitForAsset() override + { + // Stop tracking the blocking request, which will ensure that load jobs won't be provided to this instance + // for processing. + AssetManager::Instance().RemoveBlockingRequest(m_assetData.GetId(), this); + + // It shouldn't be possible to destroy a blocking load request before the load job that it's blocked on + // has been processed, so assert if it ever happens, but make sure to process it just in case. + if (m_loadJob) + { + // (If a valid case is ever found where this can occur, it should be safe to remove the assert) + AZ_Assert(false, "Blocking load request is being deleted before it could process the blocking load."); + ProcessLoadJob(); + } + } + + // Provides a blocked load with a LoadJob to process while it's blocking. + // Returns true if it can be queued, false if it can't. + bool QueueAssetLoadJob(LoadAssetJob* loadJob) + { + if(m_shouldDispatchEvents) + { + // Any load job that is going to be dispatching events should not accept additional work since dispatching events + // can lead to more code that's blocking on an asset load which prevents us from finishing the dispatch + // and doing the assigned work. + // Specifically, if dispatching leads to a second block call, the load job will be assigned to the first block call, + // which will never be completed until the second block call is finished. If both blocks are on the same asset, + // we end up deadlocked. + return false; + } + + AZStd::scoped_lock mutexLock(m_loadJobMutex); + + AZ_Assert(!m_loadJob, "Trying to process multiple load jobs for the same asset with the same blocking handler."); + if (!m_loadJob) + { + m_loadJob = loadJob; + m_waitEvent.release(); + return true; + } + + return false; + } + + void OnLoadComplete() override + { + Finish(); + } + + void OnLoadCanceled([[maybe_unused]] const AssetId assetId) override + { + Finish(); + } + + void WaitUntilReady() + { + BusConnect(m_assetData.GetId()); + + Wait(); + + BusDisconnect(m_assetData.GetId()); + } + + protected: + void Wait() + { + AZ_PROFILE_SCOPE(AzCore, "WaitForAsset - %s", m_assetData.GetHint().c_str()); + + // Continue to loop until the load completes. (Most of the time in the loop will be spent in a thread-blocking state) + while (!m_loadCompleted) + { + if (m_shouldDispatchEvents) + { + // The event will wake up either when the load finishes, a load job is queued for processing, or every + // N milliseconds to see if it should dispatch events. + constexpr int MaxWaitBetweenDispatchMs = 1; + while (!m_waitEvent.try_acquire_for(AZStd::chrono::milliseconds(MaxWaitBetweenDispatchMs))) + { + AssetManager::Instance().DispatchEvents(); + } } else { - ++asset; + + // Don't wake up until a load job is queued for processing or the load is entirely finished. + m_waitEvent.acquire(); } + + // Check to see if any load jobs have been provided for this thread to process. + // (Load jobs will attempt to reuse blocked threads before spinning off new job threads) + ProcessLoadJob(); } - // Second, release the assets themselves - - AZStd::vector assetsToRelease; - - for(auto&& asset : m_assets) + // Pump the AssetBus function queue once more after the load has completed in case additional + // functions have been queued between the last call to DispatchEvents and the completion + // of the current load job + if (m_shouldDispatchEvents) { - if(asset.second->m_weakUseCount == 0) + AssetManager::Instance().DispatchEvents(); + } + } + + void Finish() + { + AZ_PROFILE_FUNCTION(AzCore); + m_loadCompleted = true; + m_waitEvent.release(); + } + + bool ProcessLoadJob() + { + AZStd::scoped_lock mutexLock(m_loadJobMutex); + bool jobProcessed = false; + + if (m_loadJob) + { + m_loadJob->Process(); + if (m_loadJob->IsAutoDelete()) { - // Keep a separate list of assets to release, because releasing them will modify the m_assets list that we're - // currently looping on. - assetsToRelease.push_back(asset.second); + delete m_loadJob; + } + m_loadJob = nullptr; + jobProcessed = true; + } + + return jobProcessed; + } + + Asset m_assetData; + AZStd::binary_semaphore m_waitEvent; + const bool m_shouldDispatchEvents{ false }; + LoadAssetJob* m_loadJob{ nullptr }; + AZStd::mutex m_loadJobMutex; + AZStd::atomic_bool m_loadCompleted{ false }; + }; + + + /* + * This class processes async AssetDatabase save jobs + */ + class SaveAssetJob + : public AssetDatabaseAsyncJob + { + public: + AZ_CLASS_ALLOCATOR(SaveAssetJob, ThreadPoolAllocator, 0); + + SaveAssetJob(JobContext* jobContext, AssetManager* owner, const Asset& asset, AssetHandler* assetHandler) + : AssetDatabaseAsyncJob(jobContext, true, owner, asset, assetHandler) + { + } + + ~SaveAssetJob() override + { + } + + void Process() override + { + SaveAsset(); + } + + void SaveAsset() + { + auto asset = m_asset.GetStrongReference(); + AZ_PROFILE_FUNCTION(AzCore); + bool isSaved = false; + AssetStreamInfo saveInfo = m_owner->GetSaveStreamInfoForAsset(asset.GetId(), asset.GetType()); + if (saveInfo.IsValid()) + { + IO::FileIOStream stream(saveInfo.m_streamName.c_str(), saveInfo.m_streamFlags); + stream.Seek(saveInfo.m_dataOffset, IO::GenericStream::SeekMode::ST_SEEK_BEGIN); + isSaved = m_assetHandler->SaveAssetData(asset, &stream); + } + // queue broadcast message for delivery on game thread + AssetBus::QueueEvent(asset.GetId(), &AssetBus::Events::OnAssetSaved, asset, isSaved); + } + }; + + ////////////////////////////////////////////////////////////////////////// + // Globals + EnvironmentVariable AssetManager::s_assetDB = nullptr; + ////////////////////////////////////////////////////////////////////////// + + //========================================================================= + // AssetDatabaseJob + // [4/3/2014] + //========================================================================= + AssetDatabaseJob::AssetDatabaseJob(AssetManager* owner, const Asset& asset, AssetHandler* assetHandler) + { + m_owner = owner; + m_asset = AssetInternal::WeakAsset(asset); + m_assetHandler = assetHandler; + owner->AddJob(this); + } + + //========================================================================= + // ~AssetDatabaseJob + // [4/3/2014] + //========================================================================= + AssetDatabaseJob::~AssetDatabaseJob() + { + // Make sure that the asset reference is cleared out prior to removing the job registration. + // It's possible that clearing this reference will trigger the cleanup code for the asset, so if we wait for the + // destructor to clear it *after* the RemoveJob call, then HasActiveJobsOrStreamerRequests() will be able to return + // false even though the job is still executing asset-related code. + m_asset = {}; + m_owner->RemoveJob(this); + } + + //========================================================================= + // Create + // [6/12/2012] + //========================================================================= + bool AssetManager::Create(const Descriptor& desc) + { + AZ_Assert(!s_assetDB || !s_assetDB.Get(), "AssetManager already created!"); + + if (!s_assetDB) + { + s_assetDB = Environment::CreateVariable(kAssetDBInstanceVarName); + } + if (!s_assetDB.Get()) + { + s_assetDB.Set(aznew AssetManager(desc)); + } + + return true; + } + + //========================================================================= + // Destroy + // [6/12/2012] + //========================================================================= + void AssetManager::Destroy() + { + AZ_Assert(s_assetDB, "AssetManager not created!"); + delete (*s_assetDB); + *s_assetDB = nullptr; + } + + //========================================================================= + // IsReady + //========================================================================= + bool AssetManager::IsReady() + { + if (!s_assetDB) + { + s_assetDB = Environment::FindVariable(kAssetDBInstanceVarName); + } + + return s_assetDB && *s_assetDB; + } + + //========================================================================= + // Instance + //========================================================================= + AssetManager& AssetManager::Instance() + { + if (!s_assetDB) + { + s_assetDB = Environment::FindVariable(kAssetDBInstanceVarName); + } + + AZ_Assert(s_assetDB && *s_assetDB, "AssetManager not created!"); + return *(*s_assetDB); + } + + bool AssetManager::SetInstance(AssetManager* assetManager) + { + if (!s_assetDB) + { + s_assetDB = Environment::CreateVariable(kAssetDBInstanceVarName); + } + + // The old instance needs to be null or else it will leak on the assignment. + AZ_Assert(!(*s_assetDB), + "AssetManager::SetInstance was called without first destroying the old instance and setting it to nullptr. " + "This will cause the previous AssetManager instance to leak." ); + + (*s_assetDB) = assetManager; + return true; + } + + //========================================================================= + // AssetDatabase + // [6/12/2012] + //========================================================================= + AssetManager::AssetManager(const AssetManager::Descriptor& desc) + : m_mainThreadId(AZStd::this_thread::get_id()) + , m_debugAssetEvents(AZ::Interface::Get()) + { + (void)desc; + + AssetManagerBus::Handler::BusConnect(); + } + + //========================================================================= + // ~AssetManager + // [6/12/2012] + //========================================================================= + AssetManager::~AssetManager() + { + PrepareShutDown(); + + // Acquire the asset lock to make sure nobody else is trying to do anything fancy with assets + AZStd::scoped_lock assetLock(m_assetMutex); + + while (!m_handlers.empty()) + { + AssetHandlerMap::iterator it = m_handlers.begin(); + AssetHandler* handler = it->second; + UnregisterHandler(handler); + delete handler; + } + + AssetManagerBus::Handler::BusDisconnect(); + } + + //========================================================================= + // DispatchEvents + // [04/02/2014] + //========================================================================= + void AssetManager::DispatchEvents() + { + AZ_PROFILE_FUNCTION(AzCore); + AssetManagerNotificationBus::Broadcast(&AssetManagerNotificationBus::Events::OnAssetEventsDispatchBegin); + while (AssetBus::QueuedEventCount()) + { + AssetBus::ExecuteQueuedEvents(); + } + AssetManagerNotificationBus::Broadcast(&AssetManagerNotificationBus::Events::OnAssetEventsDispatchEnd); + } + + //========================================================================= + void AssetManager::SetAssetInfoUpgradingEnabled(bool enable) + { + m_assetInfoUpgradingEnabled = enable; + } + + bool AssetManager::GetAssetInfoUpgradingEnabled() const + { +#if defined(_RELEASE) + // in release ("FINAL") builds, we never do this. + return false; +#else + return m_assetInfoUpgradingEnabled; +#endif + } + + bool AssetManager::ShouldCancelAllActiveJobs() const + { + return m_cancelAllActiveJobs; + } + + void AssetManager::SetParallelDependentLoadingEnabled(bool enable) + { + m_enableParallelDependentLoading = enable; + } + + bool AssetManager::GetParallelDependentLoadingEnabled() const + { + return m_enableParallelDependentLoading; + } + + void AssetManager::PrepareShutDown() + { + m_cancelAllActiveJobs = true; + + // We want to ensure that no active load jobs are in flight and + // therefore we need to wait till all jobs have completed. Please note that jobs get deleted automatically once they complete. + WaitForActiveJobsAndStreamerRequestsToFinish(); + + m_ownedAssetContainerLookup.clear(); + m_ownedAssetContainers.clear(); + m_assetContainers.clear(); + + // Ensure that there are no queued events on the AssetBus + DispatchEvents(); + } + + void AssetManager::WaitForActiveJobsAndStreamerRequestsToFinish() + { + while (HasActiveJobsOrStreamerRequests()) + { + DispatchEvents(); + AZStd::this_thread::yield(); + } + } + + //========================================================================= + // RegisterHandler + // [7/9/2014] + //========================================================================= + void AssetManager::RegisterHandler(AssetHandler* handler, const AssetType& assetType) + { + AZ_Error("AssetDatabase", handler != nullptr, "Attempting to register a null asset handler!"); + if (handler) + { + if (m_handlers.insert(AZStd::make_pair(assetType, handler)).second) + { + handler->m_nHandledTypes++; + } + else + { + AZ_Error("AssetDatabase", false, "Asset type %s already has a handler registered! New registration ignored!", assetType.ToString().c_str()); + } + } + } + + //========================================================================= + // UnregisterHandler + // [7/9/2014] + //========================================================================= + void AssetManager::UnregisterHandler(AssetHandler* handler) + { + AZ_Error("AssetDatabase", handler != nullptr, "Attempting to unregister a null asset handler!"); + if (handler) + { + for (AssetHandlerMap::iterator it = m_handlers.begin(); it != m_handlers.end(); /*++it*/) + { + if (it->second == handler) + { + // When unregistering asset handlers, it's possible that there are still some load jobs that have "finished" but + // haven't destroyed themselves yet by the time the asset handler gets unregistered. LoadAssetJob contains a weak + // asset reference that doesn't clear until the job is destroyed, which happens *after* the OnAssetReady + // notification is triggered. If the thread gets swapped out between the OnAssetReady and the job destruction, + // the job will still be holding onto an asset reference for this asset handler, and it will trigger the + // error below. To ensure that this case doesn't happen, we will instead call + // WaitForActiveJobsAndStreamerRequestsToFinish() to make sure that any in-process jobs have completely cleaned + // themselves up before proceeding forward. + // One example of this pattern occurs in unit tests, where the test loads an asset, validates it, destroys the + // asset, and unregisters the handler, all in rapid succession. This would extremely infrequently + // (~1 per 5000 runs) trigger the error case if we didn't wait for the jobs to finish here. + WaitForActiveJobsAndStreamerRequestsToFinish(); + + { + // this scope is used to control the scope of the lock. + AZStd::lock_guard assetLock(m_assetMutex); + for (const auto &assetEntry : m_assets) + { + // is the handler that handles this type, this handler we're removing? + if (assetEntry.second->m_registeredHandler == handler) + { + AZ_Error("AssetManager", false, "Asset handler for %s is being removed, when assetid %s is still loaded!\n", + assetEntry.second->GetType().ToString().c_str(), + assetEntry.second->GetId().ToString().c_str()); // this will write the name IF AVAILABLE + assetEntry.second->UnregisterWithHandler(); + } + } + } + it = m_handlers.erase(it); + handler->m_nHandledTypes--; + } + else + { + ++it; } } + } + } - for(auto&& asset : assetsToRelease) + //========================================================================= + // RegisterCatalog + // [8/27/2012] + //========================================================================= + void AssetManager::RegisterCatalog(AssetCatalog* catalog, const AssetType& assetType) + { + AZ_Error("AssetDatabase", catalog != nullptr, "Attempting to register a null catalog!"); + if (catalog) + { + AZStd::scoped_lock l(m_catalogMutex); + if (m_catalogs.insert(AZStd::make_pair(assetType, catalog)).second == false) { - bool removeFromHash = asset->IsRegisterReadonlyAndShareable(); - // default creation token implies that the asset was not created by the asset manager and therefore it cannot be in the asset map. - removeFromHash = asset->m_creationToken == s_defaultCreationToken ? false : removeFromHash; + AZ_Error("AssetDatabase", false, "Asset type %s already has a catalog registered! New registration ignored!", assetType.ToString().c_str()); + } + } + } - ReleaseAsset(asset, asset->GetId(), asset->GetType(), removeFromHash, asset->m_creationToken); + //========================================================================= + // UnregisterCatalog + // [8/27/2012] + //========================================================================= + void AssetManager::UnregisterCatalog(AssetCatalog* catalog) + { + AZ_Error("AssetDatabase", catalog != nullptr, "Attempting to unregister a null catalog!"); + if (catalog) + { + AZStd::scoped_lock l(m_catalogMutex); + for (AssetCatalogMap::iterator iter = m_catalogs.begin(); iter != m_catalogs.end(); ) + { + if (iter->second == catalog) + { + iter = m_catalogs.erase(iter); + } + else + { + ++iter; + } + + } + } + } + + //========================================================================= + // GetHandledAssetTypes + // [6/27/2016] + //========================================================================= + void AssetManager::GetHandledAssetTypes(AssetCatalog* catalog, AZStd::vector& assetTypes) + { + for (AssetCatalogMap::iterator iter = m_catalogs.begin(); iter != m_catalogs.end(); iter++) + { + if (iter->second == catalog) + { + assetTypes.push_back(iter->first); + } + } + } + + void AssetManager::SuspendAssetRelease() + { + ++m_suspendAssetRelease; + } + + void AssetManager::ResumeAssetRelease() + { + if(--m_suspendAssetRelease != 0) + { + return; + } + + AZStd::scoped_lock assetLock(m_assetMutex); + // First, release any containers that were loading this asset + for (auto asset = m_assets.begin();asset != m_assets.end();) + { + if (asset->second->m_useCount == 0) + { + auto releaseAsset = asset->second; + ++asset; + ReleaseAssetContainersForAsset(releaseAsset); + } + else + { + ++asset; } } - AssetData::AssetStatus AssetManager::BlockUntilLoadComplete(const Asset& asset) + // Second, release the assets themselves + + AZStd::vector assetsToRelease; + + for(auto&& asset : m_assets) { - if(asset.GetStatus() == AssetData::AssetStatus::NotLoaded) + if(asset.second->m_weakUseCount == 0) { - AZ_Error("AssetManager", false, "BlockUntilLoadComplete must be called after an asset has been queued for load. Asset %s (%s) is not queued for load", - asset.GetHint().c_str(), asset.GetId().ToString().c_str()); + // Keep a separate list of assets to release, because releasing them will modify the m_assets list that we're + // currently looping on. + assetsToRelease.push_back(asset.second); } - else if(!asset.IsReady()) - { - // If this is the main thread we'll need to call DispatchEvents to make sure the events we're waiting on actually fire - // since the main thread is typically responsible for calling DispatchEvents elsewhere - const bool shouldDispatch = AZStd::this_thread::get_id() == m_mainThreadId; - - // Wait for the asset and all queued dependencies to finish loading. - WaitForAsset blockingWait(asset, shouldDispatch); - - blockingWait.WaitUntilReady(); - } - - return asset.GetStatus(); } - //========================================================================= - // FindAsset - //========================================================================= - Asset AssetManager::FindAsset(const AssetId& assetId, AssetLoadBehavior assetReferenceLoadBehavior) + for(auto&& asset : assetsToRelease) { - // Look up the asset id in the catalog, and use the result of that instead. - // If assetId is a legacy id, assetInfo.m_assetId will be the canonical id. Otherwise, assetInfo.m_assetID == assetId. - // This is because only canonical ids are stored in m_assets (see below). - // Only do the look up if upgrading is enabled - AZ::Data::AssetInfo assetInfo; - if (GetAssetInfoUpgradingEnabled()) + bool removeFromHash = asset->IsRegisterReadonlyAndShareable(); + // default creation token implies that the asset was not created by the asset manager and therefore it cannot be in the asset map. + removeFromHash = asset->m_creationToken == s_defaultCreationToken ? false : removeFromHash; + + ReleaseAsset(asset, asset->GetId(), asset->GetType(), removeFromHash, asset->m_creationToken); + } + } + + AssetData::AssetStatus AssetManager::BlockUntilLoadComplete(const Asset& asset) + { + if(asset.GetStatus() == AssetData::AssetStatus::NotLoaded) + { + AZ_Error("AssetManager", false, "BlockUntilLoadComplete must be called after an asset has been queued for load. Asset %s (%s) is not queued for load", + asset.GetHint().c_str(), asset.GetId().ToString().c_str()); + } + else if(!asset.IsReady()) + { + // If this is the main thread we'll need to call DispatchEvents to make sure the events we're waiting on actually fire + // since the main thread is typically responsible for calling DispatchEvents elsewhere + const bool shouldDispatch = AZStd::this_thread::get_id() == m_mainThreadId; + + // Wait for the asset and all queued dependencies to finish loading. + WaitForAsset blockingWait(asset, shouldDispatch); + + blockingWait.WaitUntilReady(); + } + + return asset.GetStatus(); + } + + //========================================================================= + // FindAsset + //========================================================================= + Asset AssetManager::FindAsset(const AssetId& assetId, AssetLoadBehavior assetReferenceLoadBehavior) + { + // Look up the asset id in the catalog, and use the result of that instead. + // If assetId is a legacy id, assetInfo.m_assetId will be the canonical id. Otherwise, assetInfo.m_assetID == assetId. + // This is because only canonical ids are stored in m_assets (see below). + // Only do the look up if upgrading is enabled + AZ::Data::AssetInfo assetInfo; + if (GetAssetInfoUpgradingEnabled()) + { + AssetCatalogRequestBus::BroadcastResult(assetInfo, &AssetCatalogRequestBus::Events::GetAssetInfoById, assetId); + } + + // If the catalog is not available, use the original assetId + const AssetId& assetToFind(assetInfo.m_assetId.IsValid() ? assetInfo.m_assetId : assetId); + + AZStd::scoped_lock assetLock(m_assetMutex); + AssetMap::iterator it = m_assets.find(assetToFind); + if (it != m_assets.end()) + { + Asset asset(assetReferenceLoadBehavior); + asset.SetData(it->second); + + return asset; + } + return Asset(assetReferenceLoadBehavior); + } + + AZStd::pair GetEffectiveDeadlineAndPriority( + const AssetHandler& handler, AssetType assetType, const AssetLoadParameters& loadParams) + { + AZStd::chrono::milliseconds deadline; + AZ::IO::IStreamerTypes::Priority priority; + + handler.GetDefaultAssetLoadPriority(assetType, deadline, priority); + + if (loadParams.m_deadline) + { + deadline = loadParams.m_deadline.value(); + } + + if (loadParams.m_priority) + { + priority = loadParams.m_priority.value(); + } + + return make_pair(deadline, priority); + } + + //========================================================================= + // GetAsset + // [6/19/2012] + //========================================================================= + Asset AssetManager::GetAsset(const AssetId& assetId, const AssetType& assetType, AssetLoadBehavior assetReferenceLoadBehavior, const AssetLoadParameters& loadParams) + { + // If parallel dependent loads are disabled, just try to load the requested asset directly, and let it trigger + // dependent loads as they're encountered. + // Parallel dependent loads are disabled during asset building because there is no guarantee that dependency information + // will be available and complete until after all assets are finished building. + if(!GetParallelDependentLoadingEnabled()) + { + return GetAssetInternal(assetId, assetType, assetReferenceLoadBehavior, loadParams); + } + + // Otherwise, use Asset Containers to load all dependent assets in parallel. + + Asset asset = FindOrCreateAsset(assetId, assetType, assetReferenceLoadBehavior); + + if(!asset || (!loadParams.m_reloadMissingDependencies && asset.IsReady())) + { + // If the asset is already ready, just return it and skip the container + return AZStd::move(asset); + } + + auto container = GetAssetContainer(asset, loadParams); + + AZStd::scoped_lock lock(m_assetContainerMutex); + + m_ownedAssetContainers.insert({ container.get(), container }); + + // Only insert a new entry into m_ownedAssetContainerLookup if one doesn't already exist for this container. + // Because it's a multimap, it is possible to add duplicate entries by mistake. + bool entryExists = false; + auto rangeItr = m_ownedAssetContainerLookup.equal_range(assetId); + for (auto itr = rangeItr.first; itr != rangeItr.second; ++itr) + { + if (itr->second == container.get()) + { + entryExists = true; + break; + } + } + + // Entry for this container doesn't exist yet, so add it. + if (!entryExists) + { + m_ownedAssetContainerLookup.insert({ assetId, container.get() }); + } + + return asset; + } + + Asset AssetManager::GetAssetInternal(const AssetId& assetId, [[maybe_unused]] const AssetType& assetType, + AssetLoadBehavior assetReferenceLoadBehavior, const AssetLoadParameters& loadParams, AssetInfo assetInfo /*= () */, bool signalLoaded /*= false */) + { + AZ_PROFILE_FUNCTION(AzCore); + + AZ_Error("AssetDatabase", assetId.IsValid(), "GetAsset called with invalid asset Id."); + AZ_Error("AssetDatabase", !assetType.IsNull(), "GetAsset called with invalid asset type."); + bool assetMissing = false; + + { + AZ_PROFILE_SCOPE(AzCore, "GetAsset: GetAssetInfo"); + + // Attempt to look up asset info from catalog + // This is so that when assetId is a legacy id, we're operating on the canonical id anyway + if (!assetInfo.m_assetId.IsValid()) { AssetCatalogRequestBus::BroadcastResult(assetInfo, &AssetCatalogRequestBus::Events::GetAssetInfoById, assetId); } - // If the catalog is not available, use the original assetId - const AssetId& assetToFind(assetInfo.m_assetId.IsValid() ? assetInfo.m_assetId : assetId); + // If the asset was found in the catalog, ensure the type infos match + if (assetInfo.m_assetId.IsValid()) + { + AZ_Warning("AssetManager", assetInfo.m_assetType == assetType, + "Requested asset id %s with type %s, but type is actually %s.", + assetId.ToString().c_str(), assetType.ToString().c_str(), + assetInfo.m_assetType.ToString().c_str()); + } + else + { + AZ_Warning("AssetManager", false, "GetAsset called for asset which does not exist in asset catalog and cannot be loaded. Asset may be missing, not processed or moved. AssetId: %s", + assetId.ToString().c_str()); + // If asset not found, use the id and type given. We will create a valid asset, but it will likely get an error + // status below if the asset handler doesn't reroute it to a default asset. + assetInfo.m_assetId = assetId; + assetInfo.m_assetType = assetType; + assetMissing = true; + } + } + + AZ_PROFILE_SCOPE(AzCore, "GetAsset: %s", assetInfo.m_relativePath.c_str()); + AZ_ASSET_NAMED_SCOPE("GetAsset: %s", assetInfo.m_relativePath.c_str()); + + AZStd::shared_ptr dataStream; + AssetStreamInfo loadInfo; + bool triggerAssetErrorNotification = false; + bool wasUnloaded = false; + AssetHandler* handler = nullptr; + AssetData* assetData = nullptr; + Asset asset; // Used to hold a reference while job is dispatched and while outside of the assetMutex lock. + + // Control the scope of the assetMutex lock + { AZStd::scoped_lock assetLock(m_assetMutex); - AssetMap::iterator it = m_assets.find(assetToFind); - if (it != m_assets.end()) + bool isNewEntry = false; + + // check if asset already exists { - Asset asset(assetReferenceLoadBehavior); - asset.SetData(it->second); + AZ_PROFILE_SCOPE(AzCore, "GetAsset: FindAsset"); - return asset; - } - return Asset(assetReferenceLoadBehavior); - } - - AZStd::pair GetEffectiveDeadlineAndPriority( - const AssetHandler& handler, AssetType assetType, const AssetLoadParameters& loadParams) - { - AZStd::chrono::milliseconds deadline; - AZ::IO::IStreamerTypes::Priority priority; - - handler.GetDefaultAssetLoadPriority(assetType, deadline, priority); - - if (loadParams.m_deadline) - { - deadline = loadParams.m_deadline.value(); - } - - if (loadParams.m_priority) - { - priority = loadParams.m_priority.value(); - } - - return make_pair(deadline, priority); - } - - //========================================================================= - // GetAsset - // [6/19/2012] - //========================================================================= - Asset AssetManager::GetAsset(const AssetId& assetId, const AssetType& assetType, AssetLoadBehavior assetReferenceLoadBehavior, const AssetLoadParameters& loadParams) - { - // If parallel dependent loads are disabled, just try to load the requested asset directly, and let it trigger - // dependent loads as they're encountered. - // Parallel dependent loads are disabled during asset building because there is no guarantee that dependency information - // will be available and complete until after all assets are finished building. - if(!GetParallelDependentLoadingEnabled()) - { - return GetAssetInternal(assetId, assetType, assetReferenceLoadBehavior, loadParams); - } - - // Otherwise, use Asset Containers to load all dependent assets in parallel. - - Asset asset = FindOrCreateAsset(assetId, assetType, assetReferenceLoadBehavior); - - if(!asset || (!loadParams.m_reloadMissingDependencies && asset.IsReady())) - { - // If the asset is already ready, just return it and skip the container - return AZStd::move(asset); - } - - auto container = GetAssetContainer(asset, loadParams); - - AZStd::scoped_lock lock(m_assetContainerMutex); - - m_ownedAssetContainers.insert({ container.get(), container }); - - // Only insert a new entry into m_ownedAssetContainerLookup if one doesn't already exist for this container. - // Because it's a multimap, it is possible to add duplicate entries by mistake. - bool entryExists = false; - auto rangeItr = m_ownedAssetContainerLookup.equal_range(assetId); - for (auto itr = rangeItr.first; itr != rangeItr.second; ++itr) - { - if (itr->second == container.get()) + AssetMap::iterator it = m_assets.find(assetInfo.m_assetId); + if (it != m_assets.end()) { - entryExists = true; - break; - } - } - - // Entry for this container doesn't exist yet, so add it. - if (!entryExists) - { - m_ownedAssetContainerLookup.insert({ assetId, container.get() }); - } - - return asset; - } - - Asset AssetManager::GetAssetInternal(const AssetId& assetId, [[maybe_unused]] const AssetType& assetType, - AssetLoadBehavior assetReferenceLoadBehavior, const AssetLoadParameters& loadParams, AssetInfo assetInfo /*= () */, bool signalLoaded /*= false */) - { - AZ_PROFILE_FUNCTION(AzCore); - - AZ_Error("AssetDatabase", assetId.IsValid(), "GetAsset called with invalid asset Id."); - AZ_Error("AssetDatabase", !assetType.IsNull(), "GetAsset called with invalid asset type."); - bool assetMissing = false; - - { - AZ_PROFILE_SCOPE(AzCore, "GetAsset: GetAssetInfo"); - - // Attempt to look up asset info from catalog - // This is so that when assetId is a legacy id, we're operating on the canonical id anyway - if (!assetInfo.m_assetId.IsValid()) - { - AssetCatalogRequestBus::BroadcastResult(assetInfo, &AssetCatalogRequestBus::Events::GetAssetInfoById, assetId); - } - - // If the asset was found in the catalog, ensure the type infos match - if (assetInfo.m_assetId.IsValid()) - { - AZ_Warning("AssetManager", assetInfo.m_assetType == assetType, - "Requested asset id %s with type %s, but type is actually %s.", - assetId.ToString().c_str(), assetType.ToString().c_str(), - assetInfo.m_assetType.ToString().c_str()); + assetData = it->second; + asset.SetData(assetData); } else { - AZ_Warning("AssetManager", false, "GetAsset called for asset which does not exist in asset catalog and cannot be loaded. Asset may be missing, not processed or moved. AssetId: %s", - assetId.ToString().c_str()); - - // If asset not found, use the id and type given. We will create a valid asset, but it will likely get an error - // status below if the asset handler doesn't reroute it to a default asset. - assetInfo.m_assetId = assetId; - assetInfo.m_assetType = assetType; - assetMissing = true; + isNewEntry = true; } } - AZ_PROFILE_SCOPE(AzCore, "GetAsset: %s", assetInfo.m_relativePath.c_str()); - AZ_ASSET_NAMED_SCOPE("GetAsset: %s", assetInfo.m_relativePath.c_str()); - - AZStd::shared_ptr dataStream; - AssetStreamInfo loadInfo; - bool triggerAssetErrorNotification = false; - bool wasUnloaded = false; - AssetHandler* handler = nullptr; - AssetData* assetData = nullptr; - Asset asset; // Used to hold a reference while job is dispatched and while outside of the assetMutex lock. - - // Control the scope of the assetMutex lock { - AZStd::scoped_lock assetLock(m_assetMutex); - bool isNewEntry = false; + AZ_PROFILE_SCOPE(AzCore, "GetAsset: FindAssetHandler"); - // check if asset already exists + // find the asset type handler + AssetHandlerMap::iterator handlerIt = m_handlers.find(assetInfo.m_assetType); + AZ_Error("AssetDatabase", handlerIt != m_handlers.end(), "No handler was registered for this asset [type:%s id:%s]!", + assetInfo.m_assetType.ToString().c_str(), assetInfo.m_assetId.ToString().c_str()); + if (handlerIt != m_handlers.end()) { - AZ_PROFILE_SCOPE(AzCore, "GetAsset: FindAsset"); - - AssetMap::iterator it = m_assets.find(assetInfo.m_assetId); - if (it != m_assets.end()) + // Create the asset ptr and insert it into our asset map. + handler = handlerIt->second; + if (isNewEntry) { - assetData = it->second; - asset.SetData(assetData); - } - else - { - isNewEntry = true; - } - } + AZ_PROFILE_SCOPE(AzCore, "GetAsset: CreateAsset"); - { - AZ_PROFILE_SCOPE(AzCore, "GetAsset: FindAssetHandler"); - - // find the asset type handler - AssetHandlerMap::iterator handlerIt = m_handlers.find(assetInfo.m_assetType); - AZ_Error("AssetDatabase", handlerIt != m_handlers.end(), "No handler was registered for this asset [type:%s id:%s]!", - assetInfo.m_assetType.ToString().c_str(), assetInfo.m_assetId.ToString().c_str()); - if (handlerIt != m_handlers.end()) - { - // Create the asset ptr and insert it into our asset map. - handler = handlerIt->second; - if (isNewEntry) + assetData = handler->CreateAsset(assetInfo.m_assetId, assetInfo.m_assetType); + if (assetData) { - AZ_PROFILE_SCOPE(AzCore, "GetAsset: CreateAsset"); - - assetData = handler->CreateAsset(assetInfo.m_assetId, assetInfo.m_assetType); - if (assetData) - { - assetData->m_assetId = assetInfo.m_assetId; - assetData->m_creationToken = ++m_creationTokenGenerator; - assetData->RegisterWithHandler(handler); - asset.SetData(assetData); - } - else - { - AZ_Error("AssetDatabase", false, "Failed to create asset with (id=%s, type=%s)", - assetInfo.m_assetId.ToString().c_str(), - assetInfo.m_assetType.ToString().c_str()); - } - } - } - } - - if (assetData) - { - if (isNewEntry && assetData->IsRegisterReadonlyAndShareable()) - { - AZ_PROFILE_SCOPE(AzCore, "GetAsset: RegisterAsset"); - m_assets.insert(AZStd::make_pair(assetInfo.m_assetId, assetData)); - } - if (assetData->GetStatus() == AssetData::AssetStatus::NotLoaded) - { - assetData->m_status = AssetData::AssetStatus::Queued; - UpdateDebugStatus(asset); - loadInfo = GetModifiedLoadStreamInfoForAsset(asset, handler); - wasUnloaded = true; - - if (loadInfo.IsValid()) - { - // Create the AssetDataStream instance here so it can claim an asset reference inside the lock (for a total - // count of 2 before starting the load), otherwise the refcount will be 1, and the load could be canceled - // before it is started, which creates state consistency issues. - - dataStream = AZStd::make_shared(handler->GetAssetBufferAllocator()); + assetData->m_assetId = assetInfo.m_assetId; + assetData->m_creationToken = ++m_creationTokenGenerator; + assetData->RegisterWithHandler(handler); + asset.SetData(assetData); } else { - // Asset creation was successful, but asset loading isn't, so trigger the OnAssetError notification - triggerAssetErrorNotification = true; + AZ_Error("AssetDatabase", false, "Failed to create asset with (id=%s, type=%s)", + assetInfo.m_assetId.ToString().c_str(), + assetInfo.m_assetType.ToString().c_str()); } } } } - if (!assetInfo.m_relativePath.empty()) + if (assetData) { - asset.m_assetHint = assetInfo.m_relativePath; - } - - asset.SetAutoLoadBehavior(assetReferenceLoadBehavior); - - // We delay queueing the async file I/O until we release m_assetMutex - if (dataStream) - { - AZ_Assert(loadInfo.IsValid(), "Expected valid stream info when dataStream is valid."); - constexpr bool isReload = false; - QueueAsyncStreamLoad(asset, dataStream, loadInfo, isReload, - handler, loadParams, signalLoaded); - } - else - { - AZ_Assert(!loadInfo.IsValid(), "Expected invalid stream info when dataStream is invalid."); - - if(!wasUnloaded && assetData && assetData->GetStatus() == AssetData::AssetStatus::Queued) + if (isNewEntry && assetData->IsRegisterReadonlyAndShareable()) { - auto&& [deadline, priority] = GetEffectiveDeadlineAndPriority(*handler, assetData->GetType(), loadParams); - - RescheduleStreamerRequest(assetData->GetId(), deadline, priority); + AZ_PROFILE_SCOPE(AzCore, "GetAsset: RegisterAsset"); + m_assets.insert(AZStd::make_pair(assetInfo.m_assetId, assetData)); } - - if (triggerAssetErrorNotification) + if (assetData->GetStatus() == AssetData::AssetStatus::NotLoaded) { - // If the asset was missing from the catalog, we already printed an error, so we can skip printing this one. - if (!assetMissing) + assetData->m_status = AssetData::AssetStatus::Queued; + UpdateDebugStatus(asset); + loadInfo = GetModifiedLoadStreamInfoForAsset(asset, handler); + wasUnloaded = true; + + if (loadInfo.IsValid()) { - AZ_Error("AssetDatabase", false, "Failed to retrieve required information for asset %s (%s)", - assetInfo.m_assetId.ToString().c_str(), - assetInfo.m_relativePath.empty() ? "" : assetInfo.m_relativePath.c_str()); - } + // Create the AssetDataStream instance here so it can claim an asset reference inside the lock (for a total + // count of 2 before starting the load), otherwise the refcount will be 1, and the load could be canceled + // before it is started, which creates state consistency issues. - PostLoad(asset, false, false, handler); - } - } - - return asset; - } - - void AssetManager::UpdateDebugStatus(const AZ::Data::Asset& asset) - { - if(!m_debugAssetEvents) - { - m_debugAssetEvents = AZ::Interface::Get(); - } - - if(m_debugAssetEvents) - { - m_debugAssetEvents->AssetStatusUpdate(asset.GetId(), asset.GetStatus()); - } - } - - Asset AssetManager::FindOrCreateAsset(const AssetId& assetId, const AssetType& assetType, AssetLoadBehavior assetReferenceLoadBehavior) - { - AZStd::scoped_lock asset_lock(m_assetMutex); - - Asset asset = FindAsset(assetId, assetReferenceLoadBehavior); - - if (!asset) - { - asset = CreateAsset(assetId, assetType, assetReferenceLoadBehavior); - } - - return asset; - } - - //========================================================================= - // CreateAsset - // [8/31/2012] - //========================================================================= - Asset AssetManager::CreateAsset(const AssetId& assetId, const AssetType& assetType, AssetLoadBehavior assetReferenceLoadBehavior) - { - AZStd::scoped_lock asset_lock(m_assetMutex); - - // check if asset already exist - AssetMap::iterator it = m_assets.find(assetId); - if (it == m_assets.end()) - { - // find the asset type handler - AssetHandlerMap::iterator handlerIt = m_handlers.find(assetType); - AZ_Error("AssetDatabase", handlerIt != m_handlers.end(), "No handler was registered for this asset (id=%s, type=%s)!", assetId.ToString().c_str(), assetType.ToString().c_str()); - if (handlerIt != m_handlers.end()) - { - // Create the asset ptr - AssetHandler* handler = handlerIt->second; - auto assetData = handler->CreateAsset(assetId, assetType); - AZ_Error("AssetDatabase", assetData, "Failed to create asset with (id=%s, type=%s)", assetId.ToString().c_str(), assetType.ToString().c_str()); - if (assetData) - { - assetData->m_assetId = assetId; - assetData->m_creationToken = ++m_creationTokenGenerator; - assetData->RegisterWithHandler(handler); - if (assetData->IsRegisterReadonlyAndShareable()) - { - m_assets.insert(AZStd::make_pair(assetId, assetData)); - } - - Asset asset(assetReferenceLoadBehavior); - asset.SetData(assetData); - - return asset; - } - } - } - else - { - AZ_Error("AssetDatabase", false, "Asset (id=%s, type=%s) already exists in the database! Asset not created!", assetId.ToString().c_str(), assetType.ToString().c_str()); - } - return Asset(assetReferenceLoadBehavior); - } - - //========================================================================= - // ReleaseAsset - //========================================================================= - void AssetManager::ReleaseAsset(AssetData* asset, AssetId assetId, AssetType assetType, bool removeAssetFromHash, int creationToken) - { - AZ_Assert(asset, "Cannot release NULL AssetPtr!"); - - if(m_suspendAssetRelease) - { - return; - } - - bool wasInAssetsHash = false; // We do support assets that are not registered in the asset manager (with the same ID too). - bool destroyAsset = false; - - if (removeAssetFromHash) - { - AZStd::scoped_lock asset_lock(m_assetMutex); - AssetMap::iterator it = m_assets.find(assetId); - // need to check the count again in here in case - // someone was trying to get the asset on another thread - // Set it to -1 so only this thread will attempt to clean up the cache and delete the asset - int expectedRefCount = 0; - // if the assetId is not in the map or if the identifierId - // do not match it implies that the asset has been already destroyed. - // if the usecount is non zero it implies that we cannot destroy this asset. - if (it != m_assets.end() && it->second->m_creationToken == creationToken && it->second->m_weakUseCount.compare_exchange_strong(expectedRefCount, -1)) - { - wasInAssetsHash = true; - m_assets.erase(it); - destroyAsset = true; - } - } - else - { - // if an asset is not shareable, it implies that that asset is not in the map - // and therefore once its ref count goes to zero it cannot go back up again and therefore we can safely destroy it - destroyAsset = true; - } - - // We have to separate the code which was removing the asset from the m_asset map while being locked, but then actually destroy the asset - // while the lock is not held since destroying the asset while holding the lock can cause a deadlock. - if (destroyAsset) - { - if(m_debugAssetEvents) - { - m_debugAssetEvents->ReleaseAsset(assetId); - } - - // find the asset type handler - AssetHandlerMap::iterator handlerIt = m_handlers.find(assetType); - if (handlerIt != m_handlers.end()) - { - AssetHandler* handler = handlerIt->second; - if (asset) - { - handler->DestroyAsset(asset); - - if (wasInAssetsHash) - { - AssetBus::QueueEvent(assetId, &AssetBus::Events::OnAssetUnloaded, assetId, assetType); - } - } - } - else - { - AZ_Assert(false, "No handler was registered for asset of type %s but it was still in the AssetManager as %s", assetType.ToString().c_str(), asset->GetId().ToString().c_str()); - } - } - } - - void AssetManager::OnAssetUnused(AssetData* asset) - { - // If we're currently suspending asset releases, don't get rid of the asset containers either. - if (m_suspendAssetRelease) - { - return; - } - - ReleaseAssetContainersForAsset(asset); - } - - void AssetManager::ReleaseAssetContainersForAsset(AssetData* asset) - { - // Release any containers that were loading this asset - AZStd::scoped_lock lock(m_assetContainerMutex); - - AssetId assetId = asset->GetId(); - - auto rangeItr = m_ownedAssetContainerLookup.equal_range(assetId); - - for (auto itr = rangeItr.first; itr != rangeItr.second;) - { - AZ_Assert(itr->second->GetContainerAssetId() == assetId, - "Asset container is incorrectly associated with the asset being destroyed."); - itr->second->ClearRootAsset(); - - // Only remove owned asset containers if they aren't currently loading. - // If they *are* currently loading, removing them could cause dependent asset loads that were triggered to - // remain in a perpetual loading state. Instead, leave the containers for now, they will get removed during - // the OnAssetContainerReady callback. - if (!itr->second->IsLoading()) - { - m_ownedAssetContainers.erase(itr->second); - itr = m_ownedAssetContainerLookup.erase(itr); - } - else - { - ++itr; - } - } - } - - //========================================================================= - // SaveAsset - // [9/13/2012] - //========================================================================= - void AssetManager::SaveAsset(const Asset& asset) - { - AssetHandler* handler; - { - // find the asset type handler - AssetHandlerMap::iterator handlerIt = m_handlers.find(asset.GetType()); - AZ_Assert(handlerIt != m_handlers.end(), "No handler was registered for this asset [type:%s id:%s]!", asset.GetType().ToString().c_str(), asset.GetId().ToString().c_str()); - handler = handlerIt->second; - } - - // start the data saving - SaveAssetJob* saveJob = aznew SaveAssetJob(JobContext::GetGlobalContext(), this, asset, handler); - saveJob->Start(); - } - - //========================================================================= - // ReloadAsset - //========================================================================= - void AssetManager::ReloadAsset(const AssetId& assetId, AssetLoadBehavior assetReferenceLoadBehavior, bool isAutoReload) - { - AZStd::scoped_lock assetLock(m_assetMutex); - auto assetIter = m_assets.find(assetId); - - if (assetIter == m_assets.end() || assetIter->second->IsLoading()) - { - // Only existing assets can be reloaded. - return; - } - - auto reloadIter = m_reloads.find(assetId); - if (reloadIter != m_reloads.end()) - { - auto curStatus = reloadIter->second.GetData()->GetStatus(); - // We don't need another reload if we're in "Queued" state because that reload has not actually begun yet. - // If it is in Loading state we want to pass by and allow the new assetData to be created and start the new reload - // As the current load could already be stale - if (curStatus == AssetData::AssetStatus::Queued) - { - return; - } - else if (curStatus == AssetData::AssetStatus::Loading || curStatus == AssetData::AssetStatus::StreamReady) - { - // Don't flood the tick bus - this value will be checked when the asset load completes - reloadIter->second->SetRequeue(true); - return; - } - } - - AssetData* newAssetData = nullptr; - AssetHandler* handler = nullptr; - - bool preventAutoReload = isAutoReload && assetIter->second && !assetIter->second->HandleAutoReload(); - - // when Asset's constructor is called (the one that takes an AssetData), it updates the AssetID - // of the Asset to be the real latest canonical assetId of the asset, so we cache that here instead of have it happen - // implicitly and repeatedly for anything we call. - Asset currentAsset(assetIter->second, AZ::Data::AssetLoadBehavior::Default); - - if (!assetIter->second->IsRegisterReadonlyAndShareable() && !preventAutoReload) - { - // Reloading an "instance asset" is basically a no-op. - // We'll simply notify users to reload the asset. - AssetBus::QueueFunction(&AssetManager::NotifyAssetReloaded, this, currentAsset); - return; - } - else - { - AssetBus::QueueFunction(&AssetManager::NotifyAssetPreReload, this, currentAsset); - } - - // Current AssetData has requested not to be auto reloaded - if (preventAutoReload) - { - return; - } - - // Resolve the asset handler and allocate new data for the reload. - { - AssetHandlerMap::iterator handlerIt = m_handlers.find(currentAsset.GetType()); - AZ_Assert(handlerIt != m_handlers.end(), "No handler was registered for this asset [type:%s id:%s]!", - currentAsset.GetType().ToString().c_str(), currentAsset.GetId().ToString().c_str()); - handler = handlerIt->second; - - newAssetData = handler->CreateAsset(currentAsset.GetId(), currentAsset.GetType()); - if (newAssetData) - { - newAssetData->m_assetId = currentAsset.GetId(); - newAssetData->RegisterWithHandler(handler); - } - } - - if (newAssetData) - { - // For reloaded assets, we need to hold an internal reference to ensure the data - // isn't immediately destroyed. Since reloads are not a shipping feature, we'll - // hold this reference indefinitely, but we'll only hold the most recent one for - // a given asset Id. - - newAssetData->m_status = AssetData::AssetStatus::Queued; - Asset newAsset(newAssetData, assetReferenceLoadBehavior); - - m_reloads[newAsset.GetId()] = newAsset; - - UpdateDebugStatus(newAsset); - - AZStd::shared_ptr dataStream; - AssetStreamInfo loadInfo = GetModifiedLoadStreamInfoForAsset(newAsset, handler); - constexpr bool isReload = true; - if (loadInfo.IsValid()) - { - // Create the AssetDataStream instance here so it can claim an asset reference inside the lock (for a total - // count of 2 before starting the load), otherwise the refcount will be 1, and the load could be canceled - // before it is started, which creates state consistency issues. - - dataStream = AZStd::make_shared(handler->GetAssetBufferAllocator()); - if (dataStream) - { - // Currently there isn't a clear use case for needing to adjust priority for reloads so the default load priority is used - constexpr bool signalLoaded = false; // this is a reload, so don't signal dependent-asset loads - QueueAsyncStreamLoad(newAsset, dataStream, loadInfo, isReload, - handler, {}, signalLoaded); + dataStream = AZStd::make_shared(handler->GetAssetBufferAllocator()); } else { - AZ_Assert(false, "Failed to create dataStream to reload asset %s (%s)", - newAsset.GetId().ToString().c_str(), - newAsset.GetHint().c_str()); + // Asset creation was successful, but asset loading isn't, so trigger the OnAssetError notification + triggerAssetErrorNotification = true; } } - else + } + } + + if (!assetInfo.m_relativePath.empty()) + { + asset.m_assetHint = assetInfo.m_relativePath; + } + + asset.SetAutoLoadBehavior(assetReferenceLoadBehavior); + + // We delay queueing the async file I/O until we release m_assetMutex + if (dataStream) + { + AZ_Assert(loadInfo.IsValid(), "Expected valid stream info when dataStream is valid."); + constexpr bool isReload = false; + QueueAsyncStreamLoad(asset, dataStream, loadInfo, isReload, + handler, loadParams, signalLoaded); + } + else + { + AZ_Assert(!loadInfo.IsValid(), "Expected invalid stream info when dataStream is invalid."); + + if(!wasUnloaded && assetData && assetData->GetStatus() == AssetData::AssetStatus::Queued) + { + auto&& [deadline, priority] = GetEffectiveDeadlineAndPriority(*handler, assetData->GetType(), loadParams); + + RescheduleStreamerRequest(assetData->GetId(), deadline, priority); + } + + if (triggerAssetErrorNotification) + { + // If the asset was missing from the catalog, we already printed an error, so we can skip printing this one. + if (!assetMissing) { - // Asset creation was successful, but asset loading isn't, so trigger the OnAssetError notification AZ_Error("AssetDatabase", false, "Failed to retrieve required information for asset %s (%s)", - newAsset.GetId().ToString().c_str(), - newAsset.GetHint().c_str()); - - constexpr bool loadSucceeded = false; - AssetManager::Instance().PostLoad(newAsset, loadSucceeded, isReload, handler); + assetInfo.m_assetId.ToString().c_str(), + assetInfo.m_relativePath.empty() ? "" : assetInfo.m_relativePath.c_str()); } + PostLoad(asset, false, false, handler); } } - //========================================================================= - // ReloadAssetFromData - //========================================================================= - void AssetManager::ReloadAssetFromData(const Asset& asset) + return asset; + } + + void AssetManager::UpdateDebugStatus(const AZ::Data::Asset& asset) + { + if(!m_debugAssetEvents) { - bool shouldAssignAssetData = false; - - { - AZ_Assert(asset.Get(), "Asset data for reload is missing."); - AZStd::scoped_lock assetLock(m_assetMutex); - AZ_Assert( - m_assets.find(asset.GetId()) != m_assets.end(), - "Unable to reload asset %s because it's not in the AssetManager's asset list.", asset.ToString().c_str()); - AZ_Assert( - m_assets.find(asset.GetId()) == m_assets.end() || - asset->RTTI_GetType() == m_assets.find(asset.GetId())->second->RTTI_GetType(), - "New and old data types are mismatched!"); - - auto found = m_assets.find(asset.GetId()); - if ((found == m_assets.end()) || (asset->RTTI_GetType() != found->second->RTTI_GetType())) - { - return; // this will just lead to crashes down the line and the above asserts cover this. - } - - AssetData* newData = asset.Get(); - - if (found->second != newData) - { - // Notify users that we are about to change asset - AssetBus::Event(asset.GetId(), &AssetBus::Events::OnAssetPreReload, asset); - - // Resolve the asset handler and account for the new asset instance. - { - [[maybe_unused]] AssetHandlerMap::iterator handlerIt = m_handlers.find(newData->GetType()); - AZ_Assert( - handlerIt != m_handlers.end(), "No handler was registered for this asset [type:%s id:%s]!", - newData->GetType().ToString().c_str(), newData->GetId().ToString().c_str()); - } - - shouldAssignAssetData = true; - } - } - - // We specifically perform this outside of the m_assetMutex lock so that the lock isn't held at the point that - // OnAssetReload is triggered inside of AssignAssetData. Otherwise, we open up a high potential for deadlocks. - if (shouldAssignAssetData) - { - AssignAssetData(asset); - } + m_debugAssetEvents = AZ::Interface::Get(); } - //========================================================================= - // GetHandler - //========================================================================= - AssetHandler* AssetManager::GetHandler(const AssetType& assetType) + if(m_debugAssetEvents) { - auto handlerEntry = m_handlers.find(assetType); - if (handlerEntry != m_handlers.end()) - { - return handlerEntry->second; - } - return nullptr; + m_debugAssetEvents->AssetStatusUpdate(asset.GetId(), asset.GetStatus()); + } + } + + Asset AssetManager::FindOrCreateAsset(const AssetId& assetId, const AssetType& assetType, AssetLoadBehavior assetReferenceLoadBehavior) + { + AZStd::scoped_lock asset_lock(m_assetMutex); + + Asset asset = FindAsset(assetId, assetReferenceLoadBehavior); + + if (!asset) + { + asset = CreateAsset(assetId, assetType, assetReferenceLoadBehavior); } - //========================================================================= - // AssignAssetData - //========================================================================= - void AssetManager::AssignAssetData(const Asset& asset) + return asset; + } + + //========================================================================= + // CreateAsset + // [8/31/2012] + //========================================================================= + Asset AssetManager::CreateAsset(const AssetId& assetId, const AssetType& assetType, AssetLoadBehavior assetReferenceLoadBehavior) + { + AZStd::scoped_lock asset_lock(m_assetMutex); + + // check if asset already exist + AssetMap::iterator it = m_assets.find(assetId); + if (it == m_assets.end()) { - AZ_Assert(asset.Get(), "Reloaded data is missing!"); - - const AssetId& assetId = asset.GetId(); - - asset->m_status = AssetData::AssetStatus::Ready; - UpdateDebugStatus(asset); - - if (asset->IsRegisterReadonlyAndShareable()) + // find the asset type handler + AssetHandlerMap::iterator handlerIt = m_handlers.find(assetType); + AZ_Error("AssetDatabase", handlerIt != m_handlers.end(), "No handler was registered for this asset (id=%s, type=%s)!", assetId.ToString().c_str(), assetType.ToString().c_str()); + if (handlerIt != m_handlers.end()) { - bool requeue{ false }; + // Create the asset ptr + AssetHandler* handler = handlerIt->second; + auto assetData = handler->CreateAsset(assetId, assetType); + AZ_Error("AssetDatabase", assetData, "Failed to create asset with (id=%s, type=%s)", assetId.ToString().c_str(), assetType.ToString().c_str()); + if (assetData) { - AZStd::scoped_lock assetLock(m_assetMutex); - auto found = m_assets.find(assetId); - AZ_Assert(found == m_assets.end() || asset.Get()->RTTI_GetType() == found->second->RTTI_GetType(), - "New and old data types are mismatched!"); - - // if we are here it implies that we have two assets with the same asset id, and we are - // trying to replace the old asset with the new asset which was not created using the asset manager system. - // In this scenario if any other system have cached the old asset then the asset wont be destroyed - // because of creation token mismatch when it's ref count finally goes to zero. Since the old asset is not shareable anymore - // manually setting the creationToken to default creation token will ensure that the asset is destroyed correctly. - asset.m_assetData->m_creationToken = ++m_creationTokenGenerator; - if (found != m_assets.end()) + assetData->m_assetId = assetId; + assetData->m_creationToken = ++m_creationTokenGenerator; + assetData->RegisterWithHandler(handler); + if (assetData->IsRegisterReadonlyAndShareable()) { - found->second->m_creationToken = AZ::Data::s_defaultCreationToken; + m_assets.insert(AZStd::make_pair(assetId, assetData)); } - // Held references to old data are retained, but replace the entry in the DB for future requests. - // Fire an OnAssetReloaded message so listeners can react to the new data. - m_assets[assetId] = asset.Get(); + Asset asset(assetReferenceLoadBehavior); + asset.SetData(assetData); - // Release the reload reference. - auto reloadInfo = m_reloads.find(assetId); - if (reloadInfo != m_reloads.end()) - { - requeue = reloadInfo->second->GetRequeue(); - m_reloads.erase(reloadInfo); - } + return asset; } - // Call reloaded before we can call ReloadAsset below to preserve order - AssetBus::Event(assetId, &AssetBus::Events::OnAssetReloaded, asset); - // Release the lock before we call reload - if (requeue) + } + } + else + { + AZ_Error("AssetDatabase", false, "Asset (id=%s, type=%s) already exists in the database! Asset not created!", assetId.ToString().c_str(), assetType.ToString().c_str()); + } + return Asset(assetReferenceLoadBehavior); + } + + //========================================================================= + // ReleaseAsset + //========================================================================= + void AssetManager::ReleaseAsset(AssetData* asset, AssetId assetId, AssetType assetType, bool removeAssetFromHash, int creationToken) + { + AZ_Assert(asset, "Cannot release NULL AssetPtr!"); + + if(m_suspendAssetRelease) + { + return; + } + + bool wasInAssetsHash = false; // We do support assets that are not registered in the asset manager (with the same ID too). + bool destroyAsset = false; + + if (removeAssetFromHash) + { + AZStd::scoped_lock asset_lock(m_assetMutex); + AssetMap::iterator it = m_assets.find(assetId); + // need to check the count again in here in case + // someone was trying to get the asset on another thread + // Set it to -1 so only this thread will attempt to clean up the cache and delete the asset + int expectedRefCount = 0; + // if the assetId is not in the map or if the identifierId + // do not match it implies that the asset has been already destroyed. + // if the usecount is non zero it implies that we cannot destroy this asset. + if (it != m_assets.end() && it->second->m_creationToken == creationToken && it->second->m_weakUseCount.compare_exchange_strong(expectedRefCount, -1)) + { + wasInAssetsHash = true; + m_assets.erase(it); + destroyAsset = true; + } + } + else + { + // if an asset is not shareable, it implies that that asset is not in the map + // and therefore once its ref count goes to zero it cannot go back up again and therefore we can safely destroy it + destroyAsset = true; + } + + // We have to separate the code which was removing the asset from the m_asset map while being locked, but then actually destroy the asset + // while the lock is not held since destroying the asset while holding the lock can cause a deadlock. + if (destroyAsset) + { + if(m_debugAssetEvents) + { + m_debugAssetEvents->ReleaseAsset(assetId); + } + + // find the asset type handler + AssetHandlerMap::iterator handlerIt = m_handlers.find(assetType); + if (handlerIt != m_handlers.end()) + { + AssetHandler* handler = handlerIt->second; + if (asset) { - ReloadAsset(assetId, asset.GetAutoLoadBehavior()); + handler->DestroyAsset(asset); + + if (wasInAssetsHash) + { + AssetBus::QueueEvent(assetId, &AssetBus::Events::OnAssetUnloaded, assetId, assetType); + } } } else { - AssetBus::Event(assetId, &AssetBus::Events::OnAssetReloaded, asset); + AZ_Assert(false, "No handler was registered for asset of type %s but it was still in the AssetManager as %s", assetType.ToString().c_str(), asset->GetId().ToString().c_str()); + } + } + } + + void AssetManager::OnAssetUnused(AssetData* asset) + { + // If we're currently suspending asset releases, don't get rid of the asset containers either. + if (m_suspendAssetRelease) + { + return; + } + + ReleaseAssetContainersForAsset(asset); + } + + void AssetManager::ReleaseAssetContainersForAsset(AssetData* asset) + { + // Release any containers that were loading this asset + AZStd::scoped_lock lock(m_assetContainerMutex); + + AssetId assetId = asset->GetId(); + + auto rangeItr = m_ownedAssetContainerLookup.equal_range(assetId); + + for (auto itr = rangeItr.first; itr != rangeItr.second;) + { + AZ_Assert(itr->second->GetContainerAssetId() == assetId, + "Asset container is incorrectly associated with the asset being destroyed."); + itr->second->ClearRootAsset(); + + // Only remove owned asset containers if they aren't currently loading. + // If they *are* currently loading, removing them could cause dependent asset loads that were triggered to + // remain in a perpetual loading state. Instead, leave the containers for now, they will get removed during + // the OnAssetContainerReady callback. + if (!itr->second->IsLoading()) + { + m_ownedAssetContainers.erase(itr->second); + itr = m_ownedAssetContainerLookup.erase(itr); + } + else + { + ++itr; + } + } + } + + //========================================================================= + // SaveAsset + // [9/13/2012] + //========================================================================= + void AssetManager::SaveAsset(const Asset& asset) + { + AssetHandler* handler; + { + // find the asset type handler + AssetHandlerMap::iterator handlerIt = m_handlers.find(asset.GetType()); + AZ_Assert(handlerIt != m_handlers.end(), "No handler was registered for this asset [type:%s id:%s]!", asset.GetType().ToString().c_str(), asset.GetId().ToString().c_str()); + handler = handlerIt->second; + } + + // start the data saving + SaveAssetJob* saveJob = aznew SaveAssetJob(JobContext::GetGlobalContext(), this, asset, handler); + saveJob->Start(); + } + + //========================================================================= + // ReloadAsset + //========================================================================= + void AssetManager::ReloadAsset(const AssetId& assetId, AssetLoadBehavior assetReferenceLoadBehavior, bool isAutoReload) + { + AZStd::scoped_lock assetLock(m_assetMutex); + auto assetIter = m_assets.find(assetId); + + if (assetIter == m_assets.end() || assetIter->second->IsLoading()) + { + // Only existing assets can be reloaded. + return; + } + + auto reloadIter = m_reloads.find(assetId); + if (reloadIter != m_reloads.end()) + { + auto curStatus = reloadIter->second.GetData()->GetStatus(); + // We don't need another reload if we're in "Queued" state because that reload has not actually begun yet. + // If it is in Loading state we want to pass by and allow the new assetData to be created and start the new reload + // As the current load could already be stale + if (curStatus == AssetData::AssetStatus::Queued) + { + return; + } + else if (curStatus == AssetData::AssetStatus::Loading || curStatus == AssetData::AssetStatus::StreamReady) + { + // Don't flood the tick bus - this value will be checked when the asset load completes + reloadIter->second->SetRequeue(true); + return; } } - //========================================================================= - // GetModifiedLoadStreamInfoForAsset - //========================================================================= - AssetStreamInfo AssetManager::GetModifiedLoadStreamInfoForAsset(const Asset& asset, AssetHandler* handler) + AssetData* newAssetData = nullptr; + AssetHandler* handler = nullptr; + + bool preventAutoReload = isAutoReload && assetIter->second && !assetIter->second->HandleAutoReload(); + + // when Asset's constructor is called (the one that takes an AssetData), it updates the AssetID + // of the Asset to be the real latest canonical assetId of the asset, so we cache that here instead of have it happen + // implicitly and repeatedly for anything we call. + Asset currentAsset(assetIter->second, AZ::Data::AssetLoadBehavior::Default); + + if (!assetIter->second->IsRegisterReadonlyAndShareable() && !preventAutoReload) { - AssetStreamInfo loadInfo = GetLoadStreamInfoForAsset(asset.GetId(), asset.GetType()); - if (!loadInfo.IsValid()) - { - // opportunity for handler to do default substitution: - AZ::Data::AssetId fallbackId = handler->AssetMissingInCatalog(asset); - if (fallbackId.IsValid()) - { - loadInfo = GetLoadStreamInfoForAsset(fallbackId, asset.GetType()); - } - } - - // Give the handler an opportunity to modify any of the load info before creating the dataStream. - handler->GetCustomAssetStreamInfoForLoad(loadInfo); - - return loadInfo; + // Reloading an "instance asset" is basically a no-op. + // We'll simply notify users to reload the asset. + AssetBus::QueueFunction(&AssetManager::NotifyAssetReloaded, this, currentAsset); + return; + } + else + { + AssetBus::QueueFunction(&AssetManager::NotifyAssetPreReload, this, currentAsset); } - //========================================================================= - // QueueAsyncStreamLoad - //========================================================================= - void AssetManager::QueueAsyncStreamLoad(Asset asset, AZStd::shared_ptr dataStream, - const AZ::Data::AssetStreamInfo& streamInfo, bool isReload, - AssetHandler* handler, const AssetLoadParameters& loadParams, bool signalLoaded) + // Current AssetData has requested not to be auto reloaded + if (preventAutoReload) { - AZ_PROFILE_FUNCTION(AzCore); + return; + } - // Set up the callback that will process the asset data once the raw file load is finished. - // The callback is declared as mutable so that we can clear weakAsset within the callback. The refcount in weakAsset - // can trigger an AssetManager::ReleaseAsset call. If this occurs during lambda cleanup, it could happen at any time - // on the file streamer thread as streamer requests get recycled, including during (or after) AssetManager shutdown. - // By controlling when the refcount is changed, we can ensure that it occurs while the AssetManager is still active. - auto assetDataStreamCallback = [this, loadParams, handler, dataStream, signalLoaded, isReload, - weakAsset = AssetInternal::WeakAsset(asset)] - (AZ::IO::IStreamerTypes::RequestStatus status) mutable + // Resolve the asset handler and allocate new data for the reload. + { + AssetHandlerMap::iterator handlerIt = m_handlers.find(currentAsset.GetType()); + AZ_Assert(handlerIt != m_handlers.end(), "No handler was registered for this asset [type:%s id:%s]!", + currentAsset.GetType().ToString().c_str(), currentAsset.GetId().ToString().c_str()); + handler = handlerIt->second; + + newAssetData = handler->CreateAsset(currentAsset.GetId(), currentAsset.GetType()); + if (newAssetData) { - auto assetId = weakAsset.GetId(); + newAssetData->m_assetId = currentAsset.GetId(); + newAssetData->RegisterWithHandler(handler); + } + } - Asset loadingAsset = weakAsset.GetStrongReference(); + if (newAssetData) + { + // For reloaded assets, we need to hold an internal reference to ensure the data + // isn't immediately destroyed. Since reloads are not a shipping feature, we'll + // hold this reference indefinitely, but we'll only hold the most recent one for + // a given asset Id. - if (loadingAsset) + newAssetData->m_status = AssetData::AssetStatus::Queued; + Asset newAsset(newAssetData, assetReferenceLoadBehavior); + + m_reloads[newAsset.GetId()] = newAsset; + + UpdateDebugStatus(newAsset); + + AZStd::shared_ptr dataStream; + AssetStreamInfo loadInfo = GetModifiedLoadStreamInfoForAsset(newAsset, handler); + constexpr bool isReload = true; + if (loadInfo.IsValid()) + { + // Create the AssetDataStream instance here so it can claim an asset reference inside the lock (for a total + // count of 2 before starting the load), otherwise the refcount will be 1, and the load could be canceled + // before it is started, which creates state consistency issues. + + dataStream = AZStd::make_shared(handler->GetAssetBufferAllocator()); + if (dataStream) { - AZ_PROFILE_SCOPE(AzCore, "AZ::Data::LoadAssetStreamerCallback %s", - loadingAsset.GetHint().c_str()); - { - AZStd::scoped_lock assetLock(m_assetMutex); - AssetData* data = loadingAsset.Get(); - if (data->GetStatus() != AssetData::AssetStatus::Queued) - { - AZ_Warning("AssetManager", false, "Asset %s no longer in Queued state, abandoning load", loadingAsset.GetId().ToString().c_str()); - return; - } - data->m_status = AssetData::AssetStatus::StreamReady; - } - - // The callback from AZ Streamer blocks the streaming thread until this function completes. To minimize the overhead, - // do the majority of the work in a separate job. - auto loadJob = aznew LoadAssetJob(this, loadingAsset, - dataStream, isReload, status, handler, loadParams, signalLoaded); - - bool jobQueued = false; - - // If there's already an active blocking request waiting for this load to complete, let that thread handle - // the load itself instead of consuming a second thread. - { - AZStd::scoped_lock requestLock(m_activeBlockingRequestMutex); - auto range = m_activeBlockingRequests.equal_range(assetId); - for(auto blockingRequest = range.first; blockingRequest != range.second; ++blockingRequest) - { - if(blockingRequest->second->QueueAssetLoadJob(loadJob)) - { - jobQueued = true; - break; - } - } - } - - if (!jobQueued) - { - loadJob->Start(); - } + // Currently there isn't a clear use case for needing to adjust priority for reloads so the default load priority is used + constexpr bool signalLoaded = false; // this is a reload, so don't signal dependent-asset loads + QueueAsyncStreamLoad(newAsset, dataStream, loadInfo, isReload, + handler, {}, signalLoaded); } else { - BlockingAssetLoadBus::Event(assetId, &BlockingAssetLoadBus::Events::OnLoadCanceled, assetId); - AssetManagerBus::Broadcast(&AssetManagerBus::Events::OnAssetCanceled, assetId); + AZ_Assert(false, "Failed to create dataStream to reload asset %s (%s)", + newAsset.GetId().ToString().c_str(), + newAsset.GetHint().c_str()); + } + } + else + { + // Asset creation was successful, but asset loading isn't, so trigger the OnAssetError notification + AZ_Error("AssetDatabase", false, "Failed to retrieve required information for asset %s (%s)", + newAsset.GetId().ToString().c_str(), + newAsset.GetHint().c_str()); + + constexpr bool loadSucceeded = false; + AssetManager::Instance().PostLoad(newAsset, loadSucceeded, isReload, handler); + } + + } + } + + //========================================================================= + // ReloadAssetFromData + //========================================================================= + void AssetManager::ReloadAssetFromData(const Asset& asset) + { + bool shouldAssignAssetData = false; + + { + AZ_Assert(asset.Get(), "Asset data for reload is missing."); + AZStd::scoped_lock assetLock(m_assetMutex); + AZ_Assert( + m_assets.find(asset.GetId()) != m_assets.end(), + "Unable to reload asset %s because it's not in the AssetManager's asset list.", asset.ToString().c_str()); + AZ_Assert( + m_assets.find(asset.GetId()) == m_assets.end() || + asset->RTTI_GetType() == m_assets.find(asset.GetId())->second->RTTI_GetType(), + "New and old data types are mismatched!"); + + auto found = m_assets.find(asset.GetId()); + if ((found == m_assets.end()) || (asset->RTTI_GetType() != found->second->RTTI_GetType())) + { + return; // this will just lead to crashes down the line and the above asserts cover this. + } + + AssetData* newData = asset.Get(); + + if (found->second != newData) + { + // Notify users that we are about to change asset + AssetBus::Event(asset.GetId(), &AssetBus::Events::OnAssetPreReload, asset); + + // Resolve the asset handler and account for the new asset instance. + { + [[maybe_unused]] AssetHandlerMap::iterator handlerIt = m_handlers.find(newData->GetType()); + AZ_Assert( + handlerIt != m_handlers.end(), "No handler was registered for this asset [type:%s id:%s]!", + newData->GetType().ToString().c_str(), newData->GetId().ToString().c_str()); } - // *After* the loadJob has been created, clear our asset references and remove the active streamer requests. - // This needs to happen after the loadJob creation to ensure that on AssetManager shutdown, there are no brief - // windows in which requests and/or jobs are still active after we've removed our tracking of the requests and jobs. - - // Also, if the asset references don't get cleared until after the callback completes, or at some indeterminate later - // time when the File Streamer cleans up the file requests (for the weakAsset lambda parameter), then it's possible that - // they will trigger a ReleaseAsset call sometime after the AssetManager has begun to shut down, which can lead to - // race conditions. - - // Make sure the streamer request is removed first before the asset is released - // If the asset is released first it could lead to a race condition where another thread starts loading the asset - // again and attempts to add a new streamer request with the same ID before the old one has been removed, causing - // that load request to fail - RemoveActiveStreamerRequest(assetId); - weakAsset = {}; - loadingAsset.Reset(); - }; - - auto&& [deadline, priority] = GetEffectiveDeadlineAndPriority(*handler, asset.GetType(), loadParams); - - // Track the load request and queue the asset data stream load. - AddActiveStreamerRequest(asset.GetId(), dataStream); - dataStream->Open( - streamInfo.m_streamName, - streamInfo.m_dataOffset, - streamInfo.m_dataLen, - deadline, priority, assetDataStreamCallback); + shouldAssignAssetData = true; + } } - //========================================================================= - // NotifyAssetReady - //========================================================================= - void AssetManager::NotifyAssetReady(Asset asset) - { - AssetData* data = asset.Get(); - AZ_Assert(data, "NotifyAssetReady: asset is missing info!"); - data->m_status = AssetData::AssetStatus::Ready; - - AssetBus::Event(asset.GetId(), &AssetBus::Events::OnAssetReady, asset); - } - - //========================================================================= - // NotifyAssetPreReload - //========================================================================= - void AssetManager::NotifyAssetPreReload(Asset asset) - { - AssetBus::Event(asset.GetId(), &AssetBus::Events::OnAssetPreReload, asset); - } - - //========================================================================= - // NotifyAssetReloaded - //========================================================================= - void AssetManager::NotifyAssetReloaded(Asset asset) + // We specifically perform this outside of the m_assetMutex lock so that the lock isn't held at the point that + // OnAssetReload is triggered inside of AssignAssetData. Otherwise, we open up a high potential for deadlocks. + if (shouldAssignAssetData) { AssignAssetData(asset); } + } - //========================================================================= - // NotifyAssetReloaded - //========================================================================= - void AssetManager::NotifyAssetReloadError(Asset asset) + //========================================================================= + // GetHandler + //========================================================================= + AssetHandler* AssetManager::GetHandler(const AssetType& assetType) + { + auto handlerEntry = m_handlers.find(assetType); + if (handlerEntry != m_handlers.end()) { - // Failed reloads have no side effects. Just notify observers (error reporting, etc). + return handlerEntry->second; + } + return nullptr; + } + + //========================================================================= + // AssignAssetData + //========================================================================= + void AssetManager::AssignAssetData(const Asset& asset) + { + AZ_Assert(asset.Get(), "Reloaded data is missing!"); + + const AssetId& assetId = asset.GetId(); + + asset->m_status = AssetData::AssetStatus::Ready; + UpdateDebugStatus(asset); + + if (asset->IsRegisterReadonlyAndShareable()) + { + bool requeue{ false }; { - AZStd::lock_guard assetLock(m_assetMutex); - m_reloads.erase(asset.GetId()); - } - AssetBus::Event(asset.GetId(), &AssetBus::Events::OnAssetReloadError, asset); - } - - //========================================================================= - // NotifyAssetError - //========================================================================= - void AssetManager::NotifyAssetError(Asset asset) - { - asset.Get()->m_status = AssetData::AssetStatus::Error; - AssetBus::Event(asset.GetId(), &AssetBus::Events::OnAssetError, asset); - } - - void AssetManager::NotifyAssetCanceled(AssetId assetId) - { - AssetBus::Event(assetId, &AssetBus::Events::OnAssetCanceled, assetId); - } - - void AssetManager::NotifyAssetContainerReady(Asset asset) - { - AssetBus::Event(asset.GetId(), &AssetBus::Events::OnAssetContainerReady, asset); - } - - //========================================================================= - // AddJob - // [04/02/2014] - //========================================================================= - void AssetManager::AddJob(AssetDatabaseJob* job) - { - AZStd::scoped_lock assetLock(m_activeJobOrRequestMutex); - - m_activeJobs.push_back(*job); - } - - //========================================================================= - // ValidateAndRegisterAssetLoading - //========================================================================= - bool AssetManager::ValidateAndRegisterAssetLoading(const Asset& asset) - { - AssetData* data = asset.Get(); - { - AZStd::scoped_lock assetLock(m_assetMutex); - if (data) + auto found = m_assets.find(assetId); + AZ_Assert(found == m_assets.end() || asset.Get()->RTTI_GetType() == found->second->RTTI_GetType(), + "New and old data types are mismatched!"); + + // if we are here it implies that we have two assets with the same asset id, and we are + // trying to replace the old asset with the new asset which was not created using the asset manager system. + // In this scenario if any other system have cached the old asset then the asset wont be destroyed + // because of creation token mismatch when it's ref count finally goes to zero. Since the old asset is not shareable anymore + // manually setting the creationToken to default creation token will ensure that the asset is destroyed correctly. + asset.m_assetData->m_creationToken = ++m_creationTokenGenerator; + if (found != m_assets.end()) { - // The purpose of this function is to validate this asset is still in a StreamReady - // and only then continue the load. We change status to loading if everything - // is expected which the blocking RegisterAssetLoading call does not do because it - // is already in loading status - if (data->GetStatus() != AssetData::AssetStatus::StreamReady) - { - // Something else has attempted to load this asset - return false; - } - data->m_status = AssetData::AssetStatus::Loading; - UpdateDebugStatus(asset); + found->second->m_creationToken = AZ::Data::s_defaultCreationToken; + } + + // Held references to old data are retained, but replace the entry in the DB for future requests. + // Fire an OnAssetReloaded message so listeners can react to the new data. + m_assets[assetId] = asset.Get(); + + // Release the reload reference. + auto reloadInfo = m_reloads.find(assetId); + if (reloadInfo != m_reloads.end()) + { + requeue = reloadInfo->second->GetRequeue(); + m_reloads.erase(reloadInfo); } } + // Call reloaded before we can call ReloadAsset below to preserve order + AssetBus::Event(assetId, &AssetBus::Events::OnAssetReloaded, asset); + // Release the lock before we call reload + if (requeue) + { + ReloadAsset(assetId, asset.GetAutoLoadBehavior()); + } + } + else + { + AssetBus::Event(assetId, &AssetBus::Events::OnAssetReloaded, asset); + } + } - return true; + //========================================================================= + // GetModifiedLoadStreamInfoForAsset + //========================================================================= + AssetStreamInfo AssetManager::GetModifiedLoadStreamInfoForAsset(const Asset& asset, AssetHandler* handler) + { + AssetStreamInfo loadInfo = GetLoadStreamInfoForAsset(asset.GetId(), asset.GetType()); + if (!loadInfo.IsValid()) + { + // opportunity for handler to do default substitution: + AZ::Data::AssetId fallbackId = handler->AssetMissingInCatalog(asset); + if (fallbackId.IsValid()) + { + loadInfo = GetLoadStreamInfoForAsset(fallbackId, asset.GetType()); + } } - //========================================================================= - // RegisterAssetLoading - //========================================================================= - void AssetManager::RegisterAssetLoading(const Asset& asset) - { - AZ_PROFILE_FUNCTION(AzCore); + // Give the handler an opportunity to modify any of the load info before creating the dataStream. + handler->GetCustomAssetStreamInfoForLoad(loadInfo); - AssetData* data = asset.Get(); + return loadInfo; + } + + //========================================================================= + // QueueAsyncStreamLoad + //========================================================================= + void AssetManager::QueueAsyncStreamLoad(Asset asset, AZStd::shared_ptr dataStream, + const AZ::Data::AssetStreamInfo& streamInfo, bool isReload, + AssetHandler* handler, const AssetLoadParameters& loadParams, bool signalLoaded) + { + AZ_PROFILE_FUNCTION(AzCore); + + // Set up the callback that will process the asset data once the raw file load is finished. + // The callback is declared as mutable so that we can clear weakAsset within the callback. The refcount in weakAsset + // can trigger an AssetManager::ReleaseAsset call. If this occurs during lambda cleanup, it could happen at any time + // on the file streamer thread as streamer requests get recycled, including during (or after) AssetManager shutdown. + // By controlling when the refcount is changed, we can ensure that it occurs while the AssetManager is still active. + auto assetDataStreamCallback = [this, loadParams, handler, dataStream, signalLoaded, isReload, + weakAsset = AssetInternal::WeakAsset(asset)] + (AZ::IO::IStreamerTypes::RequestStatus status) mutable + { + auto assetId = weakAsset.GetId(); + + Asset loadingAsset = weakAsset.GetStrongReference(); + + if (loadingAsset) + { + AZ_PROFILE_SCOPE(AzCore, "AZ::Data::LoadAssetStreamerCallback %s", + loadingAsset.GetHint().c_str()); + { + AZStd::scoped_lock assetLock(m_assetMutex); + AssetData* data = loadingAsset.Get(); + if (data->GetStatus() != AssetData::AssetStatus::Queued) + { + AZ_Warning("AssetManager", false, "Asset %s no longer in Queued state, abandoning load", loadingAsset.GetId().ToString().c_str()); + return; + } + data->m_status = AssetData::AssetStatus::StreamReady; + } + + // The callback from AZ Streamer blocks the streaming thread until this function completes. To minimize the overhead, + // do the majority of the work in a separate job. + auto loadJob = aznew LoadAssetJob(this, loadingAsset, + dataStream, isReload, status, handler, loadParams, signalLoaded); + + bool jobQueued = false; + + // If there's already an active blocking request waiting for this load to complete, let that thread handle + // the load itself instead of consuming a second thread. + { + AZStd::scoped_lock requestLock(m_activeBlockingRequestMutex); + auto range = m_activeBlockingRequests.equal_range(assetId); + for(auto blockingRequest = range.first; blockingRequest != range.second; ++blockingRequest) + { + if(blockingRequest->second->QueueAssetLoadJob(loadJob)) + { + jobQueued = true; + break; + } + } + } + + if (!jobQueued) + { + loadJob->Start(); + } + } + else + { + BlockingAssetLoadBus::Event(assetId, &BlockingAssetLoadBus::Events::OnLoadCanceled, assetId); + AssetManagerBus::Broadcast(&AssetManagerBus::Events::OnAssetCanceled, assetId); + } + + // *After* the loadJob has been created, clear our asset references and remove the active streamer requests. + // This needs to happen after the loadJob creation to ensure that on AssetManager shutdown, there are no brief + // windows in which requests and/or jobs are still active after we've removed our tracking of the requests and jobs. + + // Also, if the asset references don't get cleared until after the callback completes, or at some indeterminate later + // time when the File Streamer cleans up the file requests (for the weakAsset lambda parameter), then it's possible that + // they will trigger a ReleaseAsset call sometime after the AssetManager has begun to shut down, which can lead to + // race conditions. + + // Make sure the streamer request is removed first before the asset is released + // If the asset is released first it could lead to a race condition where another thread starts loading the asset + // again and attempts to add a new streamer request with the same ID before the old one has been removed, causing + // that load request to fail + RemoveActiveStreamerRequest(assetId); + weakAsset = {}; + loadingAsset.Reset(); + }; + + auto&& [deadline, priority] = GetEffectiveDeadlineAndPriority(*handler, asset.GetType(), loadParams); + + // Track the load request and queue the asset data stream load. + AddActiveStreamerRequest(asset.GetId(), dataStream); + dataStream->Open( + streamInfo.m_streamName, + streamInfo.m_dataOffset, + streamInfo.m_dataLen, + deadline, priority, assetDataStreamCallback); + } + + //========================================================================= + // NotifyAssetReady + //========================================================================= + void AssetManager::NotifyAssetReady(Asset asset) + { + AssetData* data = asset.Get(); + AZ_Assert(data, "NotifyAssetReady: asset is missing info!"); + data->m_status = AssetData::AssetStatus::Ready; + + AssetBus::Event(asset.GetId(), &AssetBus::Events::OnAssetReady, asset); + } + + //========================================================================= + // NotifyAssetPreReload + //========================================================================= + void AssetManager::NotifyAssetPreReload(Asset asset) + { + AssetBus::Event(asset.GetId(), &AssetBus::Events::OnAssetPreReload, asset); + } + + //========================================================================= + // NotifyAssetReloaded + //========================================================================= + void AssetManager::NotifyAssetReloaded(Asset asset) + { + AssignAssetData(asset); + } + + //========================================================================= + // NotifyAssetReloaded + //========================================================================= + void AssetManager::NotifyAssetReloadError(Asset asset) + { + // Failed reloads have no side effects. Just notify observers (error reporting, etc). + { + AZStd::lock_guard assetLock(m_assetMutex); + m_reloads.erase(asset.GetId()); + } + AssetBus::Event(asset.GetId(), &AssetBus::Events::OnAssetReloadError, asset); + } + + //========================================================================= + // NotifyAssetError + //========================================================================= + void AssetManager::NotifyAssetError(Asset asset) + { + asset.Get()->m_status = AssetData::AssetStatus::Error; + AssetBus::Event(asset.GetId(), &AssetBus::Events::OnAssetError, asset); + } + + void AssetManager::NotifyAssetCanceled(AssetId assetId) + { + AssetBus::Event(assetId, &AssetBus::Events::OnAssetCanceled, assetId); + } + + void AssetManager::NotifyAssetContainerReady(Asset asset) + { + AssetBus::Event(asset.GetId(), &AssetBus::Events::OnAssetContainerReady, asset); + } + + //========================================================================= + // AddJob + // [04/02/2014] + //========================================================================= + void AssetManager::AddJob(AssetDatabaseJob* job) + { + AZStd::scoped_lock assetLock(m_activeJobOrRequestMutex); + + m_activeJobs.push_back(*job); + } + + //========================================================================= + // ValidateAndRegisterAssetLoading + //========================================================================= + bool AssetManager::ValidateAndRegisterAssetLoading(const Asset& asset) + { + AssetData* data = asset.Get(); + { + + AZStd::scoped_lock assetLock(m_assetMutex); if (data) { + // The purpose of this function is to validate this asset is still in a StreamReady + // and only then continue the load. We change status to loading if everything + // is expected which the blocking RegisterAssetLoading call does not do because it + // is already in loading status + if (data->GetStatus() != AssetData::AssetStatus::StreamReady) + { + // Something else has attempted to load this asset + return false; + } data->m_status = AssetData::AssetStatus::Loading; UpdateDebugStatus(asset); } } - //========================================================================= - // UnregisterAssetLoadingByThread - //========================================================================= - void AssetManager::UnregisterAssetLoading([[maybe_unused]] const Asset& asset) + return true; + } + + //========================================================================= + // RegisterAssetLoading + //========================================================================= + void AssetManager::RegisterAssetLoading(const Asset& asset) + { + AZ_PROFILE_FUNCTION(AzCore); + + AssetData* data = asset.Get(); + if (data) { - AZ_PROFILE_FUNCTION(AzCore); - } - - //========================================================================= - // RemoveJob - // [04/02/2014] - //========================================================================= - void AssetManager::RemoveJob(AssetDatabaseJob* job) - { - AZStd::scoped_lock assetLock(m_activeJobOrRequestMutex); - - m_activeJobs.erase(*job); - } - - //========================================================================= - // AddActiveStreamerRequest - //========================================================================= - void AssetManager::AddActiveStreamerRequest(AssetId assetId, AZStd::shared_ptr readRequest) - { - AZStd::scoped_lock assetLock(m_activeJobOrRequestMutex); - - // Track the request to allow for manual cancellation and for validating completion before AssetManager shutdown - [[maybe_unused]] auto inserted = - m_activeAssetDataStreamRequests.insert(AZStd::make_pair(assetId, readRequest)); - AZ_Assert(inserted.second, "Failed to insert streaming request into map for later retrieval by asset."); - - } - - void AssetManager::RescheduleStreamerRequest(AssetId assetId, AZStd::chrono::milliseconds newDeadline, AZ::IO::IStreamerTypes::Priority newPriority) - { - AZStd::scoped_lock lock(m_activeJobOrRequestMutex); - - auto iterator = m_activeAssetDataStreamRequests.find(assetId); - - if (iterator != m_activeAssetDataStreamRequests.end()) - { - iterator->second->Reschedule(newDeadline, newPriority); - } - } - - //========================================================================= - // RemoveActiveStreamerRequest - //========================================================================= - void AssetManager::RemoveActiveStreamerRequest(AssetId assetData) - { - AZStd::scoped_lock assetLock(m_activeJobOrRequestMutex); - m_activeAssetDataStreamRequests.erase(assetData); - } - - //========================================================================= - // HasActiveJobsOrStreamerRequests - //========================================================================= - bool AssetManager::HasActiveJobsOrStreamerRequests() - { - AZStd::scoped_lock assetLock(m_activeJobOrRequestMutex); - - return (!(m_activeJobs.empty() && m_activeAssetDataStreamRequests.empty())); - } - - //========================================================================= - // AddBlockingRequest - //========================================================================= - void AssetManager::AddBlockingRequest(AssetId assetId, WaitForAsset* blockingRequest) - { - AZStd::scoped_lock requestLock(m_activeBlockingRequestMutex); - - [[maybe_unused]] auto inserted = m_activeBlockingRequests.insert(AZStd::make_pair(assetId, blockingRequest)); - AZ_Assert(inserted.second, "Failed to track blocking request for asset %s", assetId.ToString().c_str()); - } - - //========================================================================= - // RemoveBlockingRequest - //========================================================================= - void AssetManager::RemoveBlockingRequest(AssetId assetId, WaitForAsset* blockingRequest) - { - AZStd::scoped_lock requestLock(m_activeBlockingRequestMutex); - [[maybe_unused]] bool requestFound = false; - for (auto assetIdIterator = m_activeBlockingRequests.find(assetId); assetIdIterator != m_activeBlockingRequests.end(); ) - { - if (assetIdIterator->second == blockingRequest) - { - m_activeBlockingRequests.erase(assetIdIterator); - requestFound = true; - break; - } - else - { - assetIdIterator++; - } - } - - AZ_Assert(requestFound, "Failed to erase blocking request for asset %s", assetId.ToString().c_str()); - } - - - //========================================================================= - // GetLoadStreamInfoForAsset() - // [04/04/2014] - //========================================================================= - AssetStreamInfo AssetManager::GetLoadStreamInfoForAsset(const AssetId& assetId, const AssetType& assetType) - { - AZStd::scoped_lock catalogLock(m_catalogMutex); - AssetCatalogMap::iterator catIt = m_catalogs.find(assetType); - if (catIt == m_catalogs.end()) - { - AZ_Error("Asset", false, "Asset [type:%s id:%s] with this type doesn't have a catalog!", assetType.template ToString().c_str(), assetId.ToString().c_str()); - return AssetStreamInfo(); - } - return catIt->second->GetStreamInfoForLoad(assetId, assetType); - } - - //========================================================================= - // GetSaveStreamInfoForAsset() - // [04/04/2014] - //========================================================================= - AssetStreamInfo AssetManager::GetSaveStreamInfoForAsset(const AssetId& assetId, const AssetType& assetType) - { - AZStd::scoped_lock catalogLock(m_catalogMutex); - AssetCatalogMap::iterator catIt = m_catalogs.find(assetType); - if (catIt == m_catalogs.end()) - { - AZ_Error("Asset", false, "Asset [type:%s id:%s] with this type doesn't have a catalog!", assetType.template ToString().c_str(), assetId.ToString().c_str()); - return AssetStreamInfo(); - } - return catIt->second->GetStreamInfoForSave(assetId, assetType); - } - - //========================================================================= - // OnAssetReady - // [04/02/2014] - //========================================================================= - void AssetManager::OnAssetReady(const Asset& asset) - { - AZ_Assert(asset.Get(), "OnAssetReady fired for an asset with no data."); - - // Set status immediately from within the AssetManagerBus dispatch, so it's committed before anyone is notified (e.g. job to job, via AssetJobBus). - asset.Get()->m_status = AssetData::AssetStatus::ReadyPreNotify; + data->m_status = AssetData::AssetStatus::Loading; UpdateDebugStatus(asset); - - // Queue broadcast message for delivery on game thread. - AssetBus::QueueFunction(&AssetManager::NotifyAssetReady, this, Asset(asset)); } + } - //========================================================================= - // OnAssetError - //========================================================================= - void AssetManager::OnAssetError(const Asset& asset) + //========================================================================= + // UnregisterAssetLoadingByThread + //========================================================================= + void AssetManager::UnregisterAssetLoading([[maybe_unused]] const Asset& asset) + { + AZ_PROFILE_FUNCTION(AzCore); + } + + //========================================================================= + // RemoveJob + // [04/02/2014] + //========================================================================= + void AssetManager::RemoveJob(AssetDatabaseJob* job) + { + AZStd::scoped_lock assetLock(m_activeJobOrRequestMutex); + + m_activeJobs.erase(*job); + } + + //========================================================================= + // AddActiveStreamerRequest + //========================================================================= + void AssetManager::AddActiveStreamerRequest(AssetId assetId, AZStd::shared_ptr readRequest) + { + AZStd::scoped_lock assetLock(m_activeJobOrRequestMutex); + + // Track the request to allow for manual cancellation and for validating completion before AssetManager shutdown + [[maybe_unused]] auto inserted = + m_activeAssetDataStreamRequests.insert(AZStd::make_pair(assetId, readRequest)); + AZ_Assert(inserted.second, "Failed to insert streaming request into map for later retrieval by asset."); + + } + + void AssetManager::RescheduleStreamerRequest(AssetId assetId, AZStd::chrono::milliseconds newDeadline, AZ::IO::IStreamerTypes::Priority newPriority) + { + AZStd::scoped_lock lock(m_activeJobOrRequestMutex); + + auto iterator = m_activeAssetDataStreamRequests.find(assetId); + + if (iterator != m_activeAssetDataStreamRequests.end()) { - // Set status immediately from within the AssetManagerBus dispatch, so it's committed before anyone is notified (e.g. job to job, via AssetJobBus). - asset.Get()->m_status = AssetData::AssetStatus::Error; - UpdateDebugStatus(asset); - - // Queue broadcast message for delivery on game thread. - AssetBus::QueueFunction(&AssetManager::NotifyAssetError, this, Asset(asset)); + iterator->second->Reschedule(newDeadline, newPriority); } + } - void AssetManager::OnAssetCanceled(AssetId assetId) + //========================================================================= + // RemoveActiveStreamerRequest + //========================================================================= + void AssetManager::RemoveActiveStreamerRequest(AssetId assetData) + { + AZStd::scoped_lock assetLock(m_activeJobOrRequestMutex); + m_activeAssetDataStreamRequests.erase(assetData); + } + + //========================================================================= + // HasActiveJobsOrStreamerRequests + //========================================================================= + bool AssetManager::HasActiveJobsOrStreamerRequests() + { + AZStd::scoped_lock assetLock(m_activeJobOrRequestMutex); + + return (!(m_activeJobs.empty() && m_activeAssetDataStreamRequests.empty())); + } + + //========================================================================= + // AddBlockingRequest + //========================================================================= + void AssetManager::AddBlockingRequest(AssetId assetId, WaitForAsset* blockingRequest) + { + AZStd::scoped_lock requestLock(m_activeBlockingRequestMutex); + + [[maybe_unused]] auto inserted = m_activeBlockingRequests.insert(AZStd::make_pair(assetId, blockingRequest)); + AZ_Assert(inserted.second, "Failed to track blocking request for asset %s", assetId.ToString().c_str()); + } + + //========================================================================= + // RemoveBlockingRequest + //========================================================================= + void AssetManager::RemoveBlockingRequest(AssetId assetId, WaitForAsset* blockingRequest) + { + AZStd::scoped_lock requestLock(m_activeBlockingRequestMutex); + [[maybe_unused]] bool requestFound = false; + for (auto assetIdIterator = m_activeBlockingRequests.find(assetId); assetIdIterator != m_activeBlockingRequests.end(); ) { - // Queue broadcast message for delivery on game thread. - AssetBus::QueueFunction(&AssetManager::NotifyAssetCanceled, this, assetId); - } - - void AssetManager::ReleaseOwnedAssetContainer(AssetContainer* assetContainer) - { - AZ_Assert(assetContainer, "Trying to release a null assetContainer pointer!"); - AZStd::scoped_lock lock(m_assetContainerMutex); - auto rangeItr = m_ownedAssetContainerLookup.equal_range(assetContainer->GetContainerAssetId()); - - for (auto itr = rangeItr.first; itr != rangeItr.second; ++itr) + if (assetIdIterator->second == blockingRequest) { - if (itr->second == assetContainer) - { - m_ownedAssetContainerLookup.erase(itr); - break; - } - } - - m_ownedAssetContainers.erase(assetContainer); - } - - void AssetManager::OnAssetContainerReady(AssetContainer* assetContainer) - { - AssetBus::QueueFunction([this, assetContainer, asset = assetContainer->GetRootAsset()]() - { - NotifyAssetContainerReady(asset); - ReleaseOwnedAssetContainer(assetContainer); - }); - } - - void AssetManager::OnAssetContainerCanceled(AssetContainer* assetContainer) - { - AssetBus::QueueFunction([this, assetContainer]() - { - ReleaseOwnedAssetContainer(assetContainer); - }); - } - - //========================================================================= - // OnAssetReloaded - //========================================================================= - void AssetManager::OnAssetReloaded(const Asset& asset) - { - // Queue broadcast message for delivery on game thread. - AssetBus::QueueFunction(&AssetManager::NotifyAssetReloaded, this, Asset(asset)); - } - - //========================================================================= - // OnAssetReloadError - //========================================================================= - void AssetManager::OnAssetReloadError(const Asset& asset) - { - // Queue broadcast message for delivery on game thread. - AssetBus::QueueFunction(&AssetManager::NotifyAssetReloadError, this, Asset(asset)); - } - - - //========================================================================= - // AssetHandler - // [04/03/2014] - //========================================================================= - AssetHandler::AssetHandler() - : m_nHandledTypes(0) - { - } - - //========================================================================= - // ~AssetHandler - // [04/03/2014] - //========================================================================= - AssetHandler::~AssetHandler() - { - if (m_nHandledTypes > 0) - { - AssetManager::Instance().UnregisterHandler(this); - } - - AZ_Error("AssetDatabase", m_nHandledTypes == 0, "Asset handler is being destroyed but there are still %d asset types being handled by it!", (int)m_nHandledTypes); - } - - //========================================================================= - // LoadAssetDataFromStream - //========================================================================= - AssetHandler::LoadResult AssetHandler::LoadAssetDataFromStream( - const Asset& asset, - AZStd::shared_ptr stream, - const AssetFilterCB& assetLoadFilterCB) - { - AZ_PROFILE_SCOPE(AzCore, "AssetHandler::LoadAssetData - %s", asset.GetHint().c_str()); - -#ifdef AZ_ENABLE_TRACING - auto start = AZStd::chrono::system_clock::now(); -#endif - - LoadResult result = LoadAssetData(asset, stream, assetLoadFilterCB); - -#ifdef AZ_ENABLE_TRACING - auto loadMs = AZStd::chrono::duration_cast( - AZStd::chrono::system_clock::now() - start); - AZ_Warning("AssetDatabase", (!cl_assetLoadWarningEnable) || - loadMs <= AZStd::chrono::milliseconds(cl_assetLoadWarningMsThreshold), - "Load time threshold exceeded: LoadAssetData call for %s took %" PRId64 " ms", - asset.GetHint().c_str(), loadMs.count()); -#endif - - return result; - } - - //========================================================================= - // InitAsset - // [04/03/2014] - //========================================================================= - void AssetHandler::InitAsset(const Asset& asset, bool loadStageSucceeded, bool isReload) - { - if (loadStageSucceeded) - { - if (isReload) - { - AssetManagerBus::Broadcast(&AssetManagerBus::Events::OnAssetReloaded, asset); - } - else - { - AssetManagerBus::Broadcast(&AssetManagerBus::Events::OnAssetReady, asset); - } + m_activeBlockingRequests.erase(assetIdIterator); + requestFound = true; + break; } else { - if (!isReload) - { - AssetManagerBus::Broadcast(&AssetManagerBus::Events::OnAssetError, asset); - } - else - { - AssetManagerBus::Broadcast(&AssetManagerBus::Events::OnAssetReloadError, asset); - } + assetIdIterator++; } } - void AssetManager::ValidateAndPostLoad(AZ::Data::Asset& asset, bool loadSucceeded, - bool isReload, AZ::Data::AssetHandler* assetHandler) + AZ_Assert(requestFound, "Failed to erase blocking request for asset %s", assetId.ToString().c_str()); + } + + + //========================================================================= + // GetLoadStreamInfoForAsset() + // [04/04/2014] + //========================================================================= + AssetStreamInfo AssetManager::GetLoadStreamInfoForAsset(const AssetId& assetId, const AssetType& assetType) + { + AZStd::scoped_lock catalogLock(m_catalogMutex); + AssetCatalogMap::iterator catIt = m_catalogs.find(assetType); + if (catIt == m_catalogs.end()) { + AZ_Error("Asset", false, "Asset [type:%s id:%s] with this type doesn't have a catalog!", assetType.template ToString().c_str(), assetId.ToString().c_str()); + return AssetStreamInfo(); + } + return catIt->second->GetStreamInfoForLoad(assetId, assetType); + } + + //========================================================================= + // GetSaveStreamInfoForAsset() + // [04/04/2014] + //========================================================================= + AssetStreamInfo AssetManager::GetSaveStreamInfoForAsset(const AssetId& assetId, const AssetType& assetType) + { + AZStd::scoped_lock catalogLock(m_catalogMutex); + AssetCatalogMap::iterator catIt = m_catalogs.find(assetType); + if (catIt == m_catalogs.end()) + { + AZ_Error("Asset", false, "Asset [type:%s id:%s] with this type doesn't have a catalog!", assetType.template ToString().c_str(), assetId.ToString().c_str()); + return AssetStreamInfo(); + } + return catIt->second->GetStreamInfoForSave(assetId, assetType); + } + + //========================================================================= + // OnAssetReady + // [04/02/2014] + //========================================================================= + void AssetManager::OnAssetReady(const Asset& asset) + { + AZ_Assert(asset.Get(), "OnAssetReady fired for an asset with no data."); + + // Set status immediately from within the AssetManagerBus dispatch, so it's committed before anyone is notified (e.g. job to job, via AssetJobBus). + asset.Get()->m_status = AssetData::AssetStatus::ReadyPreNotify; + UpdateDebugStatus(asset); + + // Queue broadcast message for delivery on game thread. + AssetBus::QueueFunction(&AssetManager::NotifyAssetReady, this, Asset(asset)); + } + + //========================================================================= + // OnAssetError + //========================================================================= + void AssetManager::OnAssetError(const Asset& asset) + { + // Set status immediately from within the AssetManagerBus dispatch, so it's committed before anyone is notified (e.g. job to job, via AssetJobBus). + asset.Get()->m_status = AssetData::AssetStatus::Error; + UpdateDebugStatus(asset); + + // Queue broadcast message for delivery on game thread. + AssetBus::QueueFunction(&AssetManager::NotifyAssetError, this, Asset(asset)); + } + + void AssetManager::OnAssetCanceled(AssetId assetId) + { + // Queue broadcast message for delivery on game thread. + AssetBus::QueueFunction(&AssetManager::NotifyAssetCanceled, this, assetId); + } + + void AssetManager::ReleaseOwnedAssetContainer(AssetContainer* assetContainer) + { + AZ_Assert(assetContainer, "Trying to release a null assetContainer pointer!"); + AZStd::scoped_lock lock(m_assetContainerMutex); + auto rangeItr = m_ownedAssetContainerLookup.equal_range(assetContainer->GetContainerAssetId()); + + for (auto itr = rangeItr.first; itr != rangeItr.second; ++itr) + { + if (itr->second == assetContainer) { - // We may need to revalidate that this asset hasn't already passed through postLoad - AZStd::scoped_lock assetLock(m_assetMutex); - if (asset->IsReady() || asset->m_status == AssetData::AssetStatus::LoadedPreReady) - { - return; - } - asset->m_status = AssetData::AssetStatus::LoadedPreReady; - UpdateDebugStatus(asset); + m_ownedAssetContainerLookup.erase(itr); + break; } - PostLoad(asset, loadSucceeded, isReload, assetHandler); } - void AssetManager::PostLoad(AZ::Data::Asset& asset, bool loadSucceeded, - bool isReload, AZ::Data::AssetHandler* assetHandler) - { - AZ_PROFILE_FUNCTION(AzCore); - if (!assetHandler) - { - assetHandler = GetHandler(asset.GetType()); - } + m_ownedAssetContainers.erase(assetContainer); + } - if (assetHandler) + void AssetManager::OnAssetContainerReady(AssetContainer* assetContainer) + { + AssetBus::QueueFunction([this, assetContainer, asset = assetContainer->GetRootAsset()]() + { + NotifyAssetContainerReady(asset); + ReleaseOwnedAssetContainer(assetContainer); + }); + } + + void AssetManager::OnAssetContainerCanceled(AssetContainer* assetContainer) + { + AssetBus::QueueFunction([this, assetContainer]() + { + ReleaseOwnedAssetContainer(assetContainer); + }); + } + + //========================================================================= + // OnAssetReloaded + //========================================================================= + void AssetManager::OnAssetReloaded(const Asset& asset) + { + // Queue broadcast message for delivery on game thread. + AssetBus::QueueFunction(&AssetManager::NotifyAssetReloaded, this, Asset(asset)); + } + + //========================================================================= + // OnAssetReloadError + //========================================================================= + void AssetManager::OnAssetReloadError(const Asset& asset) + { + // Queue broadcast message for delivery on game thread. + AssetBus::QueueFunction(&AssetManager::NotifyAssetReloadError, this, Asset(asset)); + } + + + //========================================================================= + // AssetHandler + // [04/03/2014] + //========================================================================= + AssetHandler::AssetHandler() + : m_nHandledTypes(0) + { + } + + //========================================================================= + // ~AssetHandler + // [04/03/2014] + //========================================================================= + AssetHandler::~AssetHandler() + { + if (m_nHandledTypes > 0) + { + AssetManager::Instance().UnregisterHandler(this); + } + + AZ_Error("AssetDatabase", m_nHandledTypes == 0, "Asset handler is being destroyed but there are still %d asset types being handled by it!", (int)m_nHandledTypes); + } + + //========================================================================= + // LoadAssetDataFromStream + //========================================================================= + AssetHandler::LoadResult AssetHandler::LoadAssetDataFromStream( + const Asset& asset, + AZStd::shared_ptr stream, + const AssetFilterCB& assetLoadFilterCB) + { + AZ_PROFILE_SCOPE(AzCore, "AssetHandler::LoadAssetData - %s", asset.GetHint().c_str()); + +#ifdef AZ_ENABLE_TRACING + auto start = AZStd::chrono::system_clock::now(); +#endif + + LoadResult result = LoadAssetData(asset, stream, assetLoadFilterCB); + +#ifdef AZ_ENABLE_TRACING + auto loadMs = AZStd::chrono::duration_cast( + AZStd::chrono::system_clock::now() - start); + AZ_Warning("AssetDatabase", (!cl_assetLoadWarningEnable) || + loadMs <= AZStd::chrono::milliseconds(cl_assetLoadWarningMsThreshold), + "Load time threshold exceeded: LoadAssetData call for %s took %" PRId64 " ms", + asset.GetHint().c_str(), loadMs.count()); +#endif + + return result; + } + + //========================================================================= + // InitAsset + // [04/03/2014] + //========================================================================= + void AssetHandler::InitAsset(const Asset& asset, bool loadStageSucceeded, bool isReload) + { + if (loadStageSucceeded) + { + if (isReload) { - // Queue the result for dispatch to main thread. - assetHandler->InitAsset(asset, loadSucceeded, isReload); + AssetManagerBus::Broadcast(&AssetManagerBus::Events::OnAssetReloaded, asset); } else { - AZ_Warning("AssetManager", false, "Couldn't find handler for asset %s (%s)", asset.GetId().ToString().c_str(), asset.GetHint().c_str()); + AssetManagerBus::Broadcast(&AssetManagerBus::Events::OnAssetReady, asset); } + } + else + { + if (!isReload) + { + AssetManagerBus::Broadcast(&AssetManagerBus::Events::OnAssetError, asset); + } + else + { + AssetManagerBus::Broadcast(&AssetManagerBus::Events::OnAssetReloadError, asset); + } + } + } - // Notify any dependent jobs. - BlockingAssetLoadBus::Event(asset.GetId(), &BlockingAssetLoadBus::Events::OnLoadComplete); + void AssetManager::ValidateAndPostLoad(AZ::Data::Asset& asset, bool loadSucceeded, + bool isReload, AZ::Data::AssetHandler* assetHandler) + { + { + // We may need to revalidate that this asset hasn't already passed through postLoad + AZStd::scoped_lock assetLock(m_assetMutex); + if (asset->IsReady() || asset->m_status == AssetData::AssetStatus::LoadedPreReady) + { + return; + } + asset->m_status = AssetData::AssetStatus::LoadedPreReady; + UpdateDebugStatus(asset); + } + PostLoad(asset, loadSucceeded, isReload, assetHandler); + } - UnregisterAssetLoading(asset); + void AssetManager::PostLoad(AZ::Data::Asset& asset, bool loadSucceeded, + bool isReload, AZ::Data::AssetHandler* assetHandler) + { + AZ_PROFILE_FUNCTION(AzCore); + if (!assetHandler) + { + assetHandler = GetHandler(asset.GetType()); } - AZStd::shared_ptr AssetManager::GetAssetContainer(Asset asset, const AssetLoadParameters& loadParams) + if (assetHandler) { - // If we're doing a custom load through a filter just hand back a one off container - if (loadParams.m_assetLoadFilterCB) - { - return CreateAssetContainer(asset, loadParams); - } + // Queue the result for dispatch to main thread. + assetHandler->InitAsset(asset, loadSucceeded, isReload); + } + else + { + AZ_Warning("AssetManager", false, "Couldn't find handler for asset %s (%s)", asset.GetId().ToString().c_str(), asset.GetHint().c_str()); + } - AZStd::scoped_lock containerLock(m_assetContainerMutex); - AssetContainerKey containerKey{ asset.GetId(), loadParams }; + // Notify any dependent jobs. + BlockingAssetLoadBus::Event(asset.GetId(), &BlockingAssetLoadBus::Events::OnLoadComplete); - auto curIter = m_assetContainers.find(containerKey); - if (curIter != m_assetContainers.end()) + UnregisterAssetLoading(asset); + } + + AZStd::shared_ptr AssetManager::GetAssetContainer(Asset asset, const AssetLoadParameters& loadParams) + { + // If we're doing a custom load through a filter just hand back a one off container + if (loadParams.m_assetLoadFilterCB) + { + return CreateAssetContainer(asset, loadParams); + } + + AZStd::scoped_lock containerLock(m_assetContainerMutex); + AssetContainerKey containerKey{ asset.GetId(), loadParams }; + + auto curIter = m_assetContainers.find(containerKey); + if (curIter != m_assetContainers.end()) + { + auto newRef = curIter->second.lock(); + if (newRef && newRef->IsValid()) { - auto newRef = curIter->second.lock(); - if (newRef && newRef->IsValid()) - { - return newRef; - } - auto newContainer = CreateAssetContainer(asset, loadParams); - curIter->second = newContainer; - return newContainer; + return newRef; } auto newContainer = CreateAssetContainer(asset, loadParams); - - m_assetContainers.insert({ containerKey, newContainer }); - + curIter->second = newContainer; return newContainer; } + auto newContainer = CreateAssetContainer(asset, loadParams); - AZStd::shared_ptr AssetManager::CreateAssetContainer(Asset asset, const AssetLoadParameters& loadParams) const - { - return AZStd::shared_ptr( aznew AssetContainer(AZStd::move(asset), loadParams)); - } - } // namespace Data -} // namespace AZ + m_assetContainers.insert({ containerKey, newContainer }); + + return newContainer; + } + + AZStd::shared_ptr AssetManager::CreateAssetContainer(Asset asset, const AssetLoadParameters& loadParams) const + { + return AZStd::shared_ptr( aznew AssetContainer(AZStd::move(asset), loadParams)); + } +} // namespace AZ::Data size_t AZStd::hash::operator()(const AZ::Data::AssetContainerKey& obj) const { diff --git a/Code/Framework/AzCore/AzCore/Component/EntityUtils.cpp b/Code/Framework/AzCore/AzCore/Component/EntityUtils.cpp index fcc8cd6424..54e40c1fc0 100644 --- a/Code/Framework/AzCore/AzCore/Component/EntityUtils.cpp +++ b/Code/Framework/AzCore/AzCore/Component/EntityUtils.cpp @@ -10,292 +10,289 @@ #include #include -namespace AZ +namespace AZ::EntityUtils { - namespace EntityUtils + //========================================================================= + // Reflect + //========================================================================= + void Reflect(ReflectContext* context) { - //========================================================================= - // Reflect - //========================================================================= - void Reflect(ReflectContext* context) + if (auto serializeContext = azrtti_cast(context)) { - if (auto serializeContext = azrtti_cast(context)) + serializeContext->Class()-> + Version(1)-> + Field("Entities", &SerializableEntityContainer::m_entities); + } + } + + struct StackDataType + { + const SerializeContext::ClassData* m_classData; + const SerializeContext::ClassElement* m_elementData; + void* m_dataPtr; + bool m_isModifiedContainer; + }; + + //========================================================================= + // EnumerateEntityIds + //========================================================================= + void EnumerateEntityIds(const void* classPtr, const Uuid& classUuid, const EntityIdVisitor& visitor, SerializeContext* context) + { + AZ_PROFILE_FUNCTION(AzCore); + + if (!context) + { + context = GetApplicationSerializeContext(); + if (!context) { - serializeContext->Class()-> - Version(1)-> - Field("Entities", &SerializableEntityContainer::m_entities); + AZ_Error("Serialization", false, "No serialize context provided! Failed to get component application default serialize context! ComponentApp is not started or input serialize context should not be null!"); + return; } } + AZStd::vector parentStack; + parentStack.reserve(30); + auto beginCB = [ &](void* ptr, const SerializeContext::ClassData* classData, const SerializeContext::ClassElement* elementData) -> bool + { + (void)elementData; - struct StackDataType + if (classData->m_typeId == SerializeTypeInfo::GetUuid()) + { + // determine if this is entity ref or just entityId (please refer to the function documentation for more info) + bool isEntityId = false; + if (!parentStack.empty() && parentStack.back()->m_typeId == SerializeTypeInfo::GetUuid()) + { + // our parent in the entity (currently entity has only one EntityId member, but we can check the offset for future proof + AZ_Assert(elementData && strcmp(elementData->m_name, "Id") == 0, "class Entity, should have only ONE EntityId member, the actual entity id!"); + isEntityId = true; + } + + EntityId* entityIdPtr = (elementData->m_flags & SerializeContext::ClassElement::FLG_POINTER) ? + *reinterpret_cast(ptr) : reinterpret_cast(ptr); + visitor(*entityIdPtr, isEntityId, elementData); + } + + parentStack.push_back(classData); + return true; + }; + + auto endCB = [ &]() -> bool + { + parentStack.pop_back(); + return true; + }; + + SerializeContext::EnumerateInstanceCallContext callContext( + beginCB, + endCB, + context, + SerializeContext::ENUM_ACCESS_FOR_READ, + nullptr + ); + + context->EnumerateInstanceConst( + &callContext, + classPtr, + classUuid, + nullptr, + nullptr + ); + } + + //========================================================================= + // GetApplicationSerializeContext + //========================================================================= + SerializeContext* GetApplicationSerializeContext() + { + SerializeContext* context = nullptr; + EBUS_EVENT_RESULT(context, ComponentApplicationBus, GetSerializeContext); + return context; + } + + //========================================================================= + // FindFirstDerivedComponent + //========================================================================= + Component* FindFirstDerivedComponent(const Entity* entity, const Uuid& typeId) + { + for (AZ::Component* component : entity->GetComponents()) { - const SerializeContext::ClassData* m_classData; - const SerializeContext::ClassElement* m_elementData; - void* m_dataPtr; - bool m_isModifiedContainer; + if (azrtti_istypeof(typeId, component)) + { + return component; + } + } + return nullptr; + } + + Component* FindFirstDerivedComponent(EntityId entityId, const Uuid& typeId) + { + Entity* entity{}; + ComponentApplicationBus::BroadcastResult(entity, &ComponentApplicationRequests::FindEntity, entityId); + return entity ? FindFirstDerivedComponent(entity, typeId) : nullptr; + } + + //========================================================================= + // FindDerivedComponents + //========================================================================= + Entity::ComponentArrayType FindDerivedComponents(const Entity* entity, const Uuid& typeId) + { + Entity::ComponentArrayType result; + for (AZ::Component* component : entity->GetComponents()) + { + if (azrtti_istypeof(typeId, component)) + { + result.push_back(component); + } + } + return result; + } + + Entity::ComponentArrayType FindDerivedComponents(EntityId entityId, const Uuid& typeId) + { + Entity* entity{}; + ComponentApplicationBus::BroadcastResult(entity, &ComponentApplicationRequests::FindEntity, entityId); + return entity ? FindDerivedComponents(entity, typeId) : Entity::ComponentArrayType(); + } + + bool EnumerateBaseRecursive(SerializeContext* context, const EnumerateBaseRecursiveVisitor& baseClassVisitor, const TypeId& typeToExamine) + { + AZ_Assert(context, "CheckDeclaresSerializeBaseClass called with no serialize context."); + if (!context) + { + return false; + } + + AZStd::fixed_vector knownBaseClasses = { typeToExamine }; // avoid allocating heap here if possible. 64 types are 64*sizeof(Uuid) which is only 1k. + bool foundBaseClass = false; + auto enumerateBaseVisitor = [&baseClassVisitor, &knownBaseClasses](const AZ::SerializeContext::ClassData* classData, const TypeId& examineTypeId) + { + if (!classData) + { + return false; + } + + if (AZStd::find(knownBaseClasses.begin(), knownBaseClasses.end(), classData->m_typeId) == knownBaseClasses.end()) + { + if (knownBaseClasses.size() == 64) + { + // this should be pretty unlikely since a single class would have to have many other classes in its heirarchy + // and it'd all have to be basically in one layer, as we are popping as we explore. + AZ_WarningOnce("EntityUtils", false, "While trying to find a base class, all available slots were consumed. consider increasing the size of knownBaseClasses.\n"); + // we cannot continue any further, assume we did not find it. + return false; + } + knownBaseClasses.push_back(classData->m_typeId); + } + + return baseClassVisitor(classData, examineTypeId); }; - //========================================================================= - // EnumerateEntityIds - //========================================================================= - void EnumerateEntityIds(const void* classPtr, const Uuid& classUuid, const EntityIdVisitor& visitor, SerializeContext* context) + while (!knownBaseClasses.empty() && !foundBaseClass) { - AZ_PROFILE_FUNCTION(AzCore); + TypeId toExamine = knownBaseClasses.back(); + knownBaseClasses.pop_back(); - if (!context) - { - context = GetApplicationSerializeContext(); - if (!context) - { - AZ_Error("Serialization", false, "No serialize context provided! Failed to get component application default serialize context! ComponentApp is not started or input serialize context should not be null!"); - return; - } - } - AZStd::vector parentStack; - parentStack.reserve(30); - auto beginCB = [ &](void* ptr, const SerializeContext::ClassData* classData, const SerializeContext::ClassElement* elementData) -> bool - { - (void)elementData; - - if (classData->m_typeId == SerializeTypeInfo::GetUuid()) - { - // determine if this is entity ref or just entityId (please refer to the function documentation for more info) - bool isEntityId = false; - if (!parentStack.empty() && parentStack.back()->m_typeId == SerializeTypeInfo::GetUuid()) - { - // our parent in the entity (currently entity has only one EntityId member, but we can check the offset for future proof - AZ_Assert(elementData && strcmp(elementData->m_name, "Id") == 0, "class Entity, should have only ONE EntityId member, the actual entity id!"); - isEntityId = true; - } - - EntityId* entityIdPtr = (elementData->m_flags & SerializeContext::ClassElement::FLG_POINTER) ? - *reinterpret_cast(ptr) : reinterpret_cast(ptr); - visitor(*entityIdPtr, isEntityId, elementData); - } - - parentStack.push_back(classData); - return true; - }; - - auto endCB = [ &]() -> bool - { - parentStack.pop_back(); - return true; - }; - - SerializeContext::EnumerateInstanceCallContext callContext( - beginCB, - endCB, - context, - SerializeContext::ENUM_ACCESS_FOR_READ, - nullptr - ); - - context->EnumerateInstanceConst( - &callContext, - classPtr, - classUuid, - nullptr, - nullptr - ); + context->EnumerateBase(enumerateBaseVisitor, toExamine); } - //========================================================================= - // GetApplicationSerializeContext - //========================================================================= - SerializeContext* GetApplicationSerializeContext() - { - SerializeContext* context = nullptr; - EBUS_EVENT_RESULT(context, ComponentApplicationBus, GetSerializeContext); - return context; - } + return foundBaseClass; + } - //========================================================================= - // FindFirstDerivedComponent - //========================================================================= - Component* FindFirstDerivedComponent(const Entity* entity, const Uuid& typeId) + bool CheckIfClassIsDeprecated(SerializeContext* context, const TypeId& typeToExamine) + { + bool isDeprecated = false; + auto classVisitorFn = [&isDeprecated](const AZ::SerializeContext::ClassData* classData, const TypeId& /*rttiBase*/) { - for (AZ::Component* component : entity->GetComponents()) - { - if (azrtti_istypeof(typeId, component)) - { - return component; - } - } - return nullptr; - } - - Component* FindFirstDerivedComponent(EntityId entityId, const Uuid& typeId) - { - Entity* entity{}; - ComponentApplicationBus::BroadcastResult(entity, &ComponentApplicationRequests::FindEntity, entityId); - return entity ? FindFirstDerivedComponent(entity, typeId) : nullptr; - } - - //========================================================================= - // FindDerivedComponents - //========================================================================= - Entity::ComponentArrayType FindDerivedComponents(const Entity* entity, const Uuid& typeId) - { - Entity::ComponentArrayType result; - for (AZ::Component* component : entity->GetComponents()) - { - if (azrtti_istypeof(typeId, component)) - { - result.push_back(component); - } - } - return result; - } - - Entity::ComponentArrayType FindDerivedComponents(EntityId entityId, const Uuid& typeId) - { - Entity* entity{}; - ComponentApplicationBus::BroadcastResult(entity, &ComponentApplicationRequests::FindEntity, entityId); - return entity ? FindDerivedComponents(entity, typeId) : Entity::ComponentArrayType(); - } - - bool EnumerateBaseRecursive(SerializeContext* context, const EnumerateBaseRecursiveVisitor& baseClassVisitor, const TypeId& typeToExamine) - { - AZ_Assert(context, "CheckDeclaresSerializeBaseClass called with no serialize context."); - if (!context) + // Stop iterating once we stop receiving SerializeContext::ClassData*. + if (!classData) { return false; } - AZStd::fixed_vector knownBaseClasses = { typeToExamine }; // avoid allocating heap here if possible. 64 types are 64*sizeof(Uuid) which is only 1k. - bool foundBaseClass = false; - auto enumerateBaseVisitor = [&baseClassVisitor, &knownBaseClasses](const AZ::SerializeContext::ClassData* classData, const TypeId& examineTypeId) - { - if (!classData) - { - return false; - } - - if (AZStd::find(knownBaseClasses.begin(), knownBaseClasses.end(), classData->m_typeId) == knownBaseClasses.end()) - { - if (knownBaseClasses.size() == 64) - { - // this should be pretty unlikely since a single class would have to have many other classes in its heirarchy - // and it'd all have to be basically in one layer, as we are popping as we explore. - AZ_WarningOnce("EntityUtils", false, "While trying to find a base class, all available slots were consumed. consider increasing the size of knownBaseClasses.\n"); - // we cannot continue any further, assume we did not find it. - return false; - } - knownBaseClasses.push_back(classData->m_typeId); - } - - return baseClassVisitor(classData, examineTypeId); - }; - - while (!knownBaseClasses.empty() && !foundBaseClass) - { - TypeId toExamine = knownBaseClasses.back(); - knownBaseClasses.pop_back(); - - context->EnumerateBase(enumerateBaseVisitor, toExamine); - } - - return foundBaseClass; - } - - bool CheckIfClassIsDeprecated(SerializeContext* context, const TypeId& typeToExamine) - { - bool isDeprecated = false; - auto classVisitorFn = [&isDeprecated](const AZ::SerializeContext::ClassData* classData, const TypeId& /*rttiBase*/) - { - // Stop iterating once we stop receiving SerializeContext::ClassData*. - if (!classData) - { - return false; - } - - // Stop iterating if we've found that the class is deprecated - if (classData->IsDeprecated()) - { - isDeprecated = true; - return false; - } - - return true; // keep iterating - }; - - // Check if the type is deprecated - const AZ::SerializeContext::ClassData* classData = context->FindClassData(typeToExamine); + // Stop iterating if we've found that the class is deprecated if (classData->IsDeprecated()) { - return true; - } - - // Check if any of its bases are deprecated - EnumerateBaseRecursive(context, classVisitorFn, typeToExamine); - - return isDeprecated; - } - - bool CheckDeclaresSerializeBaseClass(SerializeContext* context, const TypeId& typeToFind, const TypeId& typeToExamine) - { - AZ_Assert(context, "CheckDeclaresSerializeBaseClass called with no serialize context."); - if (!context) - { + isDeprecated = true; return false; } - bool foundBaseClass = false; - auto baseClassVisitorFn = [&typeToFind, &foundBaseClass](const AZ::SerializeContext::ClassData* reflectedBase, const TypeId& /*rttiBase*/) - { - if (!reflectedBase) - { - foundBaseClass = false; - return false; // stop iterating - } + return true; // keep iterating + }; - foundBaseClass = (reflectedBase->m_typeId == typeToFind); - if (foundBaseClass) - { - return false; // we have a base, stop iterating - } - - return true; // keep iterating - }; - - EnumerateBaseRecursive(context, baseClassVisitorFn, typeToExamine); - - return foundBaseClass; - } - - bool RemoveDuplicateServicesOfAndAfterIterator( - const ComponentDescriptor::DependencyArrayType::iterator& iterator, - ComponentDescriptor::DependencyArrayType& providedServiceArray, - const Entity* entity) + // Check if the type is deprecated + const AZ::SerializeContext::ClassData* classData = context->FindClassData(typeToExamine); + if (classData->IsDeprecated()) { - // Build types that strip out AZ_Warnings will complain that entity is unused without this. - (void)entity; - if (iterator == providedServiceArray.end()) - { - return false; - } - - bool duplicateFound = false; - - for (ComponentDescriptor::DependencyArrayType::iterator duplicateCheckIter = AZStd::next(iterator); - duplicateCheckIter != providedServiceArray.end();) - { - if (*iterator == *duplicateCheckIter) - { - AZ_Warning("Entity", false, "Duplicate service %d found on entity %s [%s]", - *duplicateCheckIter, - entity ? entity->GetName().c_str() : "Entity not provided", - entity ? entity->GetId().ToString().c_str() : ""); - duplicateCheckIter = providedServiceArray.erase(duplicateCheckIter); - duplicateFound = true; - } - else - { - ++duplicateCheckIter; - } - } - return duplicateFound; + return true; } - } // namespace EntityUtils -} // namespace AZ + + // Check if any of its bases are deprecated + EnumerateBaseRecursive(context, classVisitorFn, typeToExamine); + + return isDeprecated; + } + + bool CheckDeclaresSerializeBaseClass(SerializeContext* context, const TypeId& typeToFind, const TypeId& typeToExamine) + { + AZ_Assert(context, "CheckDeclaresSerializeBaseClass called with no serialize context."); + if (!context) + { + return false; + } + + bool foundBaseClass = false; + auto baseClassVisitorFn = [&typeToFind, &foundBaseClass](const AZ::SerializeContext::ClassData* reflectedBase, const TypeId& /*rttiBase*/) + { + if (!reflectedBase) + { + foundBaseClass = false; + return false; // stop iterating + } + + foundBaseClass = (reflectedBase->m_typeId == typeToFind); + if (foundBaseClass) + { + return false; // we have a base, stop iterating + } + + return true; // keep iterating + }; + + EnumerateBaseRecursive(context, baseClassVisitorFn, typeToExamine); + + return foundBaseClass; + } + + bool RemoveDuplicateServicesOfAndAfterIterator( + const ComponentDescriptor::DependencyArrayType::iterator& iterator, + ComponentDescriptor::DependencyArrayType& providedServiceArray, + const Entity* entity) + { + // Build types that strip out AZ_Warnings will complain that entity is unused without this. + (void)entity; + if (iterator == providedServiceArray.end()) + { + return false; + } + + bool duplicateFound = false; + + for (ComponentDescriptor::DependencyArrayType::iterator duplicateCheckIter = AZStd::next(iterator); + duplicateCheckIter != providedServiceArray.end();) + { + if (*iterator == *duplicateCheckIter) + { + AZ_Warning("Entity", false, "Duplicate service %d found on entity %s [%s]", + *duplicateCheckIter, + entity ? entity->GetName().c_str() : "Entity not provided", + entity ? entity->GetId().ToString().c_str() : ""); + duplicateCheckIter = providedServiceArray.erase(duplicateCheckIter); + duplicateFound = true; + } + else + { + ++duplicateCheckIter; + } + } + return duplicateFound; + } +} // namespace AZ::EntityUtils diff --git a/Code/Framework/AzCore/AzCore/Compression/zstd_compression.cpp b/Code/Framework/AzCore/AzCore/Compression/zstd_compression.cpp index 9db59bc781..ecab62d123 100644 --- a/Code/Framework/AzCore/AzCore/Compression/zstd_compression.cpp +++ b/Code/Framework/AzCore/AzCore/Compression/zstd_compression.cpp @@ -57,6 +57,7 @@ void ZStd::StartCompressor(unsigned int compressionLevel) ZSTD_customMem customAlloc; customAlloc.customAlloc = reinterpret_cast(&AllocateMem); customAlloc.customFree = &FreeMem; + customAlloc.opaque = nullptr; AZ_UNUSED(compressionLevel); m_streamCompression = (ZSTD_createCStream_advanced(customAlloc)); diff --git a/Code/Framework/AzCore/AzCore/Debug/AssetTracking.cpp b/Code/Framework/AzCore/AzCore/Debug/AssetTracking.cpp index 3cb895c992..c03b75d118 100644 --- a/Code/Framework/AzCore/AzCore/Debug/AssetTracking.cpp +++ b/Code/Framework/AzCore/AzCore/Debug/AssetTracking.cpp @@ -14,323 +14,313 @@ #include #include -namespace AZ +namespace AZ::Debug { - namespace Debug + namespace { - namespace + struct AssetTreeNode; + + // Per-thread data that needs to be stored. + struct ThreadData { - struct AssetTreeNode; - - // Per-thread data that needs to be stored. - struct ThreadData - { - AZStd::vector m_currentAssetStack; - }; - - // Access thread data through a virtual function to ensure that the same thread-local data is being shared across DLLs. - // Otherwise, the thread_local variables are replicated across DLLs that link the AzCore library, and you'll get a - // different version in each module. - class ThreadDataProvider - { - public: - virtual ThreadData& GetThreadData() = 0; - }; - } - - class AssetTrackingImpl final : - public ThreadDataProvider - { - public: - AZ_TYPE_INFO(AssetTrackingImpl, "{01E2A099-3523-40BE-80E0-E0ADD861BEE1}"); - AZ_CLASS_ALLOCATOR(AssetTrackingImpl, OSAllocator, 0); - - AssetTrackingImpl(AssetTreeBase* assetTree, AssetAllocationTableBase* allocationTable); - ~AssetTrackingImpl(); - - void AssetBegin(const char* id, const char* file, int line); - void AssetAttach(void* otherAllocation, const char* file, int line); - void AssetEnd(); - - ThreadData& GetThreadData() override; - - private: - static EnvironmentVariable& GetEnvironmentVariable(); - static AssetTrackingImpl* GetSharedInstance(); - static ThreadData& GetSharedThreadData(); - - using PrimaryAssets = AZStd::unordered_map, AZStd::equal_to, AZStdAssetTrackingAllocator>; - using ThreadData = ThreadData; - using mutex_type = AZStd::mutex; - using lock_type = AZStd::lock_guard; - - mutex_type m_mutex; - PrimaryAssets m_primaryAssets; - AssetTreeNodeBase* m_assetRoot = nullptr; - AssetAllocationTableBase* m_allocationTable = nullptr; - bool m_performingAnalysis = false; - - friend class AssetTracking; - friend class AssetTracking::Scope; + AZStd::vector m_currentAssetStack; }; + // Access thread data through a virtual function to ensure that the same thread-local data is being shared across DLLs. + // Otherwise, the thread_local variables are replicated across DLLs that link the AzCore library, and you'll get a + // different version in each module. + class ThreadDataProvider + { + public: + virtual ThreadData& GetThreadData() = 0; + }; } -} + + class AssetTrackingImpl final : + public ThreadDataProvider + { + public: + AZ_TYPE_INFO(AssetTrackingImpl, "{01E2A099-3523-40BE-80E0-E0ADD861BEE1}"); + AZ_CLASS_ALLOCATOR(AssetTrackingImpl, OSAllocator, 0); + + AssetTrackingImpl(AssetTreeBase* assetTree, AssetAllocationTableBase* allocationTable); + ~AssetTrackingImpl(); + + void AssetBegin(const char* id, const char* file, int line); + void AssetAttach(void* otherAllocation, const char* file, int line); + void AssetEnd(); + + ThreadData& GetThreadData() override; + + private: + static EnvironmentVariable& GetEnvironmentVariable(); + static AssetTrackingImpl* GetSharedInstance(); + static ThreadData& GetSharedThreadData(); + + using PrimaryAssets = AZStd::unordered_map, AZStd::equal_to, AZStdAssetTrackingAllocator>; + using ThreadData = ThreadData; + using mutex_type = AZStd::mutex; + using lock_type = AZStd::lock_guard; + + mutex_type m_mutex; + PrimaryAssets m_primaryAssets; + AssetTreeNodeBase* m_assetRoot = nullptr; + AssetAllocationTableBase* m_allocationTable = nullptr; + bool m_performingAnalysis = false; + + friend class AssetTracking; + friend class AssetTracking::Scope; + }; + /////////////////////////////////////////////////////////////////////////////// // AssetTrackingImpl methods /////////////////////////////////////////////////////////////////////////////// -namespace AZ -{ - namespace Debug + AssetTrackingImpl::AssetTrackingImpl(AssetTreeBase* assetTree, AssetAllocationTableBase* allocationTable) : + m_assetRoot(&assetTree->GetRoot()), + m_allocationTable(allocationTable) { - AssetTrackingImpl::AssetTrackingImpl(AssetTreeBase* assetTree, AssetAllocationTableBase* allocationTable) : - m_assetRoot(&assetTree->GetRoot()), - m_allocationTable(allocationTable) - { - AZ_Assert(!GetSharedInstance(), "Only one AssetTrackingImpl can exist!"); + AZ_Assert(!GetSharedInstance(), "Only one AssetTrackingImpl can exist!"); - GetEnvironmentVariable().Set(this); - AllocatorManager::Instance().EnterProfilingMode(); + GetEnvironmentVariable().Set(this); + AllocatorManager::Instance().EnterProfilingMode(); + } + + AssetTrackingImpl::~AssetTrackingImpl() + { + AllocatorManager::Instance().ExitProfilingMode(); + GetEnvironmentVariable().Reset(); + } + + void AssetTrackingImpl::AssetBegin(const char* id, const char* file, int line) + { + // In the future it may be desirable to organize assets based on where in code the asset was entered into. + // For now these are ignored. + AZ_UNUSED(file); + AZ_UNUSED(line); + + using namespace Internal; + + AssetTrackingId assetId(id); + auto& threadData = GetSharedThreadData(); + AssetTreeNodeBase* parentAsset = threadData.m_currentAssetStack.empty() ? nullptr : threadData.m_currentAssetStack.back(); + AssetTreeNodeBase* childAsset; + AssetPrimaryInfo* assetPrimaryInfo; + + if (!parentAsset) + { + parentAsset = m_assetRoot; } - AssetTrackingImpl::~AssetTrackingImpl() { - AllocatorManager::Instance().ExitProfilingMode(); - GetEnvironmentVariable().Reset(); - } + lock_type lock(m_mutex); - void AssetTrackingImpl::AssetBegin(const char* id, const char* file, int line) - { - // In the future it may be desirable to organize assets based on where in code the asset was entered into. - // For now these are ignored. - AZ_UNUSED(file); - AZ_UNUSED(line); + // Locate or create the primary record for this asset + auto primaryItr = m_primaryAssets.find(assetId); - using namespace Internal; - - AssetTrackingId assetId(id); - auto& threadData = GetSharedThreadData(); - AssetTreeNodeBase* parentAsset = threadData.m_currentAssetStack.empty() ? nullptr : threadData.m_currentAssetStack.back(); - AssetTreeNodeBase* childAsset; - AssetPrimaryInfo* assetPrimaryInfo; - - if (!parentAsset) + if (primaryItr != m_primaryAssets.end()) { - parentAsset = m_assetRoot; - } - - { - lock_type lock(m_mutex); - - // Locate or create the primary record for this asset - auto primaryItr = m_primaryAssets.find(assetId); - - if (primaryItr != m_primaryAssets.end()) - { - assetPrimaryInfo = &primaryItr->second; - } - else - { - auto insertResult = m_primaryAssets.emplace(assetId, AssetPrimaryInfo()); - assetPrimaryInfo = &insertResult.first->second; - assetPrimaryInfo->m_id = &insertResult.first->first; - } - - // Add this asset to the stack for this thread's context - childAsset = parentAsset->FindOrAddChild(assetId, assetPrimaryInfo); - } - - threadData.m_currentAssetStack.push_back(childAsset); - } - - void AssetTrackingImpl::AssetAttach(void* otherAllocation, const char* file, int line) - { - AZ_UNUSED(file); - AZ_UNUSED(line); - - using namespace Internal; - - AssetTreeNodeBase* assetInfo = m_allocationTable->FindAllocation(otherAllocation); - - // We will push back a nullptr if there is no asset, this is necessary to balance the call to AssetEnd() - GetSharedThreadData().m_currentAssetStack.push_back(assetInfo); - } - - void AssetTrackingImpl::AssetEnd() - { - AZ_Assert(!GetSharedThreadData().m_currentAssetStack.empty(), "AssetEnd() called without matching AssetBegin() or AssetAttach. Use the AZ_ASSET_NAMED_SCOPE and AZ_ASSET_ATTACH_TO_SCOPE macros to avoid this!"); - GetSharedThreadData().m_currentAssetStack.pop_back(); - } - - AssetTrackingImpl* AssetTrackingImpl::GetSharedInstance() - { - auto environmentVariable = GetEnvironmentVariable(); - - if(environmentVariable) - { - return *environmentVariable; - } - - return nullptr; - } - - ThreadData& AssetTrackingImpl::GetSharedThreadData() - { - // Cast to the base type so our virtual call doesn't get optimized away. We require GetThreadData() to be executed in the same DLL every time. - return static_cast(GetSharedInstance())->GetThreadData(); - } - - AssetTrackingImpl::ThreadData& AssetTrackingImpl::GetThreadData() - { - static thread_local ThreadData* data = nullptr; - static thread_local typename AZStd::aligned_storage_t storage; - - if (!data) - { - data = new (&storage) ThreadData; - } - - return *data; - } - - EnvironmentVariable& AssetTrackingImpl::GetEnvironmentVariable() - { - static EnvironmentVariable assetTrackingImpl = Environment::CreateVariable(AzTypeInfo::Name()); - - return assetTrackingImpl; - } - - /////////////////////////////////////////////////////////////////////////////// - // AssetTracking::Scope functions - /////////////////////////////////////////////////////////////////////////////// - - AssetTracking::Scope AssetTracking::Scope::ScopeFromAssetId(const char* file, int line, const char* fmt, ...) - { - if (auto impl = AssetTrackingImpl::GetSharedInstance()) - { - static const int BUFFER_SIZE = 1024; - - char buffer[BUFFER_SIZE]; - va_list args; - va_start(args, fmt); - azvsnprintf(buffer, BUFFER_SIZE, fmt, args); - va_end(args); - - impl->AssetBegin(buffer, file, line); - } - - return Scope(); - } - - AssetTracking::Scope AssetTracking::Scope::ScopeFromAttachment(void* attachTo, const char* file, int line) - { - if (auto impl = AssetTrackingImpl::GetSharedInstance()) - { - impl->AssetAttach(attachTo, file, line); - } - - return Scope(); - } - - AssetTracking::Scope::~Scope() - { - if (auto impl = AssetTrackingImpl::GetSharedInstance()) - { - impl->AssetEnd(); - } - } - - AssetTracking::Scope::Scope() - { - } - - /////////////////////////////////////////////////////////////////////////////// - // AssetTracking functions - /////////////////////////////////////////////////////////////////////////////// - - void AssetTracking::EnterScopeByAssetId(const char* file, int line, const char* fmt, ...) - { - if (auto impl = AssetTrackingImpl::GetSharedInstance()) - { - static const int BUFFER_SIZE = 1024; - - char buffer[BUFFER_SIZE]; - va_list args; - va_start(args, fmt); - azvsnprintf(buffer, BUFFER_SIZE, fmt, args); - va_end(args); - - impl->AssetBegin(buffer, file, line); - } - } - - void AssetTracking::EnterScopeByAttachment(void* attachTo, const char* file, int line) - { - if (auto impl = AssetTrackingImpl::GetSharedInstance()) - { - impl->AssetAttach(attachTo, file, line); - } - } - - void AssetTracking::ExitScope() - { - if (auto impl = AssetTrackingImpl::GetSharedInstance()) - { - impl->AssetEnd(); - } - } - - const char* AssetTracking::GetDebugScope() - { - // Output debug information about the current asset scope in the current thread. - // Do not use in production code. -#ifndef RELEASE - static const int BUFFER_SIZE = 1024; - static char buffer[BUFFER_SIZE]; - const auto& assetStack = AssetTrackingImpl::GetSharedInstance()->GetThreadData().m_currentAssetStack; - - if (assetStack.empty()) - { - azsnprintf(buffer, BUFFER_SIZE, ""); + assetPrimaryInfo = &primaryItr->second; } else { - char* pos = buffer; - for (auto itr = assetStack.rbegin(); itr != assetStack.rend(); ++itr) - { - pos += azsnprintf(pos, BUFFER_SIZE - (pos - buffer), "%s\n", (*itr)->GetAssetPrimaryInfo()->m_id->m_id.c_str()); - - if (pos >= buffer + BUFFER_SIZE) - { - break; - } - } + auto insertResult = m_primaryAssets.emplace(assetId, AssetPrimaryInfo()); + assetPrimaryInfo = &insertResult.first->second; + assetPrimaryInfo->m_id = &insertResult.first->first; } - return buffer; -#else - return ""; -#endif + // Add this asset to the stack for this thread's context + childAsset = parentAsset->FindOrAddChild(assetId, assetPrimaryInfo); } - AssetTracking::AssetTracking(AssetTreeBase* assetTree, AssetAllocationTableBase* allocationTable) + threadData.m_currentAssetStack.push_back(childAsset); + } + + void AssetTrackingImpl::AssetAttach(void* otherAllocation, const char* file, int line) + { + AZ_UNUSED(file); + AZ_UNUSED(line); + + using namespace Internal; + + AssetTreeNodeBase* assetInfo = m_allocationTable->FindAllocation(otherAllocation); + + // We will push back a nullptr if there is no asset, this is necessary to balance the call to AssetEnd() + GetSharedThreadData().m_currentAssetStack.push_back(assetInfo); + } + + void AssetTrackingImpl::AssetEnd() + { + AZ_Assert(!GetSharedThreadData().m_currentAssetStack.empty(), "AssetEnd() called without matching AssetBegin() or AssetAttach. Use the AZ_ASSET_NAMED_SCOPE and AZ_ASSET_ATTACH_TO_SCOPE macros to avoid this!"); + GetSharedThreadData().m_currentAssetStack.pop_back(); + } + + AssetTrackingImpl* AssetTrackingImpl::GetSharedInstance() + { + auto environmentVariable = GetEnvironmentVariable(); + + if(environmentVariable) { - m_impl.reset(aznew AssetTrackingImpl(assetTree, allocationTable)); + return *environmentVariable; } - AssetTracking::~AssetTracking() + return nullptr; + } + + ThreadData& AssetTrackingImpl::GetSharedThreadData() + { + // Cast to the base type so our virtual call doesn't get optimized away. We require GetThreadData() to be executed in the same DLL every time. + return static_cast(GetSharedInstance())->GetThreadData(); + } + + AssetTrackingImpl::ThreadData& AssetTrackingImpl::GetThreadData() + { + static thread_local ThreadData* data = nullptr; + static thread_local typename AZStd::aligned_storage_t storage; + + if (!data) { + data = new (&storage) ThreadData; } - AssetTreeNodeBase* AssetTracking::GetCurrentThreadAsset() const - { - const auto& assetStack = m_impl->GetThreadData().m_currentAssetStack; - AssetTreeNodeBase* result = assetStack.empty() ? nullptr : assetStack.back(); + return *data; + } - return result; + EnvironmentVariable& AssetTrackingImpl::GetEnvironmentVariable() + { + static EnvironmentVariable assetTrackingImpl = Environment::CreateVariable(AzTypeInfo::Name()); + + return assetTrackingImpl; + } + + /////////////////////////////////////////////////////////////////////////////// + // AssetTracking::Scope functions + /////////////////////////////////////////////////////////////////////////////// + + AssetTracking::Scope AssetTracking::Scope::ScopeFromAssetId(const char* file, int line, const char* fmt, ...) + { + if (auto impl = AssetTrackingImpl::GetSharedInstance()) + { + static const int BUFFER_SIZE = 1024; + + char buffer[BUFFER_SIZE]; + va_list args; + va_start(args, fmt); + azvsnprintf(buffer, BUFFER_SIZE, fmt, args); + va_end(args); + + impl->AssetBegin(buffer, file, line); + } + + return Scope(); + } + + AssetTracking::Scope AssetTracking::Scope::ScopeFromAttachment(void* attachTo, const char* file, int line) + { + if (auto impl = AssetTrackingImpl::GetSharedInstance()) + { + impl->AssetAttach(attachTo, file, line); + } + + return Scope(); + } + + AssetTracking::Scope::~Scope() + { + if (auto impl = AssetTrackingImpl::GetSharedInstance()) + { + impl->AssetEnd(); } } + AssetTracking::Scope::Scope() + { + } + + /////////////////////////////////////////////////////////////////////////////// + // AssetTracking functions + /////////////////////////////////////////////////////////////////////////////// + + void AssetTracking::EnterScopeByAssetId(const char* file, int line, const char* fmt, ...) + { + if (auto impl = AssetTrackingImpl::GetSharedInstance()) + { + static const int BUFFER_SIZE = 1024; + + char buffer[BUFFER_SIZE]; + va_list args; + va_start(args, fmt); + azvsnprintf(buffer, BUFFER_SIZE, fmt, args); + va_end(args); + + impl->AssetBegin(buffer, file, line); + } + } + + void AssetTracking::EnterScopeByAttachment(void* attachTo, const char* file, int line) + { + if (auto impl = AssetTrackingImpl::GetSharedInstance()) + { + impl->AssetAttach(attachTo, file, line); + } + } + + void AssetTracking::ExitScope() + { + if (auto impl = AssetTrackingImpl::GetSharedInstance()) + { + impl->AssetEnd(); + } + } + + const char* AssetTracking::GetDebugScope() + { + // Output debug information about the current asset scope in the current thread. + // Do not use in production code. +#ifndef RELEASE + static const int BUFFER_SIZE = 1024; + static char buffer[BUFFER_SIZE]; + const auto& assetStack = AssetTrackingImpl::GetSharedInstance()->GetThreadData().m_currentAssetStack; + + if (assetStack.empty()) + { + azsnprintf(buffer, BUFFER_SIZE, ""); + } + else + { + char* pos = buffer; + for (auto itr = assetStack.rbegin(); itr != assetStack.rend(); ++itr) + { + pos += azsnprintf(pos, BUFFER_SIZE - (pos - buffer), "%s\n", (*itr)->GetAssetPrimaryInfo()->m_id->m_id.c_str()); + + if (pos >= buffer + BUFFER_SIZE) + { + break; + } + } + } + + return buffer; +#else + return ""; +#endif + } + + AssetTracking::AssetTracking(AssetTreeBase* assetTree, AssetAllocationTableBase* allocationTable) + { + m_impl.reset(aznew AssetTrackingImpl(assetTree, allocationTable)); + } + + AssetTracking::~AssetTracking() + { + } + + AssetTreeNodeBase* AssetTracking::GetCurrentThreadAsset() const + { + const auto& assetStack = m_impl->GetThreadData().m_currentAssetStack; + AssetTreeNodeBase* result = assetStack.empty() ? nullptr : assetStack.back(); + + return result; + } } // namespace AzFramework diff --git a/Code/Framework/AzCore/AzCore/Debug/EventTrace.cpp b/Code/Framework/AzCore/AzCore/Debug/EventTrace.cpp index 88a78de031..58fa1f6cca 100644 --- a/Code/Framework/AzCore/AzCore/Debug/EventTrace.cpp +++ b/Code/Framework/AzCore/AzCore/Debug/EventTrace.cpp @@ -11,19 +11,19 @@ #include #include -namespace AZ +namespace AZ::Debug { - namespace Debug + EventTrace::ScopedSlice::ScopedSlice(const char* name, const char* category) + : m_Name(name) + , m_Category(category) + , m_Time(AZStd::GetTimeNowMicroSecond()) { - EventTrace::ScopedSlice::ScopedSlice(const char* name, const char* category) - : m_Name(name) - , m_Category(category) - , m_Time(AZStd::GetTimeNowMicroSecond()) - {} - - EventTrace::ScopedSlice::~ScopedSlice() - { - EventTraceDrillerBus::TryQueueBroadcast(&EventTraceDrillerInterface::RecordSlice, m_Name, m_Category, AZStd::this_thread::get_id(), m_Time, (uint32_t)(AZStd::GetTimeNowMicroSecond() - m_Time)); - } } -} + + EventTrace::ScopedSlice::~ScopedSlice() + { + EventTraceDrillerBus::TryQueueBroadcast( + &EventTraceDrillerInterface::RecordSlice, m_Name, m_Category, AZStd::this_thread::get_id(), m_Time, + (uint32_t)(AZStd::GetTimeNowMicroSecond() - m_Time)); + } +} // namespace AZ::Debug diff --git a/Code/Framework/AzCore/AzCore/Debug/EventTraceDriller.cpp b/Code/Framework/AzCore/AzCore/Debug/EventTraceDriller.cpp index 658021b018..d0722616f3 100644 --- a/Code/Framework/AzCore/AzCore/Debug/EventTraceDriller.cpp +++ b/Code/Framework/AzCore/AzCore/Debug/EventTraceDriller.cpp @@ -11,151 +11,149 @@ #include #include -namespace AZ +namespace AZ::Debug { - namespace Debug + namespace Crc { - namespace Crc + constexpr u32 EventTraceDriller = AZ_CRC_CE("EventTraceDriller"); + constexpr u32 Slice = AZ_CRC_CE("Slice"); + constexpr u32 ThreadInfo = AZ_CRC_CE("ThreadInfo"); + constexpr u32 Name = AZ_CRC_CE("Name"); + constexpr u32 Category = AZ_CRC_CE("Category"); + constexpr u32 ThreadId = AZ_CRC_CE("ThreadId"); + constexpr u32 Timestamp = AZ_CRC_CE("Timestamp"); + constexpr u32 Duration = AZ_CRC_CE("Duration"); + constexpr u32 Instant = AZ_CRC_CE("Instant"); + } + + EventTraceDriller::EventTraceDriller() + { + EventTraceDrillerSetupBus::Handler::BusConnect(); + AZStd::ThreadDrillerEventBus::Handler::BusConnect(); + } + + EventTraceDriller::~EventTraceDriller() + { + AZStd::ThreadDrillerEventBus::Handler::BusDisconnect(); + EventTraceDrillerSetupBus::Handler::BusDisconnect(); + } + + void EventTraceDriller::Start(const Param* params, int numParams) + { + (void)params; + (void)numParams; + + EventTraceDrillerBus::Handler::BusConnect(); + TickBus::Handler::BusConnect(); + + EventTraceDrillerBus::AllowFunctionQueuing(true); + } + + void EventTraceDriller::Stop() + { + EventTraceDrillerBus::AllowFunctionQueuing(false); + EventTraceDrillerBus::ClearQueuedEvents(); + + EventTraceDrillerBus::Handler::BusDisconnect(); + TickBus::Handler::BusDisconnect(); + } + + void EventTraceDriller::OnTick(float deltaTime, ScriptTimePoint time) + { + (void)deltaTime; + (void)time; + + AZ_TRACE_METHOD(); + RecordThreads(); + EventTraceDrillerBus::ExecuteQueuedEvents(); + } + + void EventTraceDriller::SetThreadName(const AZStd::thread_id& id, const char* name) + { + AZStd::lock_guard lock(m_ThreadMutex); + m_Threads[(size_t)id.m_id] = ThreadData{ name }; + } + + void EventTraceDriller::OnThreadEnter(const AZStd::thread::id& id, const AZStd::thread_desc* desc) + { + if (desc && desc->m_name) { - const u32 EventTraceDriller = AZ_CRC("EventTraceDriller", 0xf7aeae55); - const u32 Slice = AZ_CRC("Slice", 0x3dae78a5); - const u32 ThreadInfo = AZ_CRC("ThreadInfo", 0x89bf78be); - const u32 Name = AZ_CRC("Name", 0x5e237e06); - const u32 Category = AZ_CRC("Category", 0x064c19c1); - const u32 ThreadId = AZ_CRC("ThreadId", 0xd0fd9043); - const u32 Timestamp = AZ_CRC("Timestamp", 0xa5d6e63e); - const u32 Duration = AZ_CRC("Duration", 0x865f80c0); - const u32 Instant = AZ_CRC("Instant", 0x0e9047ad); + SetThreadName(id, desc->m_name); } + } - EventTraceDriller::EventTraceDriller() + void EventTraceDriller::OnThreadExit(const AZStd::thread::id& id) + { + AZStd::lock_guard lock(m_ThreadMutex); + m_Threads.erase((size_t)id.m_id); + } + + void EventTraceDriller::RecordThreads() + { + if (!m_output || m_Threads.empty()) { - EventTraceDrillerSetupBus::Handler::BusConnect(); - AZStd::ThreadDrillerEventBus::Handler::BusConnect(); + return; } + // Main bus mutex guards m_output. + auto& context = EventTraceDrillerBus::GetOrCreateContext(); - EventTraceDriller::~EventTraceDriller() - { - AZStd::ThreadDrillerEventBus::Handler::BusDisconnect(); - EventTraceDrillerSetupBus::Handler::BusDisconnect(); - } - - void EventTraceDriller::Start(const Param* params, int numParams) - { - (void)params; - (void)numParams; - - EventTraceDrillerBus::Handler::BusConnect(); - TickBus::Handler::BusConnect(); - - EventTraceDrillerBus::AllowFunctionQueuing(true); - } - - void EventTraceDriller::Stop() - { - EventTraceDrillerBus::AllowFunctionQueuing(false); - EventTraceDrillerBus::ClearQueuedEvents(); - - EventTraceDrillerBus::Handler::BusDisconnect(); - TickBus::Handler::BusDisconnect(); - } - - void EventTraceDriller::OnTick(float deltaTime, ScriptTimePoint time) - { - (void)deltaTime; - (void)time; - - AZ_TRACE_METHOD(); - RecordThreads(); - EventTraceDrillerBus::ExecuteQueuedEvents(); - } - - void EventTraceDriller::SetThreadName(const AZStd::thread_id& id, const char* name) - { - AZStd::lock_guard lock(m_ThreadMutex); - m_Threads[(size_t)id.m_id] = ThreadData{ name }; - } - - void EventTraceDriller::OnThreadEnter(const AZStd::thread::id& id, const AZStd::thread_desc* desc) - { - if (desc && desc->m_name) - { - SetThreadName(id, desc->m_name); - } - } - - void EventTraceDriller::OnThreadExit(const AZStd::thread::id& id) - { - AZStd::lock_guard lock(m_ThreadMutex); - m_Threads.erase((size_t)id.m_id); - } - - void EventTraceDriller::RecordThreads() - { - if (m_output && m_Threads.size()) - { - // Main bus mutex guards m_output. - auto& context = EventTraceDrillerBus::GetOrCreateContext(); - - AZStd::scoped_lock lock(context.m_contextMutex, m_ThreadMutex); - for (const auto& keyValue : m_Threads) - { - m_output->BeginTag(Crc::EventTraceDriller); - m_output->BeginTag(Crc::ThreadInfo); - m_output->Write(Crc::ThreadId, keyValue.first); - m_output->Write(Crc::Name, keyValue.second.name); - m_output->EndTag(Crc::ThreadInfo); - m_output->EndTag(Crc::EventTraceDriller); - } - } - } - - void EventTraceDriller::RecordSlice( - const char* name, - const char* category, - const AZStd::thread_id threadId, - AZ::u64 timestamp, - AZ::u32 duration) + AZStd::scoped_lock lock(context.m_contextMutex, m_ThreadMutex); + for (const auto& keyValue : m_Threads) { m_output->BeginTag(Crc::EventTraceDriller); - m_output->BeginTag(Crc::Slice); - m_output->Write(Crc::Name, name); - m_output->Write(Crc::Category, category); - m_output->Write(Crc::ThreadId, (size_t)threadId.m_id); - m_output->Write(Crc::Timestamp, timestamp); - m_output->Write(Crc::Duration, std::max(duration, 1u)); - m_output->EndTag(Crc::Slice); - m_output->EndTag(Crc::EventTraceDriller); - } - - void EventTraceDriller::RecordInstantGlobal( - const char* name, - const char* category, - AZ::u64 timestamp) - { - m_output->BeginTag(Crc::EventTraceDriller); - m_output->BeginTag(Crc::Instant); - m_output->Write(Crc::Name, name); - m_output->Write(Crc::Category, category); - m_output->Write(Crc::Timestamp, timestamp); - m_output->EndTag(Crc::Instant); - m_output->EndTag(Crc::EventTraceDriller); - } - - void EventTraceDriller::RecordInstantThread( - const char* name, - const char* category, - const AZStd::thread_id threadId, - AZ::u64 timestamp) - { - m_output->BeginTag(Crc::EventTraceDriller); - m_output->BeginTag(Crc::Instant); - m_output->Write(Crc::Name, name); - m_output->Write(Crc::Category, category); - m_output->Write(Crc::ThreadId, (size_t)threadId.m_id); - m_output->Write(Crc::Timestamp, timestamp); - m_output->EndTag(Crc::Instant); + m_output->BeginTag(Crc::ThreadInfo); + m_output->Write(Crc::ThreadId, keyValue.first); + m_output->Write(Crc::Name, keyValue.second.name); + m_output->EndTag(Crc::ThreadInfo); m_output->EndTag(Crc::EventTraceDriller); } } -} + + void EventTraceDriller::RecordSlice( + const char* name, + const char* category, + const AZStd::thread_id threadId, + AZ::u64 timestamp, + AZ::u32 duration) + { + m_output->BeginTag(Crc::EventTraceDriller); + m_output->BeginTag(Crc::Slice); + m_output->Write(Crc::Name, name); + m_output->Write(Crc::Category, category); + m_output->Write(Crc::ThreadId, (size_t)threadId.m_id); + m_output->Write(Crc::Timestamp, timestamp); + m_output->Write(Crc::Duration, std::max(duration, 1u)); + m_output->EndTag(Crc::Slice); + m_output->EndTag(Crc::EventTraceDriller); + } + + void EventTraceDriller::RecordInstantGlobal( + const char* name, + const char* category, + AZ::u64 timestamp) + { + m_output->BeginTag(Crc::EventTraceDriller); + m_output->BeginTag(Crc::Instant); + m_output->Write(Crc::Name, name); + m_output->Write(Crc::Category, category); + m_output->Write(Crc::Timestamp, timestamp); + m_output->EndTag(Crc::Instant); + m_output->EndTag(Crc::EventTraceDriller); + } + + void EventTraceDriller::RecordInstantThread( + const char* name, + const char* category, + const AZStd::thread_id threadId, + AZ::u64 timestamp) + { + m_output->BeginTag(Crc::EventTraceDriller); + m_output->BeginTag(Crc::Instant); + m_output->Write(Crc::Name, name); + m_output->Write(Crc::Category, category); + m_output->Write(Crc::ThreadId, (size_t)threadId.m_id); + m_output->Write(Crc::Timestamp, timestamp); + m_output->EndTag(Crc::Instant); + m_output->EndTag(Crc::EventTraceDriller); + } +} // namespace AZ::Debug diff --git a/Code/Framework/AzCore/AzCore/Debug/TraceMessagesDriller.cpp b/Code/Framework/AzCore/AzCore/Debug/TraceMessagesDriller.cpp index 7e9e5b146e..dbe66838a4 100644 --- a/Code/Framework/AzCore/AzCore/Debug/TraceMessagesDriller.cpp +++ b/Code/Framework/AzCore/AzCore/Debug/TraceMessagesDriller.cpp @@ -9,94 +9,91 @@ #include #include -namespace AZ +namespace AZ::Debug { - namespace Debug + //========================================================================= + // Start + // [2/6/2013] + //========================================================================= + void TraceMessagesDriller::Start(const Param* params, int numParams) { - //========================================================================= - // Start - // [2/6/2013] - //========================================================================= - void TraceMessagesDriller::Start(const Param* params, int numParams) - { - (void)params; - (void)numParams; - BusConnect(); - } + (void)params; + (void)numParams; + BusConnect(); + } - //========================================================================= - // Stop - // [2/6/2013] - //========================================================================= - void TraceMessagesDriller::Stop() - { - BusDisconnect(); - } + //========================================================================= + // Stop + // [2/6/2013] + //========================================================================= + void TraceMessagesDriller::Stop() + { + BusDisconnect(); + } - //========================================================================= - // OnAssert - // [2/6/2013] - //========================================================================= - void TraceMessagesDriller::OnAssert(const char* message) - { - // Not sure if we can really capture assert since the code will stop executing very soon. - m_output->BeginTag(AZ_CRC("TraceMessagesDriller", 0xa61d1b00)); - m_output->Write(AZ_CRC("OnAssert", 0xb74db4ce), message); - m_output->EndTag(AZ_CRC("TraceMessagesDriller", 0xa61d1b00)); - } + //========================================================================= + // OnAssert + // [2/6/2013] + //========================================================================= + void TraceMessagesDriller::OnAssert(const char* message) + { + // Not sure if we can really capture assert since the code will stop executing very soon. + m_output->BeginTag(AZ_CRC_CE("TraceMessagesDriller")); + m_output->Write(AZ_CRC_CE("OnAssert"), message); + m_output->EndTag(AZ_CRC_CE("TraceMessagesDriller")); + } - //========================================================================= - // OnException - // [2/6/2013] - //========================================================================= - void TraceMessagesDriller::OnException(const char* message) - { - // Not sure if we can really capture exception since the code will stop executing very soon. - m_output->BeginTag(AZ_CRC("TraceMessagesDriller", 0xa61d1b00)); - m_output->Write(AZ_CRC("OnException", 0xfe457d12), message); - m_output->EndTag(AZ_CRC("TraceMessagesDriller", 0xa61d1b00)); - } + //========================================================================= + // OnException + // [2/6/2013] + //========================================================================= + void TraceMessagesDriller::OnException(const char* message) + { + // Not sure if we can really capture exception since the code will stop executing very soon. + m_output->BeginTag(AZ_CRC_CE("TraceMessagesDriller")); + m_output->Write(AZ_CRC_CE("OnException"), message); + m_output->EndTag(AZ_CRC_CE("TraceMessagesDriller")); + } - //========================================================================= - // OnError - // [2/6/2013] - //========================================================================= - void TraceMessagesDriller::OnError(const char* window, const char* message) - { - m_output->BeginTag(AZ_CRC("TraceMessagesDriller", 0xa61d1b00)); - m_output->BeginTag(AZ_CRC("OnError", 0x4993c634)); - m_output->Write(AZ_CRC("Window", 0x8be4f9dd), window); - m_output->Write(AZ_CRC("Message", 0xb6bd307f), message); - m_output->EndTag(AZ_CRC("OnError", 0x4993c634)); - m_output->EndTag(AZ_CRC("TraceMessagesDriller", 0xa61d1b00)); - } + //========================================================================= + // OnError + // [2/6/2013] + //========================================================================= + void TraceMessagesDriller::OnError(const char* window, const char* message) + { + m_output->BeginTag(AZ_CRC_CE("TraceMessagesDriller")); + m_output->BeginTag(AZ_CRC_CE("OnError")); + m_output->Write(AZ_CRC_CE("Window"), window); + m_output->Write(AZ_CRC_CE("Message"), message); + m_output->EndTag(AZ_CRC_CE("OnError")); + m_output->EndTag(AZ_CRC_CE("TraceMessagesDriller")); + } - //========================================================================= - // OnWarning - // [2/6/2013] - //========================================================================= - void TraceMessagesDriller::OnWarning(const char* window, const char* message) - { - m_output->BeginTag(AZ_CRC("TraceMessagesDriller", 0xa61d1b00)); - m_output->BeginTag(AZ_CRC("OnWarning", 0x7d90abea)); - m_output->Write(AZ_CRC("Window", 0x8be4f9dd), window); - m_output->Write(AZ_CRC("Message", 0xb6bd307f), message); - m_output->EndTag(AZ_CRC("OnWarning", 0x7d90abea)); - m_output->EndTag(AZ_CRC("TraceMessagesDriller", 0xa61d1b00)); - } + //========================================================================= + // OnWarning + // [2/6/2013] + //========================================================================= + void TraceMessagesDriller::OnWarning(const char* window, const char* message) + { + m_output->BeginTag(AZ_CRC_CE("TraceMessagesDriller")); + m_output->BeginTag(AZ_CRC_CE("OnWarning")); + m_output->Write(AZ_CRC_CE("Window"), window); + m_output->Write(AZ_CRC_CE("Message"), message); + m_output->EndTag(AZ_CRC_CE("OnWarning")); + m_output->EndTag(AZ_CRC_CE("TraceMessagesDriller")); + } - //========================================================================= - // OnPrintf - // [2/6/2013] - //========================================================================= - void TraceMessagesDriller::OnPrintf(const char* window, const char* message) - { - m_output->BeginTag(AZ_CRC("TraceMessagesDriller", 0xa61d1b00)); - m_output->BeginTag(AZ_CRC("OnPrintf", 0xd4b5c294)); - m_output->Write(AZ_CRC("Window", 0x8be4f9dd), window); - m_output->Write(AZ_CRC("Message", 0xb6bd307f), message); - m_output->EndTag(AZ_CRC("OnPrintf", 0xd4b5c294)); - m_output->EndTag(AZ_CRC("TraceMessagesDriller", 0xa61d1b00)); - } - } // namespace Debug + //========================================================================= + // OnPrintf + // [2/6/2013] + //========================================================================= + void TraceMessagesDriller::OnPrintf(const char* window, const char* message) + { + m_output->BeginTag(AZ_CRC_CE("TraceMessagesDriller")); + m_output->BeginTag(AZ_CRC_CE("OnPrintf")); + m_output->Write(AZ_CRC_CE("Window"), window); + m_output->Write(AZ_CRC_CE("Message"), message); + m_output->EndTag(AZ_CRC_CE("OnPrintf")); + m_output->EndTag(AZ_CRC_CE("TraceMessagesDriller")); + } } // namespace AZ diff --git a/Code/Framework/AzCore/AzCore/Debug/TraceReflection.cpp b/Code/Framework/AzCore/AzCore/Debug/TraceReflection.cpp index debbea5235..329a709994 100644 --- a/Code/Framework/AzCore/AzCore/Debug/TraceReflection.cpp +++ b/Code/Framework/AzCore/AzCore/Debug/TraceReflection.cpp @@ -12,283 +12,280 @@ #include #include -namespace AZ +namespace AZ::Debug { - namespace Debug + //! Trace Message Event Handler for Automation. + //! Since TraceMessageBus will be called from multiple threads and + //! python interpreter is single threaded, all the bus calls are + //! queued into a list and called at the end of the frame in the main thread. + //! @note this class is not using the usual AZ_EBUS_BEHAVIOR_BINDER + //! macro as the signature needs to be changed to connect to Tick bus. + class TraceMessageBusHandler + : public AZ::Debug::TraceMessageBus::Handler + , public AZ::BehaviorEBusHandler + , public AZ::TickBus::Handler { - //! Trace Message Event Handler for Automation. - //! Since TraceMessageBus will be called from multiple threads and - //! python interpreter is single threaded, all the bus calls are - //! queued into a list and called at the end of the frame in the main thread. - //! @note this class is not using the usual AZ_EBUS_BEHAVIOR_BINDER - //! macro as the signature needs to be changed to connect to Tick bus. - class TraceMessageBusHandler - : public AZ::Debug::TraceMessageBus::Handler - , public AZ::BehaviorEBusHandler - , public AZ::TickBus::Handler + public: + AZ_CLASS_ALLOCATOR(TraceMessageBusHandler, AZ::SystemAllocator, 0); + AZ_RTTI(TraceMessageBusHandler, "{5CDBAF09-5EB0-48AC-B327-2AF8601BB550}", AZ::BehaviorEBusHandler); + + TraceMessageBusHandler(); + + using EventFunctionsParameterPack = AZStd::Internal::pack_traits_arg_sequence< + decltype(&TraceMessageBusHandler::OnPreAssert), + decltype(&TraceMessageBusHandler::OnPreError), + decltype(&TraceMessageBusHandler::OnPreWarning), + decltype(&TraceMessageBusHandler::OnAssert), + decltype(&TraceMessageBusHandler::OnError), + decltype(&TraceMessageBusHandler::OnWarning), + decltype(&TraceMessageBusHandler::OnException), + decltype(&TraceMessageBusHandler::OnPrintf), + decltype(&TraceMessageBusHandler::OnOutput) + >; + + enum { - public: - AZ_CLASS_ALLOCATOR(TraceMessageBusHandler, AZ::SystemAllocator, 0); - AZ_RTTI(TraceMessageBusHandler, "{5CDBAF09-5EB0-48AC-B327-2AF8601BB550}", AZ::BehaviorEBusHandler); - - TraceMessageBusHandler(); - - using EventFunctionsParameterPack = AZStd::Internal::pack_traits_arg_sequence< - decltype(&TraceMessageBusHandler::OnPreAssert), - decltype(&TraceMessageBusHandler::OnPreError), - decltype(&TraceMessageBusHandler::OnPreWarning), - decltype(&TraceMessageBusHandler::OnAssert), - decltype(&TraceMessageBusHandler::OnError), - decltype(&TraceMessageBusHandler::OnWarning), - decltype(&TraceMessageBusHandler::OnException), - decltype(&TraceMessageBusHandler::OnPrintf), - decltype(&TraceMessageBusHandler::OnOutput) - >; - - enum - { - FN_OnPreAssert = 0, - FN_OnPreError, - FN_OnPreWarning, - FN_OnAssert, - FN_OnError, - FN_OnWarning, - FN_OnException, - FN_OnPrintf, - FN_OnOutput, - FN_MAX - }; - - static inline constexpr const char* m_functionNames[FN_MAX] = - { - "OnPreAssert", - "OnPreError", - "OnPreWarning", - "OnAssert", - "OnError", - "OnWarning", - "OnException", - "OnPrintf", - "OnOutput" - }; - - // AZ::BehaviorEBusHandler overrides... - int GetFunctionIndex(const char* functionName) const override; - void Disconnect() override; - bool Connect(AZ::BehaviorValueParameter* id = nullptr) override; - bool IsConnected() override; - bool IsConnectedId(AZ::BehaviorValueParameter* id) override; - - // TraceMessageBus - /* - * Note: Since at editor runtime there is already have a handler, for automation (OnPreAssert, OnPreWarning, OnPreWarning) - * must be used instead of (OnAssert, OnWarning, OnError) - */ - bool OnPreAssert(const char* fileName, int line, const char* func, const char* message) override; - bool OnPreError(const char* window, const char* fileName, int line, const char* func, const char* message) override; - bool OnPreWarning(const char* window, const char* fileName, int line, const char* func, const char* message) override; - bool OnAssert(const char* message) override; - bool OnError(const char* window, const char* message) override; - bool OnWarning(const char* window, const char* message) override; - bool OnException(const char* message) override; - bool OnPrintf(const char* window, const char* message) override; - bool OnOutput(const char* window, const char* message) override; - - // AZ::TickBus::Handler overrides ... - void OnTick(float deltaTime, AZ::ScriptTimePoint time) override; - int GetTickOrder() override; - - private: - void QueueMessageCall(AZStd::function messageCall); - void FlushMessageCalls(); - - AZStd::list> m_messageCalls; - AZStd::mutex m_messageCallsLock; + FN_OnPreAssert = 0, + FN_OnPreError, + FN_OnPreWarning, + FN_OnAssert, + FN_OnError, + FN_OnWarning, + FN_OnException, + FN_OnPrintf, + FN_OnOutput, + FN_MAX }; - TraceMessageBusHandler::TraceMessageBusHandler() + static inline constexpr const char* m_functionNames[FN_MAX] = { - m_events.resize(FN_MAX); + "OnPreAssert", + "OnPreError", + "OnPreWarning", + "OnAssert", + "OnError", + "OnWarning", + "OnException", + "OnPrintf", + "OnOutput" + }; - SetEvent(&TraceMessageBusHandler::OnPreAssert, m_functionNames[FN_OnPreAssert]); - SetEvent(&TraceMessageBusHandler::OnPreError, m_functionNames[FN_OnPreError]); - SetEvent(&TraceMessageBusHandler::OnPreWarning, m_functionNames[FN_OnPreWarning]); - SetEvent(&TraceMessageBusHandler::OnAssert, m_functionNames[FN_OnAssert]); - SetEvent(&TraceMessageBusHandler::OnError, m_functionNames[FN_OnError]); - SetEvent(&TraceMessageBusHandler::OnWarning, m_functionNames[FN_OnWarning]); - SetEvent(&TraceMessageBusHandler::OnException, m_functionNames[FN_OnException]); - SetEvent(&TraceMessageBusHandler::OnPrintf, m_functionNames[FN_OnPrintf]); - SetEvent(&TraceMessageBusHandler::OnOutput, m_functionNames[FN_OnOutput]); - } + // AZ::BehaviorEBusHandler overrides... + int GetFunctionIndex(const char* functionName) const override; + void Disconnect() override; + bool Connect(AZ::BehaviorValueParameter* id = nullptr) override; + bool IsConnected() override; + bool IsConnectedId(AZ::BehaviorValueParameter* id) override; - int TraceMessageBusHandler::GetFunctionIndex(const char* functionName) const + // TraceMessageBus + /* + * Note: Since at editor runtime there is already have a handler, for automation (OnPreAssert, OnPreWarning, OnPreWarning) + * must be used instead of (OnAssert, OnWarning, OnError) + */ + bool OnPreAssert(const char* fileName, int line, const char* func, const char* message) override; + bool OnPreError(const char* window, const char* fileName, int line, const char* func, const char* message) override; + bool OnPreWarning(const char* window, const char* fileName, int line, const char* func, const char* message) override; + bool OnAssert(const char* message) override; + bool OnError(const char* window, const char* message) override; + bool OnWarning(const char* window, const char* message) override; + bool OnException(const char* message) override; + bool OnPrintf(const char* window, const char* message) override; + bool OnOutput(const char* window, const char* message) override; + + // AZ::TickBus::Handler overrides ... + void OnTick(float deltaTime, AZ::ScriptTimePoint time) override; + int GetTickOrder() override; + + private: + void QueueMessageCall(AZStd::function messageCall); + void FlushMessageCalls(); + + AZStd::list> m_messageCalls; + AZStd::mutex m_messageCallsLock; + }; + + TraceMessageBusHandler::TraceMessageBusHandler() + { + m_events.resize(FN_MAX); + + SetEvent(&TraceMessageBusHandler::OnPreAssert, m_functionNames[FN_OnPreAssert]); + SetEvent(&TraceMessageBusHandler::OnPreError, m_functionNames[FN_OnPreError]); + SetEvent(&TraceMessageBusHandler::OnPreWarning, m_functionNames[FN_OnPreWarning]); + SetEvent(&TraceMessageBusHandler::OnAssert, m_functionNames[FN_OnAssert]); + SetEvent(&TraceMessageBusHandler::OnError, m_functionNames[FN_OnError]); + SetEvent(&TraceMessageBusHandler::OnWarning, m_functionNames[FN_OnWarning]); + SetEvent(&TraceMessageBusHandler::OnException, m_functionNames[FN_OnException]); + SetEvent(&TraceMessageBusHandler::OnPrintf, m_functionNames[FN_OnPrintf]); + SetEvent(&TraceMessageBusHandler::OnOutput, m_functionNames[FN_OnOutput]); + } + + int TraceMessageBusHandler::GetFunctionIndex(const char* functionName) const + { + for (int i = 0; i < FN_MAX; ++i) { - for (int i = 0; i < FN_MAX; ++i) + if (azstricmp(functionName, m_functionNames[i]) == 0) { - if (azstricmp(functionName, m_functionNames[i]) == 0) - { - return i; - } + return i; } - return -1; } + return -1; + } - void TraceMessageBusHandler::Disconnect() + void TraceMessageBusHandler::Disconnect() + { + AZ::Debug::TraceMessageBus::Handler::BusDisconnect(); + AZ::TickBus::Handler::BusDisconnect(); + } + + bool TraceMessageBusHandler::Connect(AZ::BehaviorValueParameter* id) + { + AZ::TickBus::Handler::BusConnect(); + return AZ::Internal::EBusConnector::Connect(this, id); + } + + bool TraceMessageBusHandler::IsConnected() + { + return AZ::Internal::EBusConnector::IsConnected(this); + } + + bool TraceMessageBusHandler::IsConnectedId(AZ::BehaviorValueParameter* id) + { + return AZ::Internal::EBusConnector::IsConnectedId(this, id); + } + + ////////////////////////////////////////////////////////////////////////// + // TraceMessageBusHandler Implementation + inline bool TraceMessageBusHandler::OnPreAssert(const char* fileName, int line, const char* func, const char* message) + { + QueueMessageCall( + [this, fileNameString = AZStd::string(fileName), line, funcString = AZStd::string(func), messageString = AZStd::string(message)]() { - AZ::Debug::TraceMessageBus::Handler::BusDisconnect(); - AZ::TickBus::Handler::BusDisconnect(); - } + Call(FN_OnPreAssert, fileNameString.c_str(), line, funcString.c_str(), messageString.c_str()); + }); + return false; + } - bool TraceMessageBusHandler::Connect(AZ::BehaviorValueParameter* id) + inline bool TraceMessageBusHandler::OnPreError(const char* window, const char* fileName, int line, const char* func, const char* message) + { + QueueMessageCall( + [this, windowString = AZStd::string(window), fileNameString = AZStd::string(fileName), line, funcString = AZStd::string(func), messageString = AZStd::string(message)]() { - AZ::TickBus::Handler::BusConnect(); - return AZ::Internal::EBusConnector::Connect(this, id); - } + Call(FN_OnPreError, windowString.c_str(), fileNameString.c_str(), line, funcString.c_str(), messageString.c_str()); + }); + return false; + } - bool TraceMessageBusHandler::IsConnected() + inline bool TraceMessageBusHandler::OnPreWarning(const char* window, const char* fileName, int line, const char* func, const char* message) + { + QueueMessageCall( + [this, windowString = AZStd::string(window), fileNameString = AZStd::string(fileName), line, funcString = AZStd::string(func), messageString = AZStd::string(message)]() { - return AZ::Internal::EBusConnector::IsConnected(this); - } + return Call(FN_OnPreWarning, windowString.c_str(), fileNameString.c_str(), line, funcString.c_str(), messageString.c_str()); + }); + return false; + } - bool TraceMessageBusHandler::IsConnectedId(AZ::BehaviorValueParameter* id) + inline bool TraceMessageBusHandler::OnAssert(const char* message) + { + QueueMessageCall( + [this, messageString = AZStd::string(message)]() { - return AZ::Internal::EBusConnector::IsConnectedId(this, id); - } + return Call(FN_OnAssert, messageString.c_str()); + }); + return false; + } - ////////////////////////////////////////////////////////////////////////// - // TraceMessageBusHandler Implementation - inline bool TraceMessageBusHandler::OnPreAssert(const char* fileName, int line, const char* func, const char* message) + inline bool TraceMessageBusHandler::OnError(const char* window, const char* message) + { + QueueMessageCall( + [this, windowString = AZStd::string(window), messageString = AZStd::string(message)]() { - QueueMessageCall( - [this, fileNameString = AZStd::string(fileName), line, funcString = AZStd::string(func), messageString = AZStd::string(message)]() - { - Call(FN_OnPreAssert, fileNameString.c_str(), line, funcString.c_str(), messageString.c_str()); - }); - return false; - } + return Call(FN_OnError, windowString.c_str(), messageString.c_str()); + }); + return false; + } - inline bool TraceMessageBusHandler::OnPreError(const char* window, const char* fileName, int line, const char* func, const char* message) + inline bool TraceMessageBusHandler::OnWarning(const char* window, const char* message) + { + QueueMessageCall( + [this, windowString = AZStd::string(window), messageString = AZStd::string(message)]() { - QueueMessageCall( - [this, windowString = AZStd::string(window), fileNameString = AZStd::string(fileName), line, funcString = AZStd::string(func), messageString = AZStd::string(message)]() - { - Call(FN_OnPreError, windowString.c_str(), fileNameString.c_str(), line, funcString.c_str(), messageString.c_str()); - }); - return false; - } + return Call(FN_OnWarning, windowString.c_str(), messageString.c_str()); + }); + return false; + } - inline bool TraceMessageBusHandler::OnPreWarning(const char* window, const char* fileName, int line, const char* func, const char* message) + inline bool TraceMessageBusHandler::OnException(const char* message) + { + QueueMessageCall( + [this, messageString = AZStd::string(message)]() { - QueueMessageCall( - [this, windowString = AZStd::string(window), fileNameString = AZStd::string(fileName), line, funcString = AZStd::string(func), messageString = AZStd::string(message)]() - { - return Call(FN_OnPreWarning, windowString.c_str(), fileNameString.c_str(), line, funcString.c_str(), messageString.c_str()); - }); - return false; - } + return Call(FN_OnException, messageString.c_str()); + }); + return false; + } - inline bool TraceMessageBusHandler::OnAssert(const char* message) + inline bool TraceMessageBusHandler::OnPrintf(const char* window, const char* message) + { + QueueMessageCall( + [this, windowString = AZStd::string(window), messageString = AZStd::string(message)]() { - QueueMessageCall( - [this, messageString = AZStd::string(message)]() - { - return Call(FN_OnAssert, messageString.c_str()); - }); - return false; - } + return Call(FN_OnPrintf, windowString.c_str(), messageString.c_str()); + }); + return false; + } - inline bool TraceMessageBusHandler::OnError(const char* window, const char* message) + inline bool TraceMessageBusHandler::OnOutput(const char* window, const char* message) + { + QueueMessageCall( + [this, windowString = AZStd::string(window), messageString = AZStd::string(message)]() { - QueueMessageCall( - [this, windowString = AZStd::string(window), messageString = AZStd::string(message)]() - { - return Call(FN_OnError, windowString.c_str(), messageString.c_str()); - }); - return false; - } + return Call(FN_OnOutput, windowString.c_str(), messageString.c_str()); + }); + return false; + } - inline bool TraceMessageBusHandler::OnWarning(const char* window, const char* message) - { - QueueMessageCall( - [this, windowString = AZStd::string(window), messageString = AZStd::string(message)]() - { - return Call(FN_OnWarning, windowString.c_str(), messageString.c_str()); - }); - return false; - } + void TraceMessageBusHandler::OnTick( + [[maybe_unused]] float deltaTime, + [[maybe_unused]] AZ::ScriptTimePoint time) + { + FlushMessageCalls(); + } - inline bool TraceMessageBusHandler::OnException(const char* message) - { - QueueMessageCall( - [this, messageString = AZStd::string(message)]() - { - return Call(FN_OnException, messageString.c_str()); - }); - return false; - } + int TraceMessageBusHandler::GetTickOrder() + { + return AZ::TICK_LAST; + } - inline bool TraceMessageBusHandler::OnPrintf(const char* window, const char* message) - { - QueueMessageCall( - [this, windowString = AZStd::string(window), messageString = AZStd::string(message)]() - { - return Call(FN_OnPrintf, windowString.c_str(), messageString.c_str()); - }); - return false; - } + void TraceMessageBusHandler::QueueMessageCall(AZStd::function messageCall) + { + AZStd::lock_guard lock(m_messageCallsLock); + m_messageCalls.emplace_back(messageCall); + } - inline bool TraceMessageBusHandler::OnOutput(const char* window, const char* message) - { - QueueMessageCall( - [this, windowString = AZStd::string(window), messageString = AZStd::string(message)]() - { - return Call(FN_OnOutput, windowString.c_str(), messageString.c_str()); - }); - return false; - } - - void TraceMessageBusHandler::OnTick( - [[maybe_unused]] float deltaTime, - [[maybe_unused]] AZ::ScriptTimePoint time) - { - FlushMessageCalls(); - } - - int TraceMessageBusHandler::GetTickOrder() - { - return AZ::TICK_LAST; - } - - void TraceMessageBusHandler::QueueMessageCall(AZStd::function messageCall) + void TraceMessageBusHandler::FlushMessageCalls() + { + AZStd::list> messageCalls; { AZStd::lock_guard lock(m_messageCallsLock); - m_messageCalls.push_back(messageCall); + m_messageCalls.swap(messageCalls); // Move calls to a new list to release the lock as soon as possible } - void TraceMessageBusHandler::FlushMessageCalls() + for (auto& messageCall : messageCalls) { - AZStd::list> messageCalls; - { - AZStd::lock_guard lock(m_messageCallsLock); - m_messageCalls.swap(messageCalls); // Move calls to a new list to release the lock as soon as possible - } - - for (auto& messageCall : messageCalls) - { - messageCall(); - } - } - - void TraceReflect(ReflectContext* context) - { - if (BehaviorContext* behaviorContext = azrtti_cast(context)) - { - behaviorContext->EBus("TraceMessageBus") - ->Attribute(AZ::Script::Attributes::Module, "debug") - ->Attribute(AZ::Script::Attributes::Scope, AZ::Script::Attributes::ScopeFlags::Automation) - ->Handler() - ; - } + messageCall(); } } -} + + void TraceReflect(ReflectContext* context) + { + if (BehaviorContext* behaviorContext = azrtti_cast(context)) + { + behaviorContext->EBus("TraceMessageBus") + ->Attribute(AZ::Script::Attributes::Module, "debug") + ->Attribute(AZ::Script::Attributes::Scope, AZ::Script::Attributes::ScopeFlags::Automation) + ->Handler() + ; + } + } +} // namespace AZ::Debug diff --git a/Code/Framework/AzCore/AzCore/Driller/Driller.cpp b/Code/Framework/AzCore/AzCore/Driller/Driller.cpp index 41abd7793e..7e986be81e 100644 --- a/Code/Framework/AzCore/AzCore/Driller/Driller.cpp +++ b/Code/Framework/AzCore/AzCore/Driller/Driller.cpp @@ -14,291 +14,288 @@ #include -namespace AZ +namespace AZ::Debug { - namespace Debug + class DrillerManagerImpl + : public DrillerManager { - class DrillerManagerImpl - : public DrillerManager + public: + AZ_CLASS_ALLOCATOR(DrillerManagerImpl, OSAllocator, 0); + + using SessionListType = forward_list::type; + SessionListType m_sessions; + using DrillerArrayType = vector::type; + DrillerArrayType m_drillers; + + ~DrillerManagerImpl() override; + + void Register(Driller* factory) override; + void Unregister(Driller* factory) override; + + void FrameUpdate() override; + + DrillerSession* Start(DrillerOutputStream& output, const DrillerListType& drillerList, int numFrames = -1) override; + void Stop(DrillerSession* session) override; + + int GetNumDrillers() const override { return static_cast(m_drillers.size()); } + Driller* GetDriller(int index) override { return m_drillers[index]; } + }; + + ////////////////////////////////////////////////////////////////////////// + // Driller + + //========================================================================= + // Register + // [3/17/2011] + //========================================================================= + AZ::u32 Driller::GetId() const + { + return AZ::Crc32(GetName()); + } + + ////////////////////////////////////////////////////////////////////////// + + ////////////////////////////////////////////////////////////////////////// + // Driller Manager + + //========================================================================= + // Register + // [3/17/2011] + //========================================================================= + DrillerManager* DrillerManager::Create(/*const Descriptor& desc*/) + { + const bool createAllocator = !AZ::AllocatorInstance::IsReady(); + if (createAllocator) { - public: - AZ_CLASS_ALLOCATOR(DrillerManagerImpl, OSAllocator, 0); - - typedef forward_list::type SessionListType; - SessionListType m_sessions; - typedef vector::type DrillerArrayType; - DrillerArrayType m_drillers; - - ~DrillerManagerImpl() override; - - void Register(Driller* factory) override; - void Unregister(Driller* factory) override; - - void FrameUpdate() override; - - DrillerSession* Start(DrillerOutputStream& output, const DrillerListType& drillerList, int numFrames = -1) override; - void Stop(DrillerSession* session) override; - - int GetNumDrillers() const override { return static_cast(m_drillers.size()); } - Driller* GetDriller(int index) override { return m_drillers[index]; } - }; - - ////////////////////////////////////////////////////////////////////////// - // Driller - - //========================================================================= - // Register - // [3/17/2011] - //========================================================================= - AZ::u32 Driller::GetId() const - { - return AZ::Crc32(GetName()); + AZ::AllocatorInstance::Create(); } - ////////////////////////////////////////////////////////////////////////// + DrillerManagerImpl* impl = aznew DrillerManagerImpl; + impl->m_ownsOSAllocator = createAllocator; + return impl; + } - ////////////////////////////////////////////////////////////////////////// - // Driller Manager - - //========================================================================= - // Register - // [3/17/2011] - //========================================================================= - DrillerManager* DrillerManager::Create(/*const Descriptor& desc*/) + //========================================================================= + // Register + // [3/17/2011] + //========================================================================= + void DrillerManager::Destroy(DrillerManager* manager) + { + const bool allocatorCreated = manager->m_ownsOSAllocator; + delete manager; + if (allocatorCreated) { - const bool createAllocator = !AZ::AllocatorInstance::IsReady(); - if (createAllocator) - { - AZ::AllocatorInstance::Create(); - } + AZ::AllocatorInstance::Destroy(); + } + } - DrillerManagerImpl* impl = aznew DrillerManagerImpl; - impl->m_ownsOSAllocator = createAllocator; - return impl; + ////////////////////////////////////////////////////////////////////////// + + ////////////////////////////////////////////////////////////////////////// + // DrillerManagerImpl + + //========================================================================= + // ~DrillerManagerImpl + // [3/17/2011] + //========================================================================= + DrillerManagerImpl::~DrillerManagerImpl() + { + while (!m_sessions.empty()) + { + Stop(&m_sessions.front()); } - //========================================================================= - // Register - // [3/17/2011] - //========================================================================= - void DrillerManager::Destroy(DrillerManager* manager) + while (!m_drillers.empty()) { - const bool allocatorCreated = manager->m_ownsOSAllocator; - delete manager; - if (allocatorCreated) - { - AZ::AllocatorInstance::Destroy(); - } + Driller* driller = m_drillers[0]; + Unregister(driller); + delete driller; } + } - ////////////////////////////////////////////////////////////////////////// - - ////////////////////////////////////////////////////////////////////////// - // DrillerManagerImpl - - //========================================================================= - // ~DrillerManagerImpl - // [3/17/2011] - //========================================================================= - DrillerManagerImpl::~DrillerManagerImpl() + //========================================================================= + // Register + // [3/17/2011] + //========================================================================= + void + DrillerManagerImpl::Register(Driller* driller) + { + AZ_Assert(driller, "You must provide a valid factory!"); + for (size_t i = 0; i < m_drillers.size(); ++i) { - while (!m_sessions.empty()) - { - Stop(&m_sessions.front()); - } - - while (!m_drillers.empty()) - { - Driller* driller = m_drillers[0]; - Unregister(driller); - delete driller; - } - } - - //========================================================================= - // Register - // [3/17/2011] - //========================================================================= - void - DrillerManagerImpl::Register(Driller* driller) - { - AZ_Assert(driller, "You must provide a valid factory!"); - for (size_t i = 0; i < m_drillers.size(); ++i) - { - if (m_drillers[i]->GetId() == driller->GetId()) - { - AZ_Error("Debug", false, "Driller with id %08x has already been registered! You can't have two factory instances for the same driller type", driller->GetId()); - return; - } - } - m_drillers.push_back(driller); - } - - //========================================================================= - // Unregister - // [3/17/2011] - //========================================================================= - void - DrillerManagerImpl::Unregister(Driller* driller) - { - AZ_Assert(driller, "You must provide a valid factory!"); - for (DrillerArrayType::iterator iter = m_drillers.begin(); iter != m_drillers.end(); ++iter) - { - if ((*iter)->GetId() == driller->GetId()) - { - m_drillers.erase(iter); - return; - } - } - - AZ_Error("Debug", false, "Failed to find driller factory with id %08x", driller->GetId()); - } - - //========================================================================= - // FrameUpdate - // [3/17/2011] - //========================================================================= - void - DrillerManagerImpl::FrameUpdate() - { - if (m_sessions.empty()) + if (m_drillers[i]->GetId() == driller->GetId()) { + AZ_Error("Debug", false, "Driller with id %08x has already been registered! You can't have two factory instances for the same driller type", driller->GetId()); return; } + } + m_drillers.push_back(driller); + } - AZStd::lock_guard lock(DrillerEBusMutex::GetMutex()); ///< Make sure no driller is writing to the stream - for (SessionListType::iterator sessionIter = m_sessions.begin(); sessionIter != m_sessions.end(); ) + //========================================================================= + // Unregister + // [3/17/2011] + //========================================================================= + void + DrillerManagerImpl::Unregister(Driller* driller) + { + AZ_Assert(driller, "You must provide a valid factory!"); + for (DrillerArrayType::iterator iter = m_drillers.begin(); iter != m_drillers.end(); ++iter) + { + if ((*iter)->GetId() == driller->GetId()) { - DrillerSession& s = *sessionIter; - - // tick the drillers directly if they care. - for (size_t i = 0; i < s.drillers.size(); ++i) - { - s.drillers[i]->Update(); - } - - s.output->EndTag(AZ_CRC("Frame", 0xb5f83ccd)); - - s.output->OnEndOfFrame(); - - s.curFrame++; - - if (s.numFrames != -1) - { - if (s.curFrame == s.numFrames) - { - Stop(&s); - continue; - } - } - - s.output->BeginTag(AZ_CRC("Frame", 0xb5f83ccd)); - s.output->Write(AZ_CRC("FrameNum", 0x85a1a919), s.curFrame); - - ++sessionIter; + m_drillers.erase(iter); + return; } } - //========================================================================= - // Start - // [3/17/2011] - //========================================================================= - DrillerSession* - DrillerManagerImpl::Start(DrillerOutputStream& output, const DrillerListType& drillerList, int numFrames) + AZ_Error("Debug", false, "Failed to find driller factory with id %08x", driller->GetId()); + } + + //========================================================================= + // FrameUpdate + // [3/17/2011] + //========================================================================= + void + DrillerManagerImpl::FrameUpdate() + { + if (m_sessions.empty()) { - if (drillerList.empty()) + return; + } + + AZStd::lock_guard lock(DrillerEBusMutex::GetMutex()); ///< Make sure no driller is writing to the stream + for (SessionListType::iterator sessionIter = m_sessions.begin(); sessionIter != m_sessions.end(); ) + { + DrillerSession& s = *sessionIter; + + // tick the drillers directly if they care. + for (size_t i = 0; i < s.drillers.size(); ++i) { - return nullptr; + s.drillers[i]->Update(); } - m_sessions.push_back(); - DrillerSession& s = m_sessions.back(); - s.curFrame = 0; - s.numFrames = numFrames; - s.output = &output; + s.output->EndTag(AZ_CRC("Frame", 0xb5f83ccd)); - s.output->WriteHeader(); // first write the header in the stream + s.output->OnEndOfFrame(); - s.output->BeginTag(AZ_CRC("StartData", 0xecf3f53f)); - s.output->Write(AZ_CRC("Platform", 0x3952d0cb), (unsigned int)g_currentPlatform); - for (DrillerListType::const_iterator iDriller = drillerList.begin(); iDriller != drillerList.end(); ++iDriller) + s.curFrame++; + + if (s.numFrames != -1) { - const DrillerInfo& di = *iDriller; - s.output->BeginTag(AZ_CRC("Driller", 0xa6e1fb73)); - s.output->Write(AZ_CRC("Name", 0x5e237e06), di.id); - for (int iParam = 0; iParam < (int)di.params.size(); ++iParam) + if (s.curFrame == s.numFrames) { - s.output->BeginTag(AZ_CRC("Param", 0xa4fa7c89)); - s.output->Write(AZ_CRC("Name", 0x5e237e06), di.params[iParam].name); - s.output->Write(AZ_CRC("Description", 0x6de44026), di.params[iParam].desc); - s.output->Write(AZ_CRC("Type", 0x8cde5729), di.params[iParam].type); - s.output->Write(AZ_CRC("Value", 0x1d775834), di.params[iParam].value); - s.output->EndTag(AZ_CRC("Param", 0xa4fa7c89)); + Stop(&s); + continue; } - s.output->EndTag(AZ_CRC("Driller", 0xa6e1fb73)); } - s.output->EndTag(AZ_CRC("StartData", 0xecf3f53f)); s.output->BeginTag(AZ_CRC("Frame", 0xb5f83ccd)); s.output->Write(AZ_CRC("FrameNum", 0x85a1a919), s.curFrame); - { - AZStd::lock_guard lock(DrillerEBusMutex::GetMutex()); ///< Make sure no driller is writing to the stream - for (DrillerListType::const_iterator iDriller = drillerList.begin(); iDriller != drillerList.end(); ++iDriller) - { - Driller* driller = nullptr; - const DrillerInfo& di = *iDriller; - for (size_t iDesc = 0; iDesc < m_drillers.size(); ++iDesc) - { - if (m_drillers[iDesc]->GetId() == di.id) - { - driller = m_drillers[iDesc]; - AZ_Assert(driller->m_output == nullptr, "Driller with id %08x is already have an output stream %p (currently we support only 1 at a time)", di.id, driller->m_output); - driller->m_output = &output; - driller->Start(di.params.data(), static_cast(di.params.size())); - s.drillers.push_back(driller); - break; - } - } - AZ_Warning("Driller", driller != nullptr, "We can't start a driller with id %d!", di.id); - } - } - return &s; + ++sessionIter; } + } - - //========================================================================= - // Stop - // [3/17/2011] - //========================================================================= - void - DrillerManagerImpl::Stop(DrillerSession* session) + //========================================================================= + // Start + // [3/17/2011] + //========================================================================= + DrillerSession* + DrillerManagerImpl::Start(DrillerOutputStream& output, const DrillerListType& drillerList, int numFrames) + { + if (drillerList.empty()) { - SessionListType::iterator iter; - for (iter = m_sessions.begin(); iter != m_sessions.end(); ++iter) + return nullptr; + } + + m_sessions.push_back(); + DrillerSession& s = m_sessions.back(); + s.curFrame = 0; + s.numFrames = numFrames; + s.output = &output; + + s.output->WriteHeader(); // first write the header in the stream + + s.output->BeginTag(AZ_CRC("StartData", 0xecf3f53f)); + s.output->Write(AZ_CRC("Platform", 0x3952d0cb), (unsigned int)g_currentPlatform); + for (DrillerListType::const_iterator iDriller = drillerList.begin(); iDriller != drillerList.end(); ++iDriller) + { + const DrillerInfo& di = *iDriller; + s.output->BeginTag(AZ_CRC("Driller", 0xa6e1fb73)); + s.output->Write(AZ_CRC("Name", 0x5e237e06), di.id); + for (int iParam = 0; iParam < (int)di.params.size(); ++iParam) { - if (&*iter == session) - { - break; - } + s.output->BeginTag(AZ_CRC("Param", 0xa4fa7c89)); + s.output->Write(AZ_CRC("Name", 0x5e237e06), di.params[iParam].name); + s.output->Write(AZ_CRC("Description", 0x6de44026), di.params[iParam].desc); + s.output->Write(AZ_CRC("Type", 0x8cde5729), di.params[iParam].type); + s.output->Write(AZ_CRC("Value", 0x1d775834), di.params[iParam].value); + s.output->EndTag(AZ_CRC("Param", 0xa4fa7c89)); } + s.output->EndTag(AZ_CRC("Driller", 0xa6e1fb73)); + } + s.output->EndTag(AZ_CRC("StartData", 0xecf3f53f)); - AZ_Assert(iter != m_sessions.end(), "We did not find session ID 0x%08x in the list!", session); - if (iter != m_sessions.end()) + s.output->BeginTag(AZ_CRC("Frame", 0xb5f83ccd)); + s.output->Write(AZ_CRC("FrameNum", 0x85a1a919), s.curFrame); + + { + AZStd::lock_guard lock(DrillerEBusMutex::GetMutex()); ///< Make sure no driller is writing to the stream + for (DrillerListType::const_iterator iDriller = drillerList.begin(); iDriller != drillerList.end(); ++iDriller) { - DrillerSession& s = *session; - + Driller* driller = nullptr; + const DrillerInfo& di = *iDriller; + for (size_t iDesc = 0; iDesc < m_drillers.size(); ++iDesc) { - AZStd::lock_guard lock(DrillerEBusMutex::GetMutex()); - for (size_t i = 0; i < s.drillers.size(); ++i) + if (m_drillers[iDesc]->GetId() == di.id) { - s.drillers[i]->Stop(); - s.drillers[i]->m_output = nullptr; + driller = m_drillers[iDesc]; + AZ_Assert(driller->m_output == nullptr, "Driller with id %08x is already have an output stream %p (currently we support only 1 at a time)", di.id, driller->m_output); + driller->m_output = &output; + driller->Start(di.params.data(), static_cast(di.params.size())); + s.drillers.push_back(driller); + break; } } - s.output->EndTag(AZ_CRC("Frame", 0xb5f83ccd)); - m_sessions.erase(iter); + AZ_Warning("Driller", driller != nullptr, "We can't start a driller with id %d!", di.id); } } - } // namespace Debug -} // namespace AZ + return &s; + } + + + //========================================================================= + // Stop + // [3/17/2011] + //========================================================================= + void + DrillerManagerImpl::Stop(DrillerSession* session) + { + SessionListType::iterator iter; + for (iter = m_sessions.begin(); iter != m_sessions.end(); ++iter) + { + if (&*iter == session) + { + break; + } + } + + AZ_Assert(iter != m_sessions.end(), "We did not find session ID 0x%08x in the list!", session); + if (iter != m_sessions.end()) + { + DrillerSession& s = *session; + + { + AZStd::lock_guard lock(DrillerEBusMutex::GetMutex()); + for (size_t i = 0; i < s.drillers.size(); ++i) + { + s.drillers[i]->Stop(); + s.drillers[i]->m_output = nullptr; + } + } + s.output->EndTag(AZ_CRC("Frame", 0xb5f83ccd)); + m_sessions.erase(iter); + } + } +} // namespace AZ::Debug diff --git a/Code/Framework/AzCore/AzCore/Driller/DrillerBus.cpp b/Code/Framework/AzCore/AzCore/Driller/DrillerBus.cpp index 163db8a68b..a46736aa87 100644 --- a/Code/Framework/AzCore/AzCore/Driller/DrillerBus.cpp +++ b/Code/Framework/AzCore/AzCore/Driller/DrillerBus.cpp @@ -12,57 +12,54 @@ #include #include -namespace AZ +namespace AZ::Debug { - namespace Debug + ////////////////////////////////////////////////////////////////////////// + // Globals + // We need to synchronize all driller evens, so we have proper order, and access to the data + // We use a global mutex which should be used for all driller operations. + // The mutex is held in an environment variable so it works across DLLs. + EnvironmentVariable s_drillerGlobalMutex; + ////////////////////////////////////////////////////////////////////////// + + + //========================================================================= + // lock + // [4/11/2011] + //========================================================================= + void DrillerEBusMutex::lock() { - ////////////////////////////////////////////////////////////////////////// - // Globals - // We need to synchronize all driller evens, so we have proper order, and access to the data - // We use a global mutex which should be used for all driller operations. - // The mutex is held in an environment variable so it works across DLLs. - EnvironmentVariable s_drillerGlobalMutex; - ////////////////////////////////////////////////////////////////////////// + GetMutex().lock(); + } - - //========================================================================= - // lock - // [4/11/2011] - //========================================================================= - void DrillerEBusMutex::lock() - { - GetMutex().lock(); - } + //========================================================================= + // try_lock + // [4/11/2011] + //========================================================================= + bool DrillerEBusMutex::try_lock() + { + return GetMutex().try_lock(); + } - //========================================================================= - // try_lock - // [4/11/2011] - //========================================================================= - bool DrillerEBusMutex::try_lock() - { - return GetMutex().try_lock(); - } + //========================================================================= + // unlock + // [4/11/2011] + //========================================================================= + void DrillerEBusMutex::unlock() + { + GetMutex().unlock(); + } - //========================================================================= - // unlock - // [4/11/2011] - //========================================================================= - void DrillerEBusMutex::unlock() + //========================================================================= + // unlock + // [4/11/2011] + //========================================================================= + AZStd::recursive_mutex& DrillerEBusMutex::GetMutex() + { + if (!s_drillerGlobalMutex) { - GetMutex().unlock(); + s_drillerGlobalMutex = Environment::CreateVariable(AZ_FUNCTION_SIGNATURE); } - - //========================================================================= - // unlock - // [4/11/2011] - //========================================================================= - AZStd::recursive_mutex& DrillerEBusMutex::GetMutex() - { - if (!s_drillerGlobalMutex) - { - s_drillerGlobalMutex = Environment::CreateVariable(AZ_FUNCTION_SIGNATURE); - } - return *s_drillerGlobalMutex; - } - } // namespace Debug -} // namespace AZ + return *s_drillerGlobalMutex; + } +} // namespace AZ::Debug diff --git a/Code/Framework/AzCore/AzCore/Driller/Stream.cpp b/Code/Framework/AzCore/AzCore/Driller/Stream.cpp index 14e983b09c..c86755149c 100644 --- a/Code/Framework/AzCore/AzCore/Driller/Stream.cpp +++ b/Code/Framework/AzCore/AzCore/Driller/Stream.cpp @@ -24,873 +24,870 @@ # include #endif // AZ_FILE_STREAM_COMPRESSION -namespace AZ +namespace AZ::Debug { - namespace Debug + ////////////////////////////////////////////////////////////////////////// + ////////////////////////////////////////////////////////////////////////// + // Driller output stream + ////////////////////////////////////////////////////////////////////////// + ////////////////////////////////////////////////////////////////////////// + void DrillerOutputStream::Write(u32 name, const AZ::Vector3& v) { - ////////////////////////////////////////////////////////////////////////// - ////////////////////////////////////////////////////////////////////////// - // Driller output stream - ////////////////////////////////////////////////////////////////////////// - ////////////////////////////////////////////////////////////////////////// - void DrillerOutputStream::Write(u32 name, const AZ::Vector3& v) - { - float data[4]; - unsigned int dataSize = 3 * sizeof(float); - v.StoreToFloat4(data); - StreamEntry de; - de.name = name; - de.sizeAndFlags = dataSize; - WriteBinary(de); - WriteBinary(data, dataSize); - } - void DrillerOutputStream::Write(u32 name, const AZ::Vector4& v) - { - float data[4]; - unsigned int dataSize = 4 * sizeof(float); - v.StoreToFloat4(data); - StreamEntry de; - de.name = name; - de.sizeAndFlags = dataSize; - WriteBinary(de); - WriteBinary(data, dataSize); - } - void DrillerOutputStream::Write(u32 name, const AZ::Aabb& aabb) - { - float data[7]; - unsigned int dataSize = 6 * sizeof(float); - aabb.GetMin().StoreToFloat4(data); - aabb.GetMax().StoreToFloat4(&data[3]); - StreamEntry de; - de.name = name; - de.sizeAndFlags = dataSize; - WriteBinary(de); - WriteBinary(data, dataSize); - } - void DrillerOutputStream::Write(u32 name, const AZ::Obb& obb) - { - float data[10]; - unsigned int dataSize = 10 * sizeof(float); // position (Vector3), rotation (Quaternion) and halfLengths (Vector3) - obb.GetPosition().StoreToFloat3(data); - obb.GetRotation().StoreToFloat4(&data[3]); - obb.GetHalfLengths().StoreToFloat3(&data[7]); - StreamEntry de; - de.name = name; - de.sizeAndFlags = dataSize; - WriteBinary(de); - WriteBinary(data, dataSize); - } - void DrillerOutputStream::Write(u32 name, const AZ::Transform& tm) - { - float data[12]; - unsigned int dataSize = 12 * sizeof(float); - const Matrix3x4 matrix3x4 = Matrix3x4::CreateFromTransform(tm); - matrix3x4.StoreToRowMajorFloat12(data); - StreamEntry de; - de.name = name; - de.sizeAndFlags = dataSize; - WriteBinary(de); - WriteBinary(data, dataSize); - } - void DrillerOutputStream::Write(u32 name, const AZ::Matrix3x3& tm) - { - float data[9]; - unsigned int dataSize = 9 * sizeof(float); - tm.StoreToRowMajorFloat9(data); - StreamEntry de; - de.name = name; - de.sizeAndFlags = dataSize; - WriteBinary(de); - WriteBinary(data, dataSize); - } - void DrillerOutputStream::Write(u32 name, const AZ::Matrix4x4& tm) - { - float data[16]; - unsigned int dataSize = 16 * sizeof(float); - tm.StoreToRowMajorFloat16(data); - StreamEntry de; - de.name = name; - de.sizeAndFlags = dataSize; - WriteBinary(de); - WriteBinary(data, dataSize); - } - void DrillerOutputStream::Write(u32 name, const AZ::Quaternion& tm) - { - float data[4]; - unsigned int dataSize = 4 * sizeof(float); - tm.StoreToFloat4(data); - StreamEntry de; - de.name = name; - de.sizeAndFlags = dataSize; - WriteBinary(de); - WriteBinary(data, dataSize); - } - void DrillerOutputStream::Write(u32 name, const AZ::Plane& plane) - { - Write(name, plane.GetPlaneEquationCoefficients()); - } - void DrillerOutputStream::WriteHeader() - { - StreamHeader sh; // StreamHeader should be endianess independent. - WriteBinary(&sh, sizeof(sh)); - } + float data[4]; + unsigned int dataSize = 3 * sizeof(float); + v.StoreToFloat4(data); + StreamEntry de; + de.name = name; + de.sizeAndFlags = dataSize; + WriteBinary(de); + WriteBinary(data, dataSize); + } + void DrillerOutputStream::Write(u32 name, const AZ::Vector4& v) + { + float data[4]; + unsigned int dataSize = 4 * sizeof(float); + v.StoreToFloat4(data); + StreamEntry de; + de.name = name; + de.sizeAndFlags = dataSize; + WriteBinary(de); + WriteBinary(data, dataSize); + } + void DrillerOutputStream::Write(u32 name, const AZ::Aabb& aabb) + { + float data[7]; + unsigned int dataSize = 6 * sizeof(float); + aabb.GetMin().StoreToFloat4(data); + aabb.GetMax().StoreToFloat4(&data[3]); + StreamEntry de; + de.name = name; + de.sizeAndFlags = dataSize; + WriteBinary(de); + WriteBinary(data, dataSize); + } + void DrillerOutputStream::Write(u32 name, const AZ::Obb& obb) + { + float data[10]; + unsigned int dataSize = 10 * sizeof(float); // position (Vector3), rotation (Quaternion) and halfLengths (Vector3) + obb.GetPosition().StoreToFloat3(data); + obb.GetRotation().StoreToFloat4(&data[3]); + obb.GetHalfLengths().StoreToFloat3(&data[7]); + StreamEntry de; + de.name = name; + de.sizeAndFlags = dataSize; + WriteBinary(de); + WriteBinary(data, dataSize); + } + void DrillerOutputStream::Write(u32 name, const AZ::Transform& tm) + { + float data[12]; + unsigned int dataSize = 12 * sizeof(float); + const Matrix3x4 matrix3x4 = Matrix3x4::CreateFromTransform(tm); + matrix3x4.StoreToRowMajorFloat12(data); + StreamEntry de; + de.name = name; + de.sizeAndFlags = dataSize; + WriteBinary(de); + WriteBinary(data, dataSize); + } + void DrillerOutputStream::Write(u32 name, const AZ::Matrix3x3& tm) + { + float data[9]; + unsigned int dataSize = 9 * sizeof(float); + tm.StoreToRowMajorFloat9(data); + StreamEntry de; + de.name = name; + de.sizeAndFlags = dataSize; + WriteBinary(de); + WriteBinary(data, dataSize); + } + void DrillerOutputStream::Write(u32 name, const AZ::Matrix4x4& tm) + { + float data[16]; + unsigned int dataSize = 16 * sizeof(float); + tm.StoreToRowMajorFloat16(data); + StreamEntry de; + de.name = name; + de.sizeAndFlags = dataSize; + WriteBinary(de); + WriteBinary(data, dataSize); + } + void DrillerOutputStream::Write(u32 name, const AZ::Quaternion& tm) + { + float data[4]; + unsigned int dataSize = 4 * sizeof(float); + tm.StoreToFloat4(data); + StreamEntry de; + de.name = name; + de.sizeAndFlags = dataSize; + WriteBinary(de); + WriteBinary(data, dataSize); + } + void DrillerOutputStream::Write(u32 name, const AZ::Plane& plane) + { + Write(name, plane.GetPlaneEquationCoefficients()); + } + void DrillerOutputStream::WriteHeader() + { + StreamHeader sh; // StreamHeader should be endianess independent. + WriteBinary(&sh, sizeof(sh)); + } - void DrillerOutputStream::WriteTimeUTC(u32 name) - { - AZStd::sys_time_t now = AZStd::GetTimeUTCMilliSecond(); - Write(name, now); - } + void DrillerOutputStream::WriteTimeUTC(u32 name) + { + AZStd::sys_time_t now = AZStd::GetTimeUTCMilliSecond(); + Write(name, now); + } - void DrillerOutputStream::WriteTimeMicrosecond(u32 name) - { - AZStd::sys_time_t now = AZStd::GetTimeNowMicroSecond(); - Write(name, now); - } + void DrillerOutputStream::WriteTimeMicrosecond(u32 name) + { + AZStd::sys_time_t now = AZStd::GetTimeNowMicroSecond(); + Write(name, now); + } - ////////////////////////////////////////////////////////////////////////// - ////////////////////////////////////////////////////////////////////////// - // Driller Input Stream - ////////////////////////////////////////////////////////////////////////// - ////////////////////////////////////////////////////////////////////////// - bool DrillerInputStream::ReadHeader() + ////////////////////////////////////////////////////////////////////////// + ////////////////////////////////////////////////////////////////////////// + // Driller Input Stream + ////////////////////////////////////////////////////////////////////////// + ////////////////////////////////////////////////////////////////////////// + bool DrillerInputStream::ReadHeader() + { + DrillerOutputStream::StreamHeader sh; // StreamHeader should be endianess independent. + unsigned int numRead = ReadBinary(&sh, sizeof(sh)); + (void)numRead; + AZ_Error("IO", numRead == sizeof(sh), "We should have atleast %d bytes in the stream to read the header!", sizeof(sh)); + if (numRead != sizeof(sh)) { - DrillerOutputStream::StreamHeader sh; // StreamHeader should be endianess independent. - unsigned int numRead = ReadBinary(&sh, sizeof(sh)); - (void)numRead; - AZ_Error("IO", numRead == sizeof(sh), "We should have atleast %d bytes in the stream to read the header!", sizeof(sh)); - if (numRead != sizeof(sh)) - { - return false; - } - m_isEndianSwap = AZ::IsBigEndian(static_cast(sh.platform)) != AZ::IsBigEndian(AZ::g_currentPlatform); - return true; - } - - ////////////////////////////////////////////////////////////////////////// - ////////////////////////////////////////////////////////////////////////// - // Driller file stream - ////////////////////////////////////////////////////////////////////////// - ////////////////////////////////////////////////////////////////////////// - //========================================================================= - // DrillerOutputFileStream::DrillerOutputFileStream - // [3/23/2011] - //========================================================================= - DrillerOutputFileStream::DrillerOutputFileStream() - { -#if defined(AZ_FILE_STREAM_COMPRESSION) - m_zlib = azcreate(ZLib, (&AllocatorInstance::GetAllocator()), OSAllocator); - m_zlib->StartCompressor(2); -#endif - } - - //========================================================================= - // DrillerOutputFileStream::~DrillerOutputFileStream - // [3/23/2011] - //========================================================================= - DrillerOutputFileStream::~DrillerOutputFileStream() - { -#if defined(AZ_FILE_STREAM_COMPRESSION) - azdestroy(m_zlib, OSAllocator); -#endif - } - - //========================================================================= - // DrillerOutputFileStream::Open - // [3/23/2011] - //========================================================================= - bool DrillerOutputFileStream::Open(const char* fileName, int mode, int platformFlags) - { - if (IO::SystemFile::Open(fileName, mode, platformFlags)) - { - m_dataBuffer.reserve(100 * 1024); -#if defined(AZ_FILE_STREAM_COMPRESSION) - // // Enable optional: encode the file in the same format as the streamer so they are interchangeable - // IO::CompressorHeader ch; - // ch.SetAZCS(); - // ch.m_compressorId = IO::CompressorZLib::TypeId(); - // ch.m_uncompressedSize = 0; // will be updated later - // AZStd::endian_swap(ch.m_compressorId); - // AZStd::endian_swap(ch.m_uncompressedSize); - // IO::SystemFile::Write(&ch,sizeof(ch)); - // IO::CompressorZLibHeader zlibHdr; - // zlibHdr.m_numSeekPoints = 0; - // IO::SystemFile::Write(&zlibHdr,sizeof(zlibHdr)); -#endif - return true; - } return false; } + m_isEndianSwap = AZ::IsBigEndian(static_cast(sh.platform)) != AZ::IsBigEndian(AZ::g_currentPlatform); + return true; + } - //========================================================================= - // DrillerOutputFileStream::Close - // [3/23/2011] - //========================================================================= - void DrillerOutputFileStream::Close() + ////////////////////////////////////////////////////////////////////////// + ////////////////////////////////////////////////////////////////////////// + // Driller file stream + ////////////////////////////////////////////////////////////////////////// + ////////////////////////////////////////////////////////////////////////// + //========================================================================= + // DrillerOutputFileStream::DrillerOutputFileStream + // [3/23/2011] + //========================================================================= + DrillerOutputFileStream::DrillerOutputFileStream() + { +#if defined(AZ_FILE_STREAM_COMPRESSION) + m_zlib = azcreate(ZLib, (&AllocatorInstance::GetAllocator()), OSAllocator); + m_zlib->StartCompressor(2); +#endif + } + + //========================================================================= + // DrillerOutputFileStream::~DrillerOutputFileStream + // [3/23/2011] + //========================================================================= + DrillerOutputFileStream::~DrillerOutputFileStream() + { +#if defined(AZ_FILE_STREAM_COMPRESSION) + azdestroy(m_zlib, OSAllocator); +#endif + } + + //========================================================================= + // DrillerOutputFileStream::Open + // [3/23/2011] + //========================================================================= + bool DrillerOutputFileStream::Open(const char* fileName, int mode, int platformFlags) + { + if (IO::SystemFile::Open(fileName, mode, platformFlags)) { - unsigned int dataSizeInBuffer = static_cast(m_dataBuffer.size()); + m_dataBuffer.reserve(100 * 1024); +#if defined(AZ_FILE_STREAM_COMPRESSION) + // // Enable optional: encode the file in the same format as the streamer so they are interchangeable + // IO::CompressorHeader ch; + // ch.SetAZCS(); + // ch.m_compressorId = IO::CompressorZLib::TypeId(); + // ch.m_uncompressedSize = 0; // will be updated later + // AZStd::endian_swap(ch.m_compressorId); + // AZStd::endian_swap(ch.m_uncompressedSize); + // IO::SystemFile::Write(&ch,sizeof(ch)); + // IO::CompressorZLibHeader zlibHdr; + // zlibHdr.m_numSeekPoints = 0; + // IO::SystemFile::Write(&zlibHdr,sizeof(zlibHdr)); +#endif + return true; + } + return false; + } + + //========================================================================= + // DrillerOutputFileStream::Close + // [3/23/2011] + //========================================================================= + void DrillerOutputFileStream::Close() + { + unsigned int dataSizeInBuffer = static_cast(m_dataBuffer.size()); + { +#if defined(AZ_FILE_STREAM_COMPRESSION) + unsigned int minCompressBufferSize = m_zlib->GetMinCompressedBufferSize(dataSizeInBuffer); + if (m_compressionBuffer.size() < minCompressBufferSize) // grow compression buffer if needed + { + m_compressionBuffer.clear(); + m_compressionBuffer.resize(minCompressBufferSize); + } + unsigned int compressedSize; + do + { + compressedSize = m_zlib->Compress(m_dataBuffer.data(), dataSizeInBuffer, m_compressionBuffer.data(), (unsigned)m_compressionBuffer.size(), ZLib::FT_FINISH); + if (compressedSize) + { + IO::SystemFile::Write(m_compressionBuffer.data(), compressedSize); + } + } while (compressedSize > 0); + m_zlib->ResetCompressor(); +#else + if (dataSizeInBuffer) + { + IO::SystemFile::Write(m_dataBuffer.data(), m_dataBuffer.size()); + } +#endif + m_dataBuffer.clear(); + } + IO::SystemFile::Close(); + } + //========================================================================= + // DrillerOutputFileStream::WriteBinary + // [3/23/2011] + //========================================================================= + void DrillerOutputFileStream::WriteBinary(const void* data, unsigned int dataSize) + { + size_t dataSizeInBuffer = m_dataBuffer.size(); + if (dataSizeInBuffer + dataSize > m_dataBuffer.capacity()) + { + if (dataSizeInBuffer > 0) { #if defined(AZ_FILE_STREAM_COMPRESSION) - unsigned int minCompressBufferSize = m_zlib->GetMinCompressedBufferSize(dataSizeInBuffer); + // we need to flush the data + unsigned int dataToCompress = static_cast(dataSizeInBuffer); + unsigned int minCompressBufferSize = m_zlib->GetMinCompressedBufferSize(dataToCompress); if (m_compressionBuffer.size() < minCompressBufferSize) // grow compression buffer if needed { m_compressionBuffer.clear(); m_compressionBuffer.resize(minCompressBufferSize); } - unsigned int compressedSize; - do + while (dataToCompress > 0) { - compressedSize = m_zlib->Compress(m_dataBuffer.data(), dataSizeInBuffer, m_compressionBuffer.data(), (unsigned)m_compressionBuffer.size(), ZLib::FT_FINISH); + unsigned int compressedSize = m_zlib->Compress(m_dataBuffer.data(), dataToCompress, m_compressionBuffer.data(), (unsigned)m_compressionBuffer.size()); if (compressedSize) { IO::SystemFile::Write(m_compressionBuffer.data(), compressedSize); } - } while (compressedSize > 0); - m_zlib->ResetCompressor(); -#else - if (dataSizeInBuffer) - { - IO::SystemFile::Write(m_dataBuffer.data(), m_dataBuffer.size()); } +#else + IO::SystemFile::Write(m_dataBuffer.data(), m_dataBuffer.size()); #endif m_dataBuffer.clear(); } - IO::SystemFile::Close(); } - //========================================================================= - // DrillerOutputFileStream::WriteBinary - // [3/23/2011] - //========================================================================= - void DrillerOutputFileStream::WriteBinary(const void* data, unsigned int dataSize) - { - size_t dataSizeInBuffer = m_dataBuffer.size(); - if (dataSizeInBuffer + dataSize > m_dataBuffer.capacity()) - { - if (dataSizeInBuffer > 0) - { + m_dataBuffer.insert(m_dataBuffer.end(), reinterpret_cast(data), reinterpret_cast(data) + dataSize); + } + + ////////////////////////////////////////////////////////////////////////// + ////////////////////////////////////////////////////////////////////////// + // Driller file input stream + ////////////////////////////////////////////////////////////////////////// + ////////////////////////////////////////////////////////////////////////// + + //========================================================================= + // DrillerInputFileStream::DrillerInputFileStream + // [3/23/2011] + //========================================================================= + DrillerInputFileStream::DrillerInputFileStream() + { #if defined(AZ_FILE_STREAM_COMPRESSION) - // we need to flush the data - unsigned int dataToCompress = static_cast(dataSizeInBuffer); - unsigned int minCompressBufferSize = m_zlib->GetMinCompressedBufferSize(dataToCompress); - if (m_compressionBuffer.size() < minCompressBufferSize) // grow compression buffer if needed - { - m_compressionBuffer.clear(); - m_compressionBuffer.resize(minCompressBufferSize); - } - while (dataToCompress > 0) - { - unsigned int compressedSize = m_zlib->Compress(m_dataBuffer.data(), dataToCompress, m_compressionBuffer.data(), (unsigned)m_compressionBuffer.size()); - if (compressedSize) - { - IO::SystemFile::Write(m_compressionBuffer.data(), compressedSize); - } - } + m_zlib = azcreate(ZLib, (&AllocatorInstance::GetAllocator()), OSAllocator); + m_zlib->StartDecompressor(); +#endif + } + + //========================================================================= + // DrillerInputFileStream::DrillerInputFileStream + // [3/23/2011] + //========================================================================= + DrillerInputFileStream::~DrillerInputFileStream() + { +#if defined(AZ_FILE_STREAM_COMPRESSION) + azdestroy(m_zlib, OSAllocator); +#endif + } + + //========================================================================= + // DrillerInputFileStream::Open + // [3/23/2011] + //========================================================================= + bool DrillerInputFileStream::Open(const char* fileName, int mode, int platformFlags) + { + if (IO::SystemFile::Open(fileName, mode, platformFlags)) + { + DrillerOutputStream::StreamHeader sh; +#if defined(AZ_FILE_STREAM_COMPRESSION) + // TODO: optional encode the file in the same format as the streamer so they are interchangeable +#endif + // first read the header of the stream file. + return ReadHeader(); + } + return false; + } + //========================================================================= + // DrillerInputFileStream::ReadBinary + // [3/23/2011] + //========================================================================= + unsigned int DrillerInputFileStream::ReadBinary(void* data, unsigned int maxDataSize) + { + // make sure the compressed buffer if full enough... + size_t dataToLoad = maxDataSize * 2; + m_compressedData.reserve(dataToLoad); + while (m_compressedData.size() < dataToLoad) + { + unsigned char buffer[10 * 1024]; + IO::SystemFile::SizeType bytesRead = Read(AZ_ARRAY_SIZE(buffer), buffer); + if (bytesRead > 0) + { + m_compressedData.insert(m_compressedData.end(), (unsigned char*)buffer, buffer + bytesRead); + } + if (bytesRead < AZ_ARRAY_SIZE(buffer)) + { + break; + } + } +#if defined(AZ_FILE_STREAM_COMPRESSION) + unsigned int dataSize = maxDataSize; + unsigned int bytesProcessed = m_zlib->Decompress(m_compressedData.data(), (unsigned)m_compressedData.size(), data, dataSize); + unsigned int readSize = maxDataSize - dataSize; // Zlib::Decompress decrements the dataSize parameter by the amount uncompressed #else - IO::SystemFile::Write(m_dataBuffer.data(), m_dataBuffer.size()); + unsigned int bytesProcessed = AZStd::GetMin((unsigned int)m_compressedData.size(), maxDataSize); + unsigned int readSize = bytesProcessed; + memcpy(data, m_compressedData.data(), readSize); #endif - m_dataBuffer.clear(); - } - } - m_dataBuffer.insert(m_dataBuffer.end(), reinterpret_cast(data), reinterpret_cast(data) + dataSize); - } + m_compressedData.erase(m_compressedData.begin(), m_compressedData.begin() + bytesProcessed); + return readSize; + } - ////////////////////////////////////////////////////////////////////////// - ////////////////////////////////////////////////////////////////////////// - // Driller file input stream - ////////////////////////////////////////////////////////////////////////// - ////////////////////////////////////////////////////////////////////////// - - //========================================================================= - // DrillerInputFileStream::DrillerInputFileStream - // [3/23/2011] - //========================================================================= - DrillerInputFileStream::DrillerInputFileStream() - { + //========================================================================= + // DrillerInputFileStream::Close + // [3/23/2011] + //========================================================================= + void DrillerInputFileStream::Close() + { #if defined(AZ_FILE_STREAM_COMPRESSION) - m_zlib = azcreate(ZLib, (&AllocatorInstance::GetAllocator()), OSAllocator); - m_zlib->StartDecompressor(); -#endif - } - - //========================================================================= - // DrillerInputFileStream::DrillerInputFileStream - // [3/23/2011] - //========================================================================= - DrillerInputFileStream::~DrillerInputFileStream() + if (m_zlib) { -#if defined(AZ_FILE_STREAM_COMPRESSION) - azdestroy(m_zlib, OSAllocator); -#endif + m_zlib->ResetDecompressor(); } +#endif // AZ_FILE_STREAM_COMPRESSION + AZ::IO::SystemFile::Close(); + } - //========================================================================= - // DrillerInputFileStream::Open - // [3/23/2011] - //========================================================================= - bool DrillerInputFileStream::Open(const char* fileName, int mode, int platformFlags) + ////////////////////////////////////////////////////////////////////////// + ////////////////////////////////////////////////////////////////////////// + // DrillerSAXParser + ////////////////////////////////////////////////////////////////////////// + ////////////////////////////////////////////////////////////////////////// + //========================================================================= + // DrillerSAXParser + // [3/23/2011] + //========================================================================= + DrillerSAXParser::DrillerSAXParser(const TagCallbackType& tcb, const DataCallbackType& dcb) + : m_tagCallback(tcb) + , m_dataCallback(dcb) + { + } + + //========================================================================= + // ProcessStream + // [3/23/2011] + //========================================================================= + void + DrillerSAXParser::ProcessStream(DrillerInputStream& stream) + { + static const int processChunkSize = 15 * 1024; + char buffer[processChunkSize]; + unsigned int dataSize; + bool isEndianSwap = stream.IsEndianSwap(); + while ((dataSize = stream.ReadBinary(buffer, processChunkSize)) > 0) { - if (IO::SystemFile::Open(fileName, mode, platformFlags)) + char* dataStart = buffer; + char* dataEnd = dataStart + dataSize; + bool dataInBuffer = false; + if (!m_buffer.empty()) { - DrillerOutputStream::StreamHeader sh; -#if defined(AZ_FILE_STREAM_COMPRESSION) - // TODO: optional encode the file in the same format as the streamer so they are interchangeable -#endif - // first read the header of the stream file. - return ReadHeader(); + m_buffer.insert(m_buffer.end(), dataStart, dataEnd); + dataStart = m_buffer.data(); + dataEnd = dataStart + m_buffer.size(); + dataInBuffer = true; } - return false; - } - //========================================================================= - // DrillerInputFileStream::ReadBinary - // [3/23/2011] - //========================================================================= - unsigned int DrillerInputFileStream::ReadBinary(void* data, unsigned int maxDataSize) - { - // make sure the compressed buffer if full enough... - size_t dataToLoad = maxDataSize * 2; - m_compressedData.reserve(dataToLoad); - while (m_compressedData.size() < dataToLoad) + const int entrySize = sizeof(DrillerOutputStream::StreamEntry); + while (dataStart != dataEnd) { - unsigned char buffer[10 * 1024]; - IO::SystemFile::SizeType bytesRead = Read(AZ_ARRAY_SIZE(buffer), buffer); - if (bytesRead > 0) - { - m_compressedData.insert(m_compressedData.end(), (unsigned char*)buffer, buffer + bytesRead); - } - if (bytesRead < AZ_ARRAY_SIZE(buffer)) + if ((dataEnd - dataStart) < entrySize) // we need at least one entry to proceed { + // not enough data to process, buffer it. + if (!dataInBuffer) + { + m_buffer.insert(m_buffer.end(), dataStart, dataEnd); + } break; } - } -#if defined(AZ_FILE_STREAM_COMPRESSION) - unsigned int dataSize = maxDataSize; - unsigned int bytesProcessed = m_zlib->Decompress(m_compressedData.data(), (unsigned)m_compressedData.size(), data, dataSize); - unsigned int readSize = maxDataSize - dataSize; // Zlib::Decompress decrements the dataSize parameter by the amount uncompressed -#else - unsigned int bytesProcessed = AZStd::GetMin((unsigned int)m_compressedData.size(), maxDataSize); - unsigned int readSize = bytesProcessed; - memcpy(data, m_compressedData.data(), readSize); -#endif - m_compressedData.erase(m_compressedData.begin(), m_compressedData.begin() + bytesProcessed); - return readSize; - } - //========================================================================= - // DrillerInputFileStream::Close - // [3/23/2011] - //========================================================================= - void DrillerInputFileStream::Close() - { -#if defined(AZ_FILE_STREAM_COMPRESSION) - if (m_zlib) - { - m_zlib->ResetDecompressor(); - } -#endif // AZ_FILE_STREAM_COMPRESSION - AZ::IO::SystemFile::Close(); - } - - ////////////////////////////////////////////////////////////////////////// - ////////////////////////////////////////////////////////////////////////// - // DrillerSAXParser - ////////////////////////////////////////////////////////////////////////// - ////////////////////////////////////////////////////////////////////////// - //========================================================================= - // DrillerSAXParser - // [3/23/2011] - //========================================================================= - DrillerSAXParser::DrillerSAXParser(const TagCallbackType& tcb, const DataCallbackType& dcb) - : m_tagCallback(tcb) - , m_dataCallback(dcb) - { - } - - //========================================================================= - // ProcessStream - // [3/23/2011] - //========================================================================= - void - DrillerSAXParser::ProcessStream(DrillerInputStream& stream) - { - static const int processChunkSize = 15 * 1024; - char buffer[processChunkSize]; - unsigned int dataSize; - bool isEndianSwap = stream.IsEndianSwap(); - while ((dataSize = stream.ReadBinary(buffer, processChunkSize)) > 0) - { - char* dataStart = buffer; - char* dataEnd = dataStart + dataSize; - bool dataInBuffer = false; - if (!m_buffer.empty()) + DrillerOutputStream::StreamEntry* se = reinterpret_cast(dataStart); + if (isEndianSwap) { - m_buffer.insert(m_buffer.end(), dataStart, dataEnd); - dataStart = m_buffer.data(); - dataEnd = dataStart + m_buffer.size(); - dataInBuffer = true; + // endian swap + AZStd::endian_swap(se->name); + AZStd::endian_swap(se->sizeAndFlags); } - const int entrySize = sizeof(DrillerOutputStream::StreamEntry); - while (dataStart != dataEnd) + + u32 dataType = (se->sizeAndFlags & DrillerOutputStream::StreamEntry::dataInternalMask) >> DrillerOutputStream::StreamEntry::dataInternalShift; + u32 value = se->sizeAndFlags & DrillerOutputStream::StreamEntry::dataSizeMask; + Data de; + de.m_name = se->name; + de.m_stringPool = stream.GetStringPool(); + de.m_isPooledString = false; + de.m_isPooledStringCrc32 = false; + switch (dataType) { - if ((dataEnd - dataStart) < entrySize) // we need at least one entry to proceed + case DrillerOutputStream::StreamEntry::INT_TAG: + { + bool isStart = (value != 0); + m_tagCallback(se->name, isStart); + dataStart += entrySize; + } break; + case DrillerOutputStream::StreamEntry::INT_DATA_U8: + { + u8 value8 = static_cast(value); + de.m_data = &value8; + de.m_dataSize = 1; + de.m_isEndianSwap = false; + m_dataCallback(de); + dataStart += entrySize; + } break; + case DrillerOutputStream::StreamEntry::INT_DATA_U16: + { + u16 value16 = static_cast(value); + de.m_data = &value16; + de.m_dataSize = 2; + de.m_isEndianSwap = false; + m_dataCallback(de); + dataStart += entrySize; + } break; + case DrillerOutputStream::StreamEntry::INT_DATA_U29: + { + de.m_data = &value; + de.m_dataSize = 4; + de.m_isEndianSwap = false; + m_dataCallback(de); + dataStart += entrySize; + } break; + case DrillerOutputStream::StreamEntry::INT_POOLED_STRING: + { + unsigned int userDataSize = value; + if ((userDataSize + entrySize) <= (unsigned)(dataEnd - dataStart)) { - // not enough data to process, buffer it. + // Add string to the pool + AZ_Assert(de.m_stringPool != nullptr, "We require a string pool to parse this stream"); + AZ::u32 crc32; + const char* stringPtr; + dataStart += entrySize; + de.m_stringPool->InsertCopy(reinterpret_cast(dataStart), userDataSize, crc32, &stringPtr); + de.m_dataSize = userDataSize; + de.m_isEndianSwap = isEndianSwap; + de.m_isPooledString = true; + de.m_data = const_cast(static_cast(stringPtr)); + m_dataCallback(de); + dataStart += userDataSize; + } + else + { + // we can't process data right now add it to the buffer (if we have not done that already) if (!dataInBuffer) { m_buffer.insert(m_buffer.end(), dataStart, dataEnd); } - break; + dataEnd = dataStart; // exit the loop } - - DrillerOutputStream::StreamEntry* se = reinterpret_cast(dataStart); - if (isEndianSwap) + } break; + case DrillerOutputStream::StreamEntry::INT_POOLED_STRING_CRC32: + { + de.m_isPooledStringCrc32 = true; + AZ_Assert(value == 4, "The data size for a pooled string crc32 should be 4 bytes!"); + } // continue to INT_SIZE + case DrillerOutputStream::StreamEntry::INT_SIZE: + { + unsigned int userDataSize = value; + if ((userDataSize + entrySize) <= (unsigned)(dataEnd - dataStart)) // do we have all the date we need to process... { - // endian swap - AZStd::endian_swap(se->name); - AZStd::endian_swap(se->sizeAndFlags); - } - - u32 dataType = (se->sizeAndFlags & DrillerOutputStream::StreamEntry::dataInternalMask) >> DrillerOutputStream::StreamEntry::dataInternalShift; - u32 value = se->sizeAndFlags & DrillerOutputStream::StreamEntry::dataSizeMask; - Data de; - de.m_name = se->name; - de.m_stringPool = stream.GetStringPool(); - de.m_isPooledString = false; - de.m_isPooledStringCrc32 = false; - switch (dataType) - { - case DrillerOutputStream::StreamEntry::INT_TAG: - { - bool isStart = (value != 0); - m_tagCallback(se->name, isStart); dataStart += entrySize; - } break; - case DrillerOutputStream::StreamEntry::INT_DATA_U8: - { - u8 value8 = static_cast(value); - de.m_data = &value8; - de.m_dataSize = 1; - de.m_isEndianSwap = false; + de.m_data = dataStart; + de.m_dataSize = userDataSize; + de.m_isEndianSwap = isEndianSwap; m_dataCallback(de); - dataStart += entrySize; - } break; - case DrillerOutputStream::StreamEntry::INT_DATA_U16: - { - u16 value16 = static_cast(value); - de.m_data = &value16; - de.m_dataSize = 2; - de.m_isEndianSwap = false; - m_dataCallback(de); - dataStart += entrySize; - } break; - case DrillerOutputStream::StreamEntry::INT_DATA_U29: - { - de.m_data = &value; - de.m_dataSize = 4; - de.m_isEndianSwap = false; - m_dataCallback(de); - dataStart += entrySize; - } break; - case DrillerOutputStream::StreamEntry::INT_POOLED_STRING: - { - unsigned int userDataSize = value; - if ((userDataSize + entrySize) <= (unsigned)(dataEnd - dataStart)) - { - // Add string to the pool - AZ_Assert(de.m_stringPool != nullptr, "We require a string pool to parse this stream"); - AZ::u32 crc32; - const char* stringPtr; - dataStart += entrySize; - de.m_stringPool->InsertCopy(reinterpret_cast(dataStart), userDataSize, crc32, &stringPtr); - de.m_dataSize = userDataSize; - de.m_isEndianSwap = isEndianSwap; - de.m_isPooledString = true; - de.m_data = const_cast(static_cast(stringPtr)); - m_dataCallback(de); - dataStart += userDataSize; - } - else - { - // we can't process data right now add it to the buffer (if we have not done that already) - if (!dataInBuffer) - { - m_buffer.insert(m_buffer.end(), dataStart, dataEnd); - } - dataEnd = dataStart; // exit the loop - } - } break; - case DrillerOutputStream::StreamEntry::INT_POOLED_STRING_CRC32: - { - de.m_isPooledStringCrc32 = true; - AZ_Assert(value == 4, "The data size for a pooled string crc32 should be 4 bytes!"); - } // continue to INT_SIZE - case DrillerOutputStream::StreamEntry::INT_SIZE: - { - unsigned int userDataSize = value; - if ((userDataSize + entrySize) <= (unsigned)(dataEnd - dataStart)) // do we have all the date we need to process... - { - dataStart += entrySize; - de.m_data = dataStart; - de.m_dataSize = userDataSize; - de.m_isEndianSwap = isEndianSwap; - m_dataCallback(de); - dataStart += userDataSize; - } - else - { - // we can't process data right now add it to the buffer (if we have not done that already) - if (!dataInBuffer) - { - m_buffer.insert(m_buffer.end(), dataStart, dataEnd); - } - dataEnd = dataStart; // exit the loop - } - } break; - default: - { - AZ_Error("DrillerSAXParser",false,"Encounted unknown symbol (%i) while processing stream (%s). Aborting stream.\n",dataType, stream.GetIdentifier()); - - // If we can't process anything, we want to just escape the loop, to avoid spinning infinitely - dataEnd = dataStart; - } break; + dataStart += userDataSize; } - } - if (dataInBuffer) // if the data was in the buffer remove the processed data! - { - m_buffer.erase(m_buffer.begin(), m_buffer.begin() + (dataStart - m_buffer.data())); - } - } - } - - void DrillerSAXParser::Data::Read(AZ::Vector3& v) const - { - AZ_Assert(m_dataSize == sizeof(float) * 3, "We are expecting 3 floats for Vector3 element 0x%08x with size %d bytes", m_name, m_dataSize); - float* data = reinterpret_cast(m_data); - if (m_isEndianSwap) - { - AZStd::endian_swap(data, data + 3); - m_isEndianSwap = false; - } - v = Vector3::CreateFromFloat3(data); - } - void DrillerSAXParser::Data::Read(AZ::Vector4& v) const - { - AZ_Assert(m_dataSize == sizeof(float) * 4, "We are expecting 4 floats for Vector4 element 0x%08x with size %d bytes", m_name, m_dataSize); - float* data = reinterpret_cast(m_data); - if (m_isEndianSwap) - { - AZStd::endian_swap(data, data + 4); - m_isEndianSwap = false; - } - v = Vector4::CreateFromFloat4(data); - } - void DrillerSAXParser::Data::Read(AZ::Aabb& aabb) const - { - AZ_Assert(m_dataSize == sizeof(float) * 6, "We are expecting 6 floats for Aabb element 0x%08x with size %d bytes", m_name, m_dataSize); - float* data = reinterpret_cast(m_data); - if (m_isEndianSwap) - { - AZStd::endian_swap(data, data + 6); - m_isEndianSwap = false; - } - Vector3 min = Vector3::CreateFromFloat3(data); - Vector3 max = Vector3::CreateFromFloat3(&data[3]); - aabb = Aabb::CreateFromMinMax(min, max); - } - void DrillerSAXParser::Data::Read(AZ::Obb& obb) const - { - AZ_Assert(m_dataSize == sizeof(float) * 10, "We are expecting 10 floats for Obb element 0x%08x with size %d bytes", m_name, m_dataSize); - float* data = reinterpret_cast(m_data); - if (m_isEndianSwap) - { - AZStd::endian_swap(data, data + 10); - m_isEndianSwap = false; - } - Vector3 position = Vector3::CreateFromFloat3(data); - Quaternion rotation = Quaternion::CreateFromFloat4(&data[3]); - Vector3 halfLengths = Vector3::CreateFromFloat3(&data[7]); - obb = Obb::CreateFromPositionRotationAndHalfLengths(position, rotation, halfLengths); - } - void DrillerSAXParser::Data::Read(AZ::Transform& tm) const - { - AZ_Assert(m_dataSize == sizeof(float) * 12, "We are expecting 12 floats for Transform element 0x%08x with size %d bytes", m_name, m_dataSize); - float* data = reinterpret_cast(m_data); - if (m_isEndianSwap) - { - AZStd::endian_swap(data, data + 12); - m_isEndianSwap = false; - } - const Matrix3x4 matrix3x4 = Matrix3x4::CreateFromRowMajorFloat12(data); - tm = Transform::CreateFromMatrix3x4(matrix3x4); - } - void DrillerSAXParser::Data::Read(AZ::Matrix3x3& tm) const - { - AZ_Assert(m_dataSize == sizeof(float) * 9, "We are expecting 9 floats for Matrix3x3 element 0x%08x with size %d bytes", m_name, m_dataSize); - float* data = reinterpret_cast(m_data); - if (m_isEndianSwap) - { - AZStd::endian_swap(data, data + 9); - m_isEndianSwap = false; - } - tm = Matrix3x3::CreateFromRowMajorFloat9(data); - } - void DrillerSAXParser::Data::Read(AZ::Matrix4x4& tm) const - { - AZ_Assert(m_dataSize == sizeof(float) * 16, "We are expecting 16 floats for Matrix4x4 element 0x%08x with size %d bytes", m_name, m_dataSize); - float* data = reinterpret_cast(m_data); - if (m_isEndianSwap) - { - AZStd::endian_swap(data, data + 16); - m_isEndianSwap = false; - } - tm = Matrix4x4::CreateFromRowMajorFloat16(data); - } - void DrillerSAXParser::Data::Read(AZ::Quaternion& tm) const - { - AZ_Assert(m_dataSize == sizeof(float) * 4, "We are expecting 4 floats for Quaternion element 0x%08x with size %d bytes", m_name, m_dataSize); - float* data = reinterpret_cast(m_data); - if (m_isEndianSwap) - { - AZStd::endian_swap(data, data + 4); - m_isEndianSwap = false; - } - tm = Quaternion::CreateFromFloat4(data); - } - void DrillerSAXParser::Data::Read(AZ::Plane& plane) const - { - AZ::Vector4 coeff; - Read(coeff); - plane = Plane::CreateFromCoefficients(coeff.GetX(), coeff.GetY(), coeff.GetZ(), coeff.GetW()); - } - - const char* DrillerSAXParser::Data::PrepareString(unsigned int& stringLength) const - { - const char* srcData = reinterpret_cast(m_data); - stringLength = m_dataSize; - if (m_stringPool) - { - AZ::u32 crc32; - const char* stringPtr; - if (m_isPooledStringCrc32) - { - crc32 = *reinterpret_cast(m_data); - if (m_isEndianSwap) + else { - AZStd::endian_swap(crc32); + // we can't process data right now add it to the buffer (if we have not done that already) + if (!dataInBuffer) + { + m_buffer.insert(m_buffer.end(), dataStart, dataEnd); + } + dataEnd = dataStart; // exit the loop } - stringPtr = m_stringPool->Find(crc32); - AZ_Assert(stringPtr != nullptr, "Failed to find string with id 0x%08x in the string pool, proper stream read is impossible!", crc32); - stringLength = static_cast(strlen(stringPtr)); - } - else if (m_isPooledString) + } break; + default: { - stringPtr = srcData; // already stored in the pool just transfer the pointer + AZ_Error("DrillerSAXParser",false,"Encounted unknown symbol (%i) while processing stream (%s). Aborting stream.\n",dataType, stream.GetIdentifier()); + + // If we can't process anything, we want to just escape the loop, to avoid spinning infinitely + dataEnd = dataStart; + } break; } - else + } + if (dataInBuffer) // if the data was in the buffer remove the processed data! + { + m_buffer.erase(m_buffer.begin(), m_buffer.begin() + (dataStart - m_buffer.data())); + } + } + } + + void DrillerSAXParser::Data::Read(AZ::Vector3& v) const + { + AZ_Assert(m_dataSize == sizeof(float) * 3, "We are expecting 3 floats for Vector3 element 0x%08x with size %d bytes", m_name, m_dataSize); + float* data = reinterpret_cast(m_data); + if (m_isEndianSwap) + { + AZStd::endian_swap(data, data + 3); + m_isEndianSwap = false; + } + v = Vector3::CreateFromFloat3(data); + } + void DrillerSAXParser::Data::Read(AZ::Vector4& v) const + { + AZ_Assert(m_dataSize == sizeof(float) * 4, "We are expecting 4 floats for Vector4 element 0x%08x with size %d bytes", m_name, m_dataSize); + float* data = reinterpret_cast(m_data); + if (m_isEndianSwap) + { + AZStd::endian_swap(data, data + 4); + m_isEndianSwap = false; + } + v = Vector4::CreateFromFloat4(data); + } + void DrillerSAXParser::Data::Read(AZ::Aabb& aabb) const + { + AZ_Assert(m_dataSize == sizeof(float) * 6, "We are expecting 6 floats for Aabb element 0x%08x with size %d bytes", m_name, m_dataSize); + float* data = reinterpret_cast(m_data); + if (m_isEndianSwap) + { + AZStd::endian_swap(data, data + 6); + m_isEndianSwap = false; + } + Vector3 min = Vector3::CreateFromFloat3(data); + Vector3 max = Vector3::CreateFromFloat3(&data[3]); + aabb = Aabb::CreateFromMinMax(min, max); + } + void DrillerSAXParser::Data::Read(AZ::Obb& obb) const + { + AZ_Assert(m_dataSize == sizeof(float) * 10, "We are expecting 10 floats for Obb element 0x%08x with size %d bytes", m_name, m_dataSize); + float* data = reinterpret_cast(m_data); + if (m_isEndianSwap) + { + AZStd::endian_swap(data, data + 10); + m_isEndianSwap = false; + } + Vector3 position = Vector3::CreateFromFloat3(data); + Quaternion rotation = Quaternion::CreateFromFloat4(&data[3]); + Vector3 halfLengths = Vector3::CreateFromFloat3(&data[7]); + obb = Obb::CreateFromPositionRotationAndHalfLengths(position, rotation, halfLengths); + } + void DrillerSAXParser::Data::Read(AZ::Transform& tm) const + { + AZ_Assert(m_dataSize == sizeof(float) * 12, "We are expecting 12 floats for Transform element 0x%08x with size %d bytes", m_name, m_dataSize); + float* data = reinterpret_cast(m_data); + if (m_isEndianSwap) + { + AZStd::endian_swap(data, data + 12); + m_isEndianSwap = false; + } + const Matrix3x4 matrix3x4 = Matrix3x4::CreateFromRowMajorFloat12(data); + tm = Transform::CreateFromMatrix3x4(matrix3x4); + } + void DrillerSAXParser::Data::Read(AZ::Matrix3x3& tm) const + { + AZ_Assert(m_dataSize == sizeof(float) * 9, "We are expecting 9 floats for Matrix3x3 element 0x%08x with size %d bytes", m_name, m_dataSize); + float* data = reinterpret_cast(m_data); + if (m_isEndianSwap) + { + AZStd::endian_swap(data, data + 9); + m_isEndianSwap = false; + } + tm = Matrix3x3::CreateFromRowMajorFloat9(data); + } + void DrillerSAXParser::Data::Read(AZ::Matrix4x4& tm) const + { + AZ_Assert(m_dataSize == sizeof(float) * 16, "We are expecting 16 floats for Matrix4x4 element 0x%08x with size %d bytes", m_name, m_dataSize); + float* data = reinterpret_cast(m_data); + if (m_isEndianSwap) + { + AZStd::endian_swap(data, data + 16); + m_isEndianSwap = false; + } + tm = Matrix4x4::CreateFromRowMajorFloat16(data); + } + void DrillerSAXParser::Data::Read(AZ::Quaternion& tm) const + { + AZ_Assert(m_dataSize == sizeof(float) * 4, "We are expecting 4 floats for Quaternion element 0x%08x with size %d bytes", m_name, m_dataSize); + float* data = reinterpret_cast(m_data); + if (m_isEndianSwap) + { + AZStd::endian_swap(data, data + 4); + m_isEndianSwap = false; + } + tm = Quaternion::CreateFromFloat4(data); + } + void DrillerSAXParser::Data::Read(AZ::Plane& plane) const + { + AZ::Vector4 coeff; + Read(coeff); + plane = Plane::CreateFromCoefficients(coeff.GetX(), coeff.GetY(), coeff.GetZ(), coeff.GetW()); + } + + const char* DrillerSAXParser::Data::PrepareString(unsigned int& stringLength) const + { + const char* srcData = reinterpret_cast(m_data); + stringLength = m_dataSize; + if (m_stringPool) + { + AZ::u32 crc32; + const char* stringPtr; + if (m_isPooledStringCrc32) + { + crc32 = *reinterpret_cast(m_data); + if (m_isEndianSwap) { - // Store copy of the string in the pool to save memory (keep only one reference of the string). - m_stringPool->InsertCopy(reinterpret_cast(srcData), stringLength, crc32, &stringPtr); + AZStd::endian_swap(crc32); } - srcData = stringPtr; + stringPtr = m_stringPool->Find(crc32); + AZ_Assert(stringPtr != nullptr, "Failed to find string with id 0x%08x in the string pool, proper stream read is impossible!", crc32); + stringLength = static_cast(strlen(stringPtr)); + } + else if (m_isPooledString) + { + stringPtr = srcData; // already stored in the pool just transfer the pointer } else { - AZ_Assert(m_isPooledString == false && m_isPooledStringCrc32 == false, "This stream requires using of a string pool as the string is send only once and afterwards only the Crc32 is used!"); + // Store copy of the string in the pool to save memory (keep only one reference of the string). + m_stringPool->InsertCopy(reinterpret_cast(srcData), stringLength, crc32, &stringPtr); } - return srcData; + srcData = stringPtr; + } + else + { + AZ_Assert(m_isPooledString == false && m_isPooledStringCrc32 == false, "This stream requires using of a string pool as the string is send only once and afterwards only the Crc32 is used!"); + } + return srcData; + } + + ////////////////////////////////////////////////////////////////////////// + ////////////////////////////////////////////////////////////////////////// + // DrillerDOMParser + ////////////////////////////////////////////////////////////////////////// + ////////////////////////////////////////////////////////////////////////// + + //========================================================================= + // Node::GetTag + // [1/23/2013] + //========================================================================= + const DrillerDOMParser::Node* DrillerDOMParser::Node::GetTag(u32 tagName) const + { + const Node* tagNode = nullptr; + for (Node::NodeListType::const_iterator i = m_tags.begin(); i != m_tags.end(); ++i) + { + if ((*i).m_name == tagName) + { + tagNode = &*i; + break; + } + } + return tagNode; + } + + //========================================================================= + // Node::GetData + // [3/23/2011] + //========================================================================= + const DrillerDOMParser::Data* DrillerDOMParser::Node::GetData(u32 dataName) const + { + const Data* dataNode = nullptr; + for (Node::DataListType::const_iterator i = m_data.begin(); i != m_data.end(); ++i) + { + if (i->m_name == dataName) + { + dataNode = &*i; + break; + } + } + return dataNode; + } + + //========================================================================= + // DrillerDOMParser + // [3/23/2011] + //========================================================================= + DrillerDOMParser::DrillerDOMParser(bool isPersistentInputData) + : DrillerSAXParser(TagCallbackType(this, &DrillerDOMParser::OnTag), DataCallbackType(this, &DrillerDOMParser::OnData)) + , m_isPersistentInputData(isPersistentInputData) + { + m_root.m_name = 0; + m_root.m_parent = nullptr; + m_topNode = &m_root; + } + static int g_numFree = 0; + //========================================================================= + // ~DrillerDOMParser + // [3/23/2011] + //========================================================================= + DrillerDOMParser::~DrillerDOMParser() + { + DeleteNode(m_root); + } + + //========================================================================= + // OnTag + // [3/23/2011] + //========================================================================= + void + DrillerDOMParser::OnTag(AZ::u32 name, bool isOpen) + { + if (isOpen) + { + m_topNode->m_tags.push_back(); + Node& node = m_topNode->m_tags.back(); + node.m_name = name; + node.m_parent = m_topNode; + + m_topNode = &node; + } + else + { + AZ_Assert(m_topNode->m_name == name, "We have opened tag with name 0x%08x and closing with name 0x%08x", m_topNode->m_name, name); + m_topNode = m_topNode->m_parent; + } + } + //========================================================================= + // OnData + // [3/23/2011] + //========================================================================= + void + DrillerDOMParser::OnData(const Data& data) + { + Data de = data; + if (!m_isPersistentInputData) + { + de.m_data = azmalloc(data.m_dataSize, 1, OSAllocator); + memcpy(const_cast(de.m_data), data.m_data, data.m_dataSize); + } + m_topNode->m_data.push_back(de); + } + //========================================================================= + // DeleteNode + // [3/23/2011] + //========================================================================= + void + DrillerDOMParser::DeleteNode(Node& node) + { + if (!m_isPersistentInputData) + { + for (Node::DataListType::iterator iter = node.m_data.begin(); iter != node.m_data.end(); ++iter) + { + azfree(iter->m_data, OSAllocator, iter->m_dataSize); + ++g_numFree; + } + node.m_data.clear(); + } + for (Node::NodeListType::iterator iter = node.m_tags.begin(); iter != node.m_tags.end(); ++iter) + { + DeleteNode(*iter); + } + node.m_tags.clear(); + } + + ////////////////////////////////////////////////////////////////////////// + ////////////////////////////////////////////////////////////////////////// + // DrillerSAXParserHandler + ////////////////////////////////////////////////////////////////////////// + ////////////////////////////////////////////////////////////////////////// + + //========================================================================= + // DrillerSAXParserHandler + // [3/14/2013] + //========================================================================= + DrillerSAXParserHandler::DrillerSAXParserHandler(DrillerHandlerParser* rootHandler) + : DrillerSAXParser(TagCallbackType(this, &DrillerSAXParserHandler::OnTag), DataCallbackType(this, &DrillerSAXParserHandler::OnData)) + { + // Push the root element + m_stack.push_back(rootHandler); + } + + //========================================================================= + // OnTag + // [3/14/2013] + //========================================================================= + void DrillerSAXParserHandler::OnTag(u32 name, bool isOpen) + { + if (m_stack.empty()) + { + return; } - ////////////////////////////////////////////////////////////////////////// - ////////////////////////////////////////////////////////////////////////// - // DrillerDOMParser - ////////////////////////////////////////////////////////////////////////// - ////////////////////////////////////////////////////////////////////////// - - //========================================================================= - // Node::GetTag - // [1/23/2013] - //========================================================================= - const DrillerDOMParser::Node* DrillerDOMParser::Node::GetTag(u32 tagName) const + DrillerHandlerParser* childHandler = nullptr; + DrillerHandlerParser* currentHandler = m_stack.back(); + if (isOpen) { - const Node* tagNode = nullptr; - for (Node::NodeListType::const_iterator i = m_tags.begin(); i != m_tags.end(); ++i) + if (currentHandler != nullptr) { - if ((*i).m_name == tagName) + childHandler = currentHandler->OnEnterTag(name); + AZ_Warning("Driller", !currentHandler->IsWarnOnUnsupportedTags() || childHandler != nullptr, "Could not find handler for tag 0x%08x", name); + } + m_stack.push_back(childHandler); + } + else + { + m_stack.pop_back(); + if (!m_stack.empty()) + { + DrillerHandlerParser* parentHandler = m_stack.back(); + if (parentHandler) { - tagNode = &*i; - break; - } - } - return tagNode; - } - - //========================================================================= - // Node::GetData - // [3/23/2011] - //========================================================================= - const DrillerDOMParser::Data* DrillerDOMParser::Node::GetData(u32 dataName) const - { - const Data* dataNode = nullptr; - for (Node::DataListType::const_iterator i = m_data.begin(); i != m_data.end(); ++i) - { - if (i->m_name == dataName) - { - dataNode = &*i; - break; - } - } - return dataNode; - } - - //========================================================================= - // DrillerDOMParser - // [3/23/2011] - //========================================================================= - DrillerDOMParser::DrillerDOMParser(bool isPersistentInputData) - : DrillerSAXParser(TagCallbackType(this, &DrillerDOMParser::OnTag), DataCallbackType(this, &DrillerDOMParser::OnData)) - , m_isPersistentInputData(isPersistentInputData) - { - m_root.m_name = 0; - m_root.m_parent = nullptr; - m_topNode = &m_root; - } - static int g_numFree = 0; - //========================================================================= - // ~DrillerDOMParser - // [3/23/2011] - //========================================================================= - DrillerDOMParser::~DrillerDOMParser() - { - DeleteNode(m_root); - } - - //========================================================================= - // OnTag - // [3/23/2011] - //========================================================================= - void - DrillerDOMParser::OnTag(AZ::u32 name, bool isOpen) - { - if (isOpen) - { - m_topNode->m_tags.push_back(); - Node& node = m_topNode->m_tags.back(); - node.m_name = name; - node.m_parent = m_topNode; - - m_topNode = &node; - } - else - { - AZ_Assert(m_topNode->m_name == name, "We have opened tag with name 0x%08x and closing with name 0x%08x", m_topNode->m_name, name); - m_topNode = m_topNode->m_parent; - } - } - //========================================================================= - // OnData - // [3/23/2011] - //========================================================================= - void - DrillerDOMParser::OnData(const Data& data) - { - Data de = data; - if (!m_isPersistentInputData) - { - de.m_data = azmalloc(data.m_dataSize, 1, OSAllocator); - memcpy(const_cast(de.m_data), data.m_data, data.m_dataSize); - } - m_topNode->m_data.push_back(de); - } - //========================================================================= - // DeleteNode - // [3/23/2011] - //========================================================================= - void - DrillerDOMParser::DeleteNode(Node& node) - { - if (!m_isPersistentInputData) - { - for (Node::DataListType::iterator iter = node.m_data.begin(); iter != node.m_data.end(); ++iter) - { - azfree(iter->m_data, OSAllocator, iter->m_dataSize); - ++g_numFree; - } - node.m_data.clear(); - } - for (Node::NodeListType::iterator iter = node.m_tags.begin(); iter != node.m_tags.end(); ++iter) - { - DeleteNode(*iter); - } - node.m_tags.clear(); - } - - ////////////////////////////////////////////////////////////////////////// - ////////////////////////////////////////////////////////////////////////// - // DrillerSAXParserHandler - ////////////////////////////////////////////////////////////////////////// - ////////////////////////////////////////////////////////////////////////// - - //========================================================================= - // DrillerSAXParserHandler - // [3/14/2013] - //========================================================================= - DrillerSAXParserHandler::DrillerSAXParserHandler(DrillerHandlerParser* rootHandler) - : DrillerSAXParser(TagCallbackType(this, &DrillerSAXParserHandler::OnTag), DataCallbackType(this, &DrillerSAXParserHandler::OnData)) - { - // Push the root element - m_stack.push_back(rootHandler); - } - - //========================================================================= - // OnTag - // [3/14/2013] - //========================================================================= - void DrillerSAXParserHandler::OnTag(u32 name, bool isOpen) - { - if (m_stack.size() == 0) - { - return; - } - - DrillerHandlerParser* childHandler = nullptr; - DrillerHandlerParser* currentHandler = m_stack.back(); - if (isOpen) - { - if (currentHandler != nullptr) - { - childHandler = currentHandler->OnEnterTag(name); - AZ_Warning("Driller", !currentHandler->IsWarnOnUnsupportedTags() || childHandler != nullptr, "Could not find handler for tag 0x%08x", name); - } - m_stack.push_back(childHandler); - } - else - { - m_stack.pop_back(); - if (m_stack.size() > 0) - { - DrillerHandlerParser* parentHandler = m_stack.back(); - if (parentHandler) - { - parentHandler->OnExitTag(currentHandler, name); - } + parentHandler->OnExitTag(currentHandler, name); } } } + } - //========================================================================= - // OnData - // [3/14/2013] - //========================================================================= - void DrillerSAXParserHandler::OnData(const DrillerSAXParser::Data& data) + //========================================================================= + // OnData + // [3/14/2013] + //========================================================================= + void DrillerSAXParserHandler::OnData(const DrillerSAXParser::Data& data) + { + if (m_stack.empty()) { - if (m_stack.size() == 0) - { - return; - } - - DrillerHandlerParser* currentHandler = m_stack.back(); - if (currentHandler) - { - currentHandler->OnData(data); - } + return; } - } // namespace Debug -} // namespace AZ + + DrillerHandlerParser* currentHandler = m_stack.back(); + if (currentHandler) + { + currentHandler->OnData(data); + } + } +} // namespace AZ::Debug diff --git a/Code/Framework/AzCore/AzCore/IO/CompressionBus.cpp b/Code/Framework/AzCore/AzCore/IO/CompressionBus.cpp index fb9f8841a2..7878ec6e9e 100644 --- a/Code/Framework/AzCore/AzCore/IO/CompressionBus.cpp +++ b/Code/Framework/AzCore/AzCore/IO/CompressionBus.cpp @@ -8,38 +8,35 @@ #include -namespace AZ +namespace AZ::IO { - namespace IO + CompressionInfo::CompressionInfo(CompressionInfo&& rhs) { - CompressionInfo::CompressionInfo(CompressionInfo&& rhs) - { - *this = AZStd::move(rhs); - } + *this = AZStd::move(rhs); + } - CompressionInfo& CompressionInfo::operator=(CompressionInfo&& rhs) - { - m_decompressor = AZStd::move(rhs.m_decompressor); - m_archiveFilename = AZStd::move(rhs.m_archiveFilename); - m_compressionTag = rhs.m_compressionTag; - m_offset = rhs.m_offset; - m_compressedSize = rhs.m_compressedSize; - m_uncompressedSize = rhs.m_uncompressedSize; - m_conflictResolution = rhs.m_conflictResolution; - m_isCompressed = rhs.m_isCompressed; - m_isSharedPak = rhs.m_isSharedPak; + CompressionInfo& CompressionInfo::operator=(CompressionInfo&& rhs) + { + m_decompressor = AZStd::move(rhs.m_decompressor); + m_archiveFilename = AZStd::move(rhs.m_archiveFilename); + m_compressionTag = rhs.m_compressionTag; + m_offset = rhs.m_offset; + m_compressedSize = rhs.m_compressedSize; + m_uncompressedSize = rhs.m_uncompressedSize; + m_conflictResolution = rhs.m_conflictResolution; + m_isCompressed = rhs.m_isCompressed; + m_isSharedPak = rhs.m_isSharedPak; - return *this; - } + return *this; + } - namespace CompressionUtils + namespace CompressionUtils + { + bool FindCompressionInfo(CompressionInfo& info, const AZStd::string_view filename) { - bool FindCompressionInfo(CompressionInfo& info, const AZStd::string_view filename) - { - bool result = false; - CompressionBus::Broadcast(&CompressionBus::Events::FindCompressionInfo, result, info, filename); - return result; - } + bool result = false; + CompressionBus::Broadcast(&CompressionBus::Events::FindCompressionInfo, result, info, filename); + return result; } } -} +} // namespace AZ::IO diff --git a/Code/Framework/AzCore/AzCore/IO/Compressor.cpp b/Code/Framework/AzCore/AzCore/IO/Compressor.cpp index e223730ce8..16527422ad 100644 --- a/Code/Framework/AzCore/AzCore/IO/Compressor.cpp +++ b/Code/Framework/AzCore/AzCore/IO/Compressor.cpp @@ -10,32 +10,29 @@ #include #include -namespace AZ +namespace AZ::IO { - namespace IO + //========================================================================= + // WriteHeaderAndData + // [12/13/2012] + //========================================================================= + bool Compressor::WriteHeaderAndData(CompressorStream* compressorStream) { - //========================================================================= - // WriteHeaderAndData - // [12/13/2012] - //========================================================================= - bool Compressor::WriteHeaderAndData(CompressorStream* compressorStream) + AZ_Assert(compressorStream->CanWrite(), "Stream is not open for write!"); + AZ_Assert(compressorStream->GetCompressorData(), "Stream doesn't have attached compressor, call WriteCompressed first!"); + AZ_Assert(compressorStream->GetCompressorData()->m_compressor == this, "Invalid compressor data! Data belongs to a different compressor"); + CompressorHeader header; + header.SetAZCS(); + header.m_compressorId = GetTypeId(); + header.m_uncompressedSize = compressorStream->GetCompressorData()->m_uncompressedSize; + AZStd::endian_swap(header.m_compressorId); + AZStd::endian_swap(header.m_uncompressedSize); + GenericStream* baseStream = compressorStream->GetWrappedStream(); + if (baseStream->WriteAtOffset(sizeof(CompressorHeader), &header, 0U) == sizeof(CompressorHeader)) { - AZ_Assert(compressorStream->CanWrite(), "Stream is not open for write!"); - AZ_Assert(compressorStream->GetCompressorData(), "Stream doesn't have attached compressor, call WriteCompressed first!"); - AZ_Assert(compressorStream->GetCompressorData()->m_compressor == this, "Invalid compressor data! Data belongs to a different compressor"); - CompressorHeader header; - header.SetAZCS(); - header.m_compressorId = GetTypeId(); - header.m_uncompressedSize = compressorStream->GetCompressorData()->m_uncompressedSize; - AZStd::endian_swap(header.m_compressorId); - AZStd::endian_swap(header.m_uncompressedSize); - GenericStream* baseStream = compressorStream->GetWrappedStream(); - if (baseStream->WriteAtOffset(sizeof(CompressorHeader), &header, 0U) == sizeof(CompressorHeader)) - { - return true; - } - - return false; + return true; } - } // namespace IO -} // namespace AZ + + return false; + } +} // namespace AZ::IO diff --git a/Code/Framework/AzCore/AzCore/IO/Compressor.h b/Code/Framework/AzCore/AzCore/IO/Compressor.h index 9b910a0ea4..340366d82f 100644 --- a/Code/Framework/AzCore/AzCore/IO/Compressor.h +++ b/Code/Framework/AzCore/AzCore/IO/Compressor.h @@ -5,74 +5,67 @@ * SPDX-License-Identifier: Apache-2.0 OR MIT * */ -#ifndef AZCORE_IO_COMPRESSOR_H -#define AZCORE_IO_COMPRESSOR_H +#pragma once #include -namespace AZ +namespace AZ::IO { - namespace IO + class CompressorStream; + + /** + * Compressor/Decompressor base interface. + * Used for all stream compressors. + */ + class Compressor { - class CompressorStream; + public: + typedef AZ::u64 SizeType; + static const int m_maxHeaderSize = 4096; /// When we open a stream to check if it's compressed we read the first m_maxHeaderSize bytes. - /** - * Compressor/Decompressor base interface. - * Used for all stream compressors. - */ - class Compressor - { - public: - typedef AZ::u64 SizeType; - static const int m_maxHeaderSize = 4096; /// When we open a stream to check if it's compressed we read the first m_maxHeaderSize bytes. + virtual ~Compressor() {} + /// Return compressor type id. + virtual AZ::u32 GetTypeId() const = 0; + /// Called when we open a stream to Read for the first time. Data contains the first. dataSize <= m_maxHeaderSize. + virtual bool ReadHeaderAndData(CompressorStream* stream, AZ::u8* data, unsigned int dataSize) = 0; + /// Called when we are about to start writing to a compressed stream. (Must be called first to write compressor header) + virtual bool WriteHeaderAndData(CompressorStream* stream); + /// Forwarded function from the Device when we from a compressed stream. + virtual SizeType Read(CompressorStream* stream, SizeType byteSize, SizeType offset, void* buffer) = 0; + /// Forwarded function from the Device when we write to a compressed stream. + virtual SizeType Write(CompressorStream* stream, SizeType byteSize, const void* data, SizeType offset = SizeType(-1)) = 0; + /// Write a seek point. + virtual bool WriteSeekPoint(CompressorStream* stream) { (void)stream; return false; } + /// Initializes Compressor for writing data. + virtual bool StartCompressor(CompressorStream* stream, int compressionLevel, SizeType autoSeekDataSize) { (void)stream; (void)compressionLevel; (void)autoSeekDataSize; return false; } + /// Called just before we close the stream. All compression data will be flushed and finalized. (You can't add data afterwards). + virtual bool Close(CompressorStream* stream) = 0; + }; - virtual ~Compressor() {} - /// Return compressor type id. - virtual AZ::u32 GetTypeId() const = 0; - /// Called when we open a stream to Read for the first time. Data contains the first. dataSize <= m_maxHeaderSize. - virtual bool ReadHeaderAndData(CompressorStream* stream, AZ::u8* data, unsigned int dataSize) = 0; - /// Called when we are about to start writing to a compressed stream. (Must be called first to write compressor header) - virtual bool WriteHeaderAndData(CompressorStream* stream); - /// Forwarded function from the Device when we from a compressed stream. - virtual SizeType Read(CompressorStream* stream, SizeType byteSize, SizeType offset, void* buffer) = 0; - /// Forwarded function from the Device when we write to a compressed stream. - virtual SizeType Write(CompressorStream* stream, SizeType byteSize, const void* data, SizeType offset = SizeType(-1)) = 0; - /// Write a seek point. - virtual bool WriteSeekPoint(CompressorStream* stream) { (void)stream; return false; } - /// Initializes Compressor for writing data. - virtual bool StartCompressor(CompressorStream* stream, int compressionLevel, SizeType autoSeekDataSize) { (void)stream; (void)compressionLevel; (void)autoSeekDataSize; return false; } - /// Called just before we close the stream. All compression data will be flushed and finalized. (You can't add data afterwards). - virtual bool Close(CompressorStream* stream) = 0; - }; + /** + * Base compressor data assigned for all compressors. + */ + class CompressorData + { + public: + virtual ~CompressorData() {} - /** - * Base compressor data assigned for all compressors. - */ - class CompressorData - { - public: - virtual ~CompressorData() {} + Compressor* m_compressor; + AZ::u64 m_uncompressedSize; + }; - Compressor* m_compressor; - AZ::u64 m_uncompressedSize; - }; + /** + * All data is stored in network order (big endian). + */ + struct CompressorHeader + { + CompressorHeader() { m_azcs[0] = 0; m_azcs[1] = 0; m_azcs[2] = 0; m_azcs[3] = 0; } - /** - * All data is stored in network order (big endian). - */ - struct CompressorHeader - { - CompressorHeader() { m_azcs[0] = 0; m_azcs[1] = 0; m_azcs[2] = 0; m_azcs[3] = 0; } + bool IsValid() const { return (m_azcs[0] == 'A' && m_azcs[1] == 'Z' && m_azcs[2] == 'C' && m_azcs[3] == 'S'); } + void SetAZCS() { m_azcs[0] = 'A'; m_azcs[1] = 'Z'; m_azcs[2] = 'C'; m_azcs[3] = 'S'; } - inline bool IsValid() const { return (m_azcs[0] == 'A' && m_azcs[1] == 'Z' && m_azcs[2] == 'C' && m_azcs[3] == 'S'); } - void SetAZCS() { m_azcs[0] = 'A'; m_azcs[1] = 'Z'; m_azcs[2] = 'C'; m_azcs[3] = 'S'; } - - char m_azcs[4]; ///< String contains 'AZCS' AmaZon Compressed Stream - AZ::u32 m_compressorId; ///< Compression method. - AZ::u64 m_uncompressedSize; ///< Uncompressed file size. - }; - } // namespace IO -} // namespace AZ - -#endif // AZCORE_IO_COMPRESSOR_H -#pragma once + char m_azcs[4]; ///< String contains 'AZCS' AmaZon Compressed Stream + AZ::u32 m_compressorId; ///< Compression method. + AZ::u64 m_uncompressedSize; ///< Uncompressed file size. + }; +} // namespace AZ::IO diff --git a/Code/Framework/AzCore/AzCore/IO/CompressorStream.cpp b/Code/Framework/AzCore/AzCore/IO/CompressorStream.cpp index 6f82cc428d..a1012ee404 100644 --- a/Code/Framework/AzCore/AzCore/IO/CompressorStream.cpp +++ b/Code/Framework/AzCore/AzCore/IO/CompressorStream.cpp @@ -15,9 +15,7 @@ #include #include -namespace AZ -{ -namespace IO +namespace AZ::IO { /*! \brief Constructs a compressor stream using the supplied filename and OpenFlags to open a file on disk @@ -300,7 +298,4 @@ Compressor* CompressorStream::CreateCompressor(AZ::u32 compressorId) return m_compressor.get(); } -} // namespace IO -} // namespace AZ - - +} // namespace AZ::IO diff --git a/Code/Framework/AzCore/AzCore/IO/CompressorZLib.cpp b/Code/Framework/AzCore/AzCore/IO/CompressorZLib.cpp index f973c0e95a..03a218dbfe 100644 --- a/Code/Framework/AzCore/AzCore/IO/CompressorZLib.cpp +++ b/Code/Framework/AzCore/AzCore/IO/CompressorZLib.cpp @@ -13,543 +13,540 @@ #include #include -namespace AZ +namespace AZ::IO { - namespace IO + //========================================================================= + // CompressorZLib + // [12/13/2012] + //========================================================================= + CompressorZLib::CompressorZLib(unsigned int decompressionCachePerStream, unsigned int dataBufferSize) + : m_lastReadStream(nullptr) + , m_lastReadStreamOffset(0) + , m_lastReadStreamSize(0) + , m_compressedDataBuffer(nullptr) + , m_compressedDataBufferSize(dataBufferSize) + , m_compressedDataBufferUseCount(0) + , m_decompressionCachePerStream(decompressionCachePerStream) + { - //========================================================================= - // CompressorZLib - // [12/13/2012] - //========================================================================= - CompressorZLib::CompressorZLib(unsigned int decompressionCachePerStream, unsigned int dataBufferSize) - : m_lastReadStream(nullptr) - , m_lastReadStreamOffset(0) - , m_lastReadStreamSize(0) - , m_compressedDataBuffer(nullptr) - , m_compressedDataBufferSize(dataBufferSize) - , m_compressedDataBufferUseCount(0) - , m_decompressionCachePerStream(decompressionCachePerStream) + AZ_Assert((dataBufferSize % (32 * 1024)) == 0, "Data buffer size %d must be multiple of 32 KB!", dataBufferSize); + AZ_Assert((decompressionCachePerStream % (32 * 1024)) == 0, "Decompress cache size %d must be multiple of 32 KB!", decompressionCachePerStream); + } + //========================================================================= + // !CompressorZLib + // [12/13/2012] + //========================================================================= + CompressorZLib::~CompressorZLib() + { + AZ_Warning("IO", m_compressedDataBufferUseCount == 0, "CompressorZLib has it's data buffer still referenced, it means that %d compressed streams have NOT closed! Freeing data...", m_compressedDataBufferUseCount); + while (m_compressedDataBufferUseCount) { - AZ_Assert((dataBufferSize % (32 * 1024)) == 0, "Data buffer size %d must be multiple of 32 KB!", dataBufferSize); - AZ_Assert((decompressionCachePerStream % (32 * 1024)) == 0, "Decompress cache size %d must be multiple of 32 KB!", decompressionCachePerStream); + ReleaseDataBuffer(); } + } - //========================================================================= - // !CompressorZLib - // [12/13/2012] - //========================================================================= - CompressorZLib::~CompressorZLib() + //========================================================================= + // GetTypeId + // [12/13/2012] + //========================================================================= + AZ::u32 CompressorZLib::TypeId() + { + return AZ_CRC("ZLib", 0x73887d3a); + } + + //========================================================================= + // ReadHeaderAndData + // [12/13/2012] + //========================================================================= + bool CompressorZLib::ReadHeaderAndData(CompressorStream* stream, AZ::u8* data, unsigned int dataSize) + { + if (stream->GetCompressorData() != nullptr) // we already have compressor data { - AZ_Warning("IO", m_compressedDataBufferUseCount == 0, "CompressorZLib has it's data buffer still referenced, it means that %d compressed streams have NOT closed! Freeing data...", m_compressedDataBufferUseCount); - while (m_compressedDataBufferUseCount) - { - ReleaseDataBuffer(); - } - } - - //========================================================================= - // GetTypeId - // [12/13/2012] - //========================================================================= - AZ::u32 CompressorZLib::TypeId() - { - return AZ_CRC("ZLib", 0x73887d3a); - } - - //========================================================================= - // ReadHeaderAndData - // [12/13/2012] - //========================================================================= - bool CompressorZLib::ReadHeaderAndData(CompressorStream* stream, AZ::u8* data, unsigned int dataSize) - { - if (stream->GetCompressorData() != nullptr) // we already have compressor data - { - return false; - } - - // Read the ZLib header should be after the default compression header... - // We should not be in this function otherwise. - if (dataSize < sizeof(CompressorZLibHeader) + sizeof(ZLib::Header)) - { - AZ_Assert(false, "We did not read enough data, we have only %d bytes left in the buffer and we need %d!", dataSize, sizeof(CompressorZLibHeader) + sizeof(ZLib::Header)); - return false; - } - - AcquireDataBuffer(); - - CompressorZLibHeader* hdr = reinterpret_cast(data); - AZStd::endian_swap(hdr->m_numSeekPoints); - dataSize -= sizeof(CompressorZLibHeader); - data += sizeof(CompressorZLibHeader); - - CompressorZLibData* zlibData = aznew CompressorZLibData; - zlibData->m_compressor = this; - zlibData->m_uncompressedSize = 0; - zlibData->m_zlibHeader = *reinterpret_cast(data); - dataSize -= sizeof(zlibData->m_zlibHeader); - data += sizeof(zlibData->m_zlibHeader); - zlibData->m_decompressNextOffset = sizeof(CompressorHeader) + sizeof(CompressorZLibHeader) + sizeof(ZLib::Header); // start after the headers - - AZ_Assert(hdr->m_numSeekPoints > 0, "We should have at least one seek point for the entire stream!"); - - // go the end of the file and read all sync points. - SizeType compressedFileEnd = stream->GetLength(); - if (compressedFileEnd == 0) - { - delete zlibData; - return false; - } - - zlibData->m_seekPoints.resize(hdr->m_numSeekPoints); - SizeType dataToRead = sizeof(CompressorZLibSeekPoint) * static_cast(hdr->m_numSeekPoints); - SizeType seekPointOffset = compressedFileEnd - dataToRead; - AZ_Assert(seekPointOffset <= compressedFileEnd, "We have an invalid archive, this is impossible!"); - GenericStream* baseStream = stream->GetWrappedStream(); - if (baseStream->ReadAtOffset(dataToRead, zlibData->m_seekPoints.data(), seekPointOffset) != dataToRead) - { - delete zlibData; - return false; - } - for (size_t i = 0; i < zlibData->m_seekPoints.size(); ++i) - { - AZStd::endian_swap(zlibData->m_seekPoints[i].m_compressedOffset); - AZStd::endian_swap(zlibData->m_seekPoints[i].m_uncompressedOffset); - } - - if (m_decompressionCachePerStream) - { - zlibData->m_decompressedCache = reinterpret_cast(azmalloc(m_decompressionCachePerStream, m_CompressedDataBufferAlignment, AZ::SystemAllocator, "CompressorZLib")); - } - - zlibData->m_decompressLastOffset = seekPointOffset; // set the start address of the seek points as the last valid read address for the compressed stream. - - zlibData->m_zlib.StartDecompressor(&zlibData->m_zlibHeader); - - stream->SetCompressorData(zlibData); - - return true; - } - - //========================================================================= - // WriteHeaderAndData - // [12/13/2012] - //========================================================================= - bool CompressorZLib::WriteHeaderAndData(CompressorStream* stream) - { - if (!Compressor::WriteHeaderAndData(stream)) - { - return false; - } - - CompressorZLibData* compressorData = static_cast(stream->GetCompressorData()); - CompressorZLibHeader header; - header.m_numSeekPoints = static_cast(compressorData->m_seekPoints.size()); - AZStd::endian_swap(header.m_numSeekPoints); - GenericStream* baseStream = stream->GetWrappedStream(); - if (baseStream->WriteAtOffset(sizeof(header), &header, sizeof(CompressorHeader)) == sizeof(header)) - { - return true; - } - return false; } - //========================================================================= - // FillFromDecompressCache - // [12/14/2012] - //========================================================================= - inline CompressorZLib::SizeType CompressorZLib::FillFromDecompressCache(CompressorZLibData* zlibData, void*& buffer, SizeType& byteSize, SizeType& offset) + // Read the ZLib header should be after the default compression header... + // We should not be in this function otherwise. + if (dataSize < sizeof(CompressorZLibHeader) + sizeof(ZLib::Header)) { - SizeType firstOffsetInCache = zlibData->m_decompressedCacheOffset; - SizeType lastOffsetInCache = firstOffsetInCache + zlibData->m_decompressedCacheDataSize; - SizeType firstDataOffset = offset; - SizeType lastDataOffset = offset + byteSize; - SizeType numCopied = 0; - if (firstOffsetInCache < lastDataOffset && lastOffsetInCache > firstDataOffset) // check if there is data in the cache - { - size_t copyOffsetStart = 0; - size_t copyOffsetEnd = zlibData->m_decompressedCacheDataSize; - - size_t bufferCopyOffset = 0; - - if (firstOffsetInCache < firstDataOffset) - { - copyOffsetStart = static_cast(firstDataOffset - firstOffsetInCache); - } - else - { - bufferCopyOffset = static_cast(firstOffsetInCache - firstDataOffset); - } - - if (lastOffsetInCache >= lastDataOffset) - { - copyOffsetEnd -= static_cast(lastOffsetInCache - lastDataOffset); - } - else if (bufferCopyOffset > 0) // the cache block is in the middle of the data, we can't use it (since we need to split buffer request into 2) - { - return 0; - } - - numCopied = copyOffsetEnd - copyOffsetStart; - memcpy(static_cast(buffer) + bufferCopyOffset, zlibData->m_decompressedCache + copyOffsetStart, static_cast(numCopied)); - - // adjust pointers and sizes - byteSize -= numCopied; - if (bufferCopyOffset == 0) - { - // copied in the start - buffer = reinterpret_cast(buffer) + numCopied; - offset += numCopied; - } - } - - return numCopied; + AZ_Assert(false, "We did not read enough data, we have only %d bytes left in the buffer and we need %d!", dataSize, sizeof(CompressorZLibHeader) + sizeof(ZLib::Header)); + return false; } - //========================================================================= - // FillFromCompressedCache - // [12/17/2012] - //========================================================================= - inline CompressorZLib::SizeType CompressorZLib::FillCompressedBuffer(CompressorStream* stream) + AcquireDataBuffer(); + + CompressorZLibHeader* hdr = reinterpret_cast(data); + AZStd::endian_swap(hdr->m_numSeekPoints); + dataSize -= sizeof(CompressorZLibHeader); + data += sizeof(CompressorZLibHeader); + + CompressorZLibData* zlibData = aznew CompressorZLibData; + zlibData->m_compressor = this; + zlibData->m_uncompressedSize = 0; + zlibData->m_zlibHeader = *reinterpret_cast(data); + dataSize -= sizeof(zlibData->m_zlibHeader); + data += sizeof(zlibData->m_zlibHeader); + zlibData->m_decompressNextOffset = sizeof(CompressorHeader) + sizeof(CompressorZLibHeader) + sizeof(ZLib::Header); // start after the headers + + AZ_Assert(hdr->m_numSeekPoints > 0, "We should have at least one seek point for the entire stream!"); + + // go the end of the file and read all sync points. + SizeType compressedFileEnd = stream->GetLength(); + if (compressedFileEnd == 0) { - CompressorZLibData* zlibData = static_cast(stream->GetCompressorData()); - SizeType dataFromBuffer = 0; - if (stream == m_lastReadStream) // if the buffer is filled with data from the current stream, try to reuse - { - if (zlibData->m_decompressNextOffset > m_lastReadStreamOffset) - { - SizeType offsetInCache = zlibData->m_decompressNextOffset - m_lastReadStreamOffset; - if (offsetInCache < m_lastReadStreamSize) // last check if there is data overlap - { - // copy the usable part at the start of the - SizeType toMove = m_lastReadStreamSize - offsetInCache; - memmove(m_compressedDataBuffer, &m_compressedDataBuffer[static_cast(offsetInCache)], static_cast(toMove)); - dataFromBuffer += toMove; - } - } - } - - SizeType toReadFromStream = m_compressedDataBufferSize - dataFromBuffer; - SizeType readOffset = zlibData->m_decompressNextOffset + dataFromBuffer; - if (readOffset + toReadFromStream > zlibData->m_decompressLastOffset) - { - // don't read pass the end - AZ_Assert(readOffset <= zlibData->m_decompressLastOffset, "Read offset should always be before the end of stream!"); - toReadFromStream = zlibData->m_decompressLastOffset - readOffset; - } - - SizeType numReadFromStream = 0; - if (toReadFromStream) // if we did not reuse the whole buffer, read some data from the stream - { - GenericStream* baseStream = stream->GetWrappedStream(); - numReadFromStream = baseStream->ReadAtOffset(toReadFromStream, &m_compressedDataBuffer[static_cast(dataFromBuffer)], readOffset); - } - - // update what's actually in the read data buffer. - m_lastReadStream = stream; - m_lastReadStreamOffset = zlibData->m_decompressNextOffset; - m_lastReadStreamSize = dataFromBuffer + numReadFromStream; - return m_lastReadStreamSize; + delete zlibData; + return false; } - /** - * Helper class to find the best seek point for a specific offset. - */ - struct CompareUpper + zlibData->m_seekPoints.resize(hdr->m_numSeekPoints); + SizeType dataToRead = sizeof(CompressorZLibSeekPoint) * static_cast(hdr->m_numSeekPoints); + SizeType seekPointOffset = compressedFileEnd - dataToRead; + AZ_Assert(seekPointOffset <= compressedFileEnd, "We have an invalid archive, this is impossible!"); + GenericStream* baseStream = stream->GetWrappedStream(); + if (baseStream->ReadAtOffset(dataToRead, zlibData->m_seekPoints.data(), seekPointOffset) != dataToRead) { - inline bool operator()(const AZ::u64& offset, const CompressorZLibSeekPoint& sp) const {return offset < sp.m_uncompressedOffset; } - }; - - //========================================================================= - // Read - // [12/13/2012] - //========================================================================= - CompressorZLib::SizeType CompressorZLib::Read(CompressorStream* stream, SizeType byteSize, SizeType offset, void* buffer) + delete zlibData; + return false; + } + for (size_t i = 0; i < zlibData->m_seekPoints.size(); ++i) { - AZ_Assert(stream->GetCompressorData(), "This stream doesn't have decompression enabled!"); - CompressorZLibData* zlibData = static_cast(stream->GetCompressorData()); - AZ_Assert(!zlibData->m_zlib.IsCompressorStarted(), "You can't read/decompress while writing a compressed stream %s!"); + AZStd::endian_swap(zlibData->m_seekPoints[i].m_compressedOffset); + AZStd::endian_swap(zlibData->m_seekPoints[i].m_uncompressedOffset); + } - // check if the request can be finished from the decompressed cache - SizeType numRead = FillFromDecompressCache(zlibData, buffer, byteSize, offset); - if (byteSize == 0) // are we done + if (m_decompressionCachePerStream) + { + zlibData->m_decompressedCache = reinterpret_cast(azmalloc(m_decompressionCachePerStream, m_CompressedDataBufferAlignment, AZ::SystemAllocator, "CompressorZLib")); + } + + zlibData->m_decompressLastOffset = seekPointOffset; // set the start address of the seek points as the last valid read address for the compressed stream. + + zlibData->m_zlib.StartDecompressor(&zlibData->m_zlibHeader); + + stream->SetCompressorData(zlibData); + + return true; + } + + //========================================================================= + // WriteHeaderAndData + // [12/13/2012] + //========================================================================= + bool CompressorZLib::WriteHeaderAndData(CompressorStream* stream) + { + if (!Compressor::WriteHeaderAndData(stream)) + { + return false; + } + + CompressorZLibData* compressorData = static_cast(stream->GetCompressorData()); + CompressorZLibHeader header; + header.m_numSeekPoints = static_cast(compressorData->m_seekPoints.size()); + AZStd::endian_swap(header.m_numSeekPoints); + GenericStream* baseStream = stream->GetWrappedStream(); + if (baseStream->WriteAtOffset(sizeof(header), &header, sizeof(CompressorHeader)) == sizeof(header)) + { + return true; + } + + return false; + } + + //========================================================================= + // FillFromDecompressCache + // [12/14/2012] + //========================================================================= + inline CompressorZLib::SizeType CompressorZLib::FillFromDecompressCache(CompressorZLibData* zlibData, void*& buffer, SizeType& byteSize, SizeType& offset) + { + SizeType firstOffsetInCache = zlibData->m_decompressedCacheOffset; + SizeType lastOffsetInCache = firstOffsetInCache + zlibData->m_decompressedCacheDataSize; + SizeType firstDataOffset = offset; + SizeType lastDataOffset = offset + byteSize; + SizeType numCopied = 0; + if (firstOffsetInCache < lastDataOffset && lastOffsetInCache > firstDataOffset) // check if there is data in the cache + { + size_t copyOffsetStart = 0; + size_t copyOffsetEnd = zlibData->m_decompressedCacheDataSize; + + size_t bufferCopyOffset = 0; + + if (firstOffsetInCache < firstDataOffset) { - return numRead; + copyOffsetStart = static_cast(firstDataOffset - firstOffsetInCache); + } + else + { + bufferCopyOffset = static_cast(firstOffsetInCache - firstDataOffset); } - // find the best seek point for current offset - CompressorZLibData::SeekPointArray::iterator it = AZStd::upper_bound(zlibData->m_seekPoints.begin(), zlibData->m_seekPoints.end(), offset, CompareUpper()); - AZ_Assert(it != zlibData->m_seekPoints.begin(), "This should be impossible, we should always have a valid seek point at 0 offset!"); - const CompressorZLibSeekPoint& bestSeekPoint = *(--it); // get the previous (so it includes the current offset) - - // if read is continuous continue with decompression - bool isJumpToSeekPoint = false; - SizeType lastOffsetInCache = zlibData->m_decompressedCacheOffset + zlibData->m_decompressedCacheDataSize; - if (bestSeekPoint.m_uncompressedOffset > lastOffsetInCache) // if the best seek point is forward, jump forward to it. + if (lastOffsetInCache >= lastDataOffset) { - isJumpToSeekPoint = true; + copyOffsetEnd -= static_cast(lastOffsetInCache - lastDataOffset); } - else if (offset < lastOffsetInCache) // if the seek point is in the past and the requested offset is not in the cache jump back to it. + else if (bufferCopyOffset > 0) // the cache block is in the middle of the data, we can't use it (since we need to split buffer request into 2) { - isJumpToSeekPoint = true; + return 0; } - if (isJumpToSeekPoint) - { - zlibData->m_decompressNextOffset = bestSeekPoint.m_compressedOffset; // set next read point - zlibData->m_decompressedCacheOffset = bestSeekPoint.m_uncompressedOffset; // set uncompressed offset - zlibData->m_decompressedCacheDataSize = 0; // invalidate the cache - zlibData->m_zlib.ResetDecompressor(&zlibData->m_zlibHeader); // reset decompressor and setup the header. - } + numCopied = copyOffsetEnd - copyOffsetStart; + memcpy(static_cast(buffer) + bufferCopyOffset, zlibData->m_decompressedCache + copyOffsetStart, static_cast(numCopied)); - // decompress and move forward until the request is done - while (byteSize > 0) + // adjust pointers and sizes + byteSize -= numCopied; + if (bufferCopyOffset == 0) { - // fill buffer with compressed data - SizeType compressedDataSize = FillCompressedBuffer(stream); - if (compressedDataSize == 0) + // copied in the start + buffer = reinterpret_cast(buffer) + numCopied; + offset += numCopied; + } + } + + return numCopied; + } + + //========================================================================= + // FillFromCompressedCache + // [12/17/2012] + //========================================================================= + inline CompressorZLib::SizeType CompressorZLib::FillCompressedBuffer(CompressorStream* stream) + { + CompressorZLibData* zlibData = static_cast(stream->GetCompressorData()); + SizeType dataFromBuffer = 0; + if (stream == m_lastReadStream) // if the buffer is filled with data from the current stream, try to reuse + { + if (zlibData->m_decompressNextOffset > m_lastReadStreamOffset) + { + SizeType offsetInCache = zlibData->m_decompressNextOffset - m_lastReadStreamOffset; + if (offsetInCache < m_lastReadStreamSize) // last check if there is data overlap { - return numRead; // we are done reading and obviously we did not managed to read all data + // copy the usable part at the start of the + SizeType toMove = m_lastReadStreamSize - offsetInCache; + memmove(m_compressedDataBuffer, &m_compressedDataBuffer[static_cast(offsetInCache)], static_cast(toMove)); + dataFromBuffer += toMove; } - unsigned int processedCompressedData = 0; - while (byteSize > 0 && processedCompressedData < compressedDataSize) // decompressed data either until we are done with the request (byteSize == 0) or we need to fill the compression buffer again. - { - // if we have data in the cache move to the next offset, we always move forward by default. - zlibData->m_decompressedCacheOffset += zlibData->m_decompressedCacheDataSize; - - // decompress in the cache buffer - u32 availDecompressedCacheSize = m_decompressionCachePerStream; // reset buffer size - unsigned int processed = zlibData->m_zlib.Decompress(&m_compressedDataBuffer[processedCompressedData], static_cast(compressedDataSize) - processedCompressedData, zlibData->m_decompressedCache, availDecompressedCacheSize); - zlibData->m_decompressedCacheDataSize = m_decompressionCachePerStream - availDecompressedCacheSize; - if (processed == 0) - { - break; // we processed everything we could, load more compressed data. - } - processedCompressedData += processed; - // fill what we can from the cache - numRead += FillFromDecompressCache(zlibData, buffer, byteSize, offset); - } - // update next read position the the compressed stream - zlibData->m_decompressNextOffset += processedCompressedData; } + } + + SizeType toReadFromStream = m_compressedDataBufferSize - dataFromBuffer; + SizeType readOffset = zlibData->m_decompressNextOffset + dataFromBuffer; + if (readOffset + toReadFromStream > zlibData->m_decompressLastOffset) + { + // don't read pass the end + AZ_Assert(readOffset <= zlibData->m_decompressLastOffset, "Read offset should always be before the end of stream!"); + toReadFromStream = zlibData->m_decompressLastOffset - readOffset; + } + + SizeType numReadFromStream = 0; + if (toReadFromStream) // if we did not reuse the whole buffer, read some data from the stream + { + GenericStream* baseStream = stream->GetWrappedStream(); + numReadFromStream = baseStream->ReadAtOffset(toReadFromStream, &m_compressedDataBuffer[static_cast(dataFromBuffer)], readOffset); + } + + // update what's actually in the read data buffer. + m_lastReadStream = stream; + m_lastReadStreamOffset = zlibData->m_decompressNextOffset; + m_lastReadStreamSize = dataFromBuffer + numReadFromStream; + return m_lastReadStreamSize; + } + + /** + * Helper class to find the best seek point for a specific offset. + */ + struct CompareUpper + { + inline bool operator()(const AZ::u64& offset, const CompressorZLibSeekPoint& sp) const {return offset < sp.m_uncompressedOffset; } + }; + + //========================================================================= + // Read + // [12/13/2012] + //========================================================================= + CompressorZLib::SizeType CompressorZLib::Read(CompressorStream* stream, SizeType byteSize, SizeType offset, void* buffer) + { + AZ_Assert(stream->GetCompressorData(), "This stream doesn't have decompression enabled!"); + CompressorZLibData* zlibData = static_cast(stream->GetCompressorData()); + AZ_Assert(!zlibData->m_zlib.IsCompressorStarted(), "You can't read/decompress while writing a compressed stream %s!"); + + // check if the request can be finished from the decompressed cache + SizeType numRead = FillFromDecompressCache(zlibData, buffer, byteSize, offset); + if (byteSize == 0) // are we done + { return numRead; } - //========================================================================= - // Write - // [12/13/2012] - //========================================================================= - CompressorZLib::SizeType CompressorZLib::Write(CompressorStream* stream, SizeType byteSize, const void* data, SizeType offset) + // find the best seek point for current offset + CompressorZLibData::SeekPointArray::iterator it = AZStd::upper_bound(zlibData->m_seekPoints.begin(), zlibData->m_seekPoints.end(), offset, CompareUpper()); + AZ_Assert(it != zlibData->m_seekPoints.begin(), "This should be impossible, we should always have a valid seek point at 0 offset!"); + const CompressorZLibSeekPoint& bestSeekPoint = *(--it); // get the previous (so it includes the current offset) + + // if read is continuous continue with decompression + bool isJumpToSeekPoint = false; + SizeType lastOffsetInCache = zlibData->m_decompressedCacheOffset + zlibData->m_decompressedCacheDataSize; + if (bestSeekPoint.m_uncompressedOffset > lastOffsetInCache) // if the best seek point is forward, jump forward to it. { - (void)offset; + isJumpToSeekPoint = true; + } + else if (offset < lastOffsetInCache) // if the seek point is in the past and the requested offset is not in the cache jump back to it. + { + isJumpToSeekPoint = true; + } - AZ_Assert(stream && stream->GetCompressorData(), "This stream doesn't have compression enabled! Call Stream::WriteCompressed after you create the file!"); - AZ_Assert(offset == SizeType(-1) || offset == stream->GetCurPos(), "We can write compressed data only at the end of the stream!"); + if (isJumpToSeekPoint) + { + zlibData->m_decompressNextOffset = bestSeekPoint.m_compressedOffset; // set next read point + zlibData->m_decompressedCacheOffset = bestSeekPoint.m_uncompressedOffset; // set uncompressed offset + zlibData->m_decompressedCacheDataSize = 0; // invalidate the cache + zlibData->m_zlib.ResetDecompressor(&zlibData->m_zlibHeader); // reset decompressor and setup the header. + } - m_lastReadStream = nullptr; // invalidate last read position, otherwise m_dataBuffer will be corrupted (as we are about to write in it). - - CompressorZLibData* zlibData = static_cast(stream->GetCompressorData()); - AZ_Assert(!zlibData->m_zlib.IsDecompressorStarted(), "You can't write while reading/decompressing a compressed stream!"); - - const u8* bytes = reinterpret_cast(data); - unsigned int dataToCompress = static_cast(byteSize); - while (dataToCompress != 0) + // decompress and move forward until the request is done + while (byteSize > 0) + { + // fill buffer with compressed data + SizeType compressedDataSize = FillCompressedBuffer(stream); + if (compressedDataSize == 0) { - unsigned int oldDataToCompress = dataToCompress; - unsigned int compressedSize = zlibData->m_zlib.Compress(bytes, dataToCompress, m_compressedDataBuffer, m_compressedDataBufferSize); - if (compressedSize) - { - GenericStream* baseStream = stream->GetWrappedStream(); - SizeType numWritten = baseStream->Write(compressedSize, m_compressedDataBuffer); - if (numWritten != compressedSize) - { - return numWritten; // error we could not write all data - } - } - bytes += oldDataToCompress - dataToCompress; + return numRead; // we are done reading and obviously we did not managed to read all data } - zlibData->m_uncompressedSize += byteSize; - - if (zlibData->m_autoSeekSize > 0) + unsigned int processedCompressedData = 0; + while (byteSize > 0 && processedCompressedData < compressedDataSize) // decompressed data either until we are done with the request (byteSize == 0) or we need to fill the compression buffer again. { - // insert a seek point if needed. - if (zlibData->m_seekPoints.empty()) + // if we have data in the cache move to the next offset, we always move forward by default. + zlibData->m_decompressedCacheOffset += zlibData->m_decompressedCacheDataSize; + + // decompress in the cache buffer + u32 availDecompressedCacheSize = m_decompressionCachePerStream; // reset buffer size + unsigned int processed = zlibData->m_zlib.Decompress(&m_compressedDataBuffer[processedCompressedData], static_cast(compressedDataSize) - processedCompressedData, zlibData->m_decompressedCache, availDecompressedCacheSize); + zlibData->m_decompressedCacheDataSize = m_decompressionCachePerStream - availDecompressedCacheSize; + if (processed == 0) { - if (zlibData->m_uncompressedSize >= zlibData->m_autoSeekSize) - { - WriteSeekPoint(stream); - } + break; // we processed everything we could, load more compressed data. } - else if ((zlibData->m_uncompressedSize - zlibData->m_seekPoints.back().m_uncompressedOffset) > zlibData->m_autoSeekSize) + processedCompressedData += processed; + // fill what we can from the cache + numRead += FillFromDecompressCache(zlibData, buffer, byteSize, offset); + } + // update next read position the the compressed stream + zlibData->m_decompressNextOffset += processedCompressedData; + } + return numRead; + } + + //========================================================================= + // Write + // [12/13/2012] + //========================================================================= + CompressorZLib::SizeType CompressorZLib::Write(CompressorStream* stream, SizeType byteSize, const void* data, SizeType offset) + { + (void)offset; + + AZ_Assert(stream && stream->GetCompressorData(), "This stream doesn't have compression enabled! Call Stream::WriteCompressed after you create the file!"); + AZ_Assert(offset == SizeType(-1) || offset == stream->GetCurPos(), "We can write compressed data only at the end of the stream!"); + + m_lastReadStream = nullptr; // invalidate last read position, otherwise m_dataBuffer will be corrupted (as we are about to write in it). + + CompressorZLibData* zlibData = static_cast(stream->GetCompressorData()); + AZ_Assert(!zlibData->m_zlib.IsDecompressorStarted(), "You can't write while reading/decompressing a compressed stream!"); + + const u8* bytes = reinterpret_cast(data); + unsigned int dataToCompress = static_cast(byteSize); + while (dataToCompress != 0) + { + unsigned int oldDataToCompress = dataToCompress; + unsigned int compressedSize = zlibData->m_zlib.Compress(bytes, dataToCompress, m_compressedDataBuffer, m_compressedDataBufferSize); + if (compressedSize) + { + GenericStream* baseStream = stream->GetWrappedStream(); + SizeType numWritten = baseStream->Write(compressedSize, m_compressedDataBuffer); + if (numWritten != compressedSize) + { + return numWritten; // error we could not write all data + } + } + bytes += oldDataToCompress - dataToCompress; + } + zlibData->m_uncompressedSize += byteSize; + + if (zlibData->m_autoSeekSize > 0) + { + // insert a seek point if needed. + if (zlibData->m_seekPoints.empty()) + { + if (zlibData->m_uncompressedSize >= zlibData->m_autoSeekSize) { WriteSeekPoint(stream); } } - return byteSize; + else if ((zlibData->m_uncompressedSize - zlibData->m_seekPoints.back().m_uncompressedOffset) > zlibData->m_autoSeekSize) + { + WriteSeekPoint(stream); + } } + return byteSize; + } - //========================================================================= - // WriteSeekPoint - // [12/13/2012] - //========================================================================= - bool CompressorZLib::WriteSeekPoint(CompressorStream* stream) + //========================================================================= + // WriteSeekPoint + // [12/13/2012] + //========================================================================= + bool CompressorZLib::WriteSeekPoint(CompressorStream* stream) + { + AZ_Assert(stream && stream->GetCompressorData(), "This stream doesn't have compression enabled! Call Stream::WriteCompressed after you create the file!"); + CompressorZLibData* zlibData = static_cast(stream->GetCompressorData()); + + m_lastReadStream = nullptr; // invalidate last read position, otherwise m_dataBuffer will be corrupted (as we are about to write in it). + + unsigned int compressedSize; + unsigned int dataToCompress = 0; + do { - AZ_Assert(stream && stream->GetCompressorData(), "This stream doesn't have compression enabled! Call Stream::WriteCompressed after you create the file!"); - CompressorZLibData* zlibData = static_cast(stream->GetCompressorData()); + compressedSize = zlibData->m_zlib.Compress(nullptr, dataToCompress, m_compressedDataBuffer, m_compressedDataBufferSize, ZLib::FT_FULL_FLUSH); + if (compressedSize) + { + GenericStream* baseStream = stream->GetWrappedStream(); + SizeType numWritten = baseStream->Write(compressedSize, m_compressedDataBuffer); + if (numWritten != compressedSize) + { + return false; // error we wrote less than than requested! + } + } + } while (dataToCompress != 0); + CompressorZLibSeekPoint sp; + sp.m_compressedOffset = stream->GetLength(); + sp.m_uncompressedOffset = zlibData->m_uncompressedSize; + zlibData->m_seekPoints.push_back(sp); + return true; + } + + //========================================================================= + // StartCompressor + // [12/13/2012] + //========================================================================= + bool CompressorZLib::StartCompressor(CompressorStream* stream, int compressionLevel, SizeType autoSeekDataSize) + { + AZ_Assert(stream && stream->GetCompressorData() == nullptr, "Stream has compressor already enabled!"); + + AcquireDataBuffer(); + + CompressorZLibData* zlibData = aznew CompressorZLibData; + zlibData->m_compressor = this; + zlibData->m_zlibHeader = 0; // not used for compression + zlibData->m_uncompressedSize = 0; + zlibData->m_autoSeekSize = autoSeekDataSize; + compressionLevel = AZ::GetClamp(compressionLevel, 1, 9); // remap to zlib levels + + zlibData->m_zlib.StartCompressor(compressionLevel); + + stream->SetCompressorData(zlibData); + + if (WriteHeaderAndData(stream)) + { + // add the first and always present seek point at the start of the compressed stream + CompressorZLibSeekPoint sp; + sp.m_compressedOffset = sizeof(CompressorHeader) + sizeof(CompressorZLibHeader) + sizeof(zlibData->m_zlibHeader); + sp.m_uncompressedOffset = 0; + zlibData->m_seekPoints.push_back(sp); + return true; + } + return false; + } + + //========================================================================= + // Close + // [12/13/2012] + //========================================================================= + bool CompressorZLib::Close(CompressorStream* stream) + { + AZ_Assert(stream->IsOpen(), "Stream is not open to be closed!"); + + CompressorZLibData* zlibData = static_cast(stream->GetCompressorData()); + GenericStream* baseStream = stream->GetWrappedStream(); + + bool result = true; + if (zlibData->m_zlib.IsCompressorStarted()) + { m_lastReadStream = nullptr; // invalidate last read position, otherwise m_dataBuffer will be corrupted (as we are about to write in it). + // flush all compressed data unsigned int compressedSize; unsigned int dataToCompress = 0; do { - compressedSize = zlibData->m_zlib.Compress(nullptr, dataToCompress, m_compressedDataBuffer, m_compressedDataBufferSize, ZLib::FT_FULL_FLUSH); + compressedSize = zlibData->m_zlib.Compress(nullptr, dataToCompress, m_compressedDataBuffer, m_compressedDataBufferSize, ZLib::FT_FINISH); if (compressedSize) { - GenericStream* baseStream = stream->GetWrappedStream(); - SizeType numWritten = baseStream->Write(compressedSize, m_compressedDataBuffer); - if (numWritten != compressedSize) - { - return false; // error we wrote less than than requested! - } + baseStream->Write(compressedSize, m_compressedDataBuffer); } } while (dataToCompress != 0); - CompressorZLibSeekPoint sp; - sp.m_compressedOffset = stream->GetLength(); - sp.m_uncompressedOffset = zlibData->m_uncompressedSize; - zlibData->m_seekPoints.push_back(sp); - return true; - } - - //========================================================================= - // StartCompressor - // [12/13/2012] - //========================================================================= - bool CompressorZLib::StartCompressor(CompressorStream* stream, int compressionLevel, SizeType autoSeekDataSize) - { - AZ_Assert(stream && stream->GetCompressorData() == nullptr, "Stream has compressor already enabled!"); - - AcquireDataBuffer(); - - CompressorZLibData* zlibData = aznew CompressorZLibData; - zlibData->m_compressor = this; - zlibData->m_zlibHeader = 0; // not used for compression - zlibData->m_uncompressedSize = 0; - zlibData->m_autoSeekSize = autoSeekDataSize; - compressionLevel = AZ::GetClamp(compressionLevel, 1, 9); // remap to zlib levels - - zlibData->m_zlib.StartCompressor(compressionLevel); - - stream->SetCompressorData(zlibData); - - if (WriteHeaderAndData(stream)) + result = WriteHeaderAndData(stream); + if (result) { - // add the first and always present seek point at the start of the compressed stream - CompressorZLibSeekPoint sp; - sp.m_compressedOffset = sizeof(CompressorHeader) + sizeof(CompressorZLibHeader) + sizeof(zlibData->m_zlibHeader); - sp.m_uncompressedOffset = 0; - zlibData->m_seekPoints.push_back(sp); - return true; - } - return false; - } - - //========================================================================= - // Close - // [12/13/2012] - //========================================================================= - bool CompressorZLib::Close(CompressorStream* stream) - { - AZ_Assert(stream->IsOpen(), "Stream is not open to be closed!"); - - CompressorZLibData* zlibData = static_cast(stream->GetCompressorData()); - GenericStream* baseStream = stream->GetWrappedStream(); - - bool result = true; - if (zlibData->m_zlib.IsCompressorStarted()) - { - m_lastReadStream = nullptr; // invalidate last read position, otherwise m_dataBuffer will be corrupted (as we are about to write in it). - - // flush all compressed data - unsigned int compressedSize; - unsigned int dataToCompress = 0; - do + // now write the seek points and the end of the file + for (size_t i = 0; i < zlibData->m_seekPoints.size(); ++i) { - compressedSize = zlibData->m_zlib.Compress(nullptr, dataToCompress, m_compressedDataBuffer, m_compressedDataBufferSize, ZLib::FT_FINISH); - if (compressedSize) - { - baseStream->Write(compressedSize, m_compressedDataBuffer); - } - } while (dataToCompress != 0); - - result = WriteHeaderAndData(stream); - if (result) - { - // now write the seek points and the end of the file - for (size_t i = 0; i < zlibData->m_seekPoints.size(); ++i) - { - AZStd::endian_swap(zlibData->m_seekPoints[i].m_compressedOffset); - AZStd::endian_swap(zlibData->m_seekPoints[i].m_uncompressedOffset); - } - SizeType dataToWrite = zlibData->m_seekPoints.size() * sizeof(CompressorZLibSeekPoint); - baseStream->Seek(0U, GenericStream::SeekMode::ST_SEEK_END); - result = (baseStream->Write(dataToWrite, zlibData->m_seekPoints.data()) == dataToWrite); + AZStd::endian_swap(zlibData->m_seekPoints[i].m_compressedOffset); + AZStd::endian_swap(zlibData->m_seekPoints[i].m_uncompressedOffset); } + SizeType dataToWrite = zlibData->m_seekPoints.size() * sizeof(CompressorZLibSeekPoint); + baseStream->Seek(0U, GenericStream::SeekMode::ST_SEEK_END); + result = (baseStream->Write(dataToWrite, zlibData->m_seekPoints.data()) == dataToWrite); } - else - { - if (m_lastReadStream == stream) - { - m_lastReadStream = nullptr; // invalidate the data in m_dataBuffer if it was from the current stream. - } - } - - // if we have decompressor cache delete it - if (zlibData->m_decompressedCache) - { - azfree(zlibData->m_decompressedCache, AZ::SystemAllocator, m_decompressionCachePerStream, m_CompressedDataBufferAlignment); - } - - ReleaseDataBuffer(); - - // last step reset strream compressor data. - stream->SetCompressorData(nullptr); - return result; } - - //========================================================================= - // AcquireDataBuffer - // [2/27/2013] - //========================================================================= - void CompressorZLib::AcquireDataBuffer() + else { - if (m_compressedDataBuffer == nullptr) + if (m_lastReadStream == stream) { - AZ_Assert(m_compressedDataBufferUseCount == 0, "Buffer usecount should be 0 if the buffer is NULL"); - m_compressedDataBuffer = reinterpret_cast(azmalloc(m_compressedDataBufferSize, m_CompressedDataBufferAlignment, AZ::SystemAllocator, "CompressorZLib")); - m_lastReadStream = nullptr; // reset the cache info in the m_dataBuffer + m_lastReadStream = nullptr; // invalidate the data in m_dataBuffer if it was from the current stream. } - ++m_compressedDataBufferUseCount; } - //========================================================================= - // ReleaseDataBuffer - // [2/27/2013] - //========================================================================= - void CompressorZLib::ReleaseDataBuffer() + // if we have decompressor cache delete it + if (zlibData->m_decompressedCache) { - --m_compressedDataBufferUseCount; - if (m_compressedDataBufferUseCount == 0) - { - AZ_Assert(m_compressedDataBuffer != nullptr, "Invalid data buffer! We should have a non null pointer!"); - azfree(m_compressedDataBuffer, AZ::SystemAllocator, m_compressedDataBufferSize, m_CompressedDataBufferAlignment); - m_compressedDataBuffer = nullptr; - m_lastReadStream = nullptr; // reset the cache info in the m_dataBuffer - } + azfree(zlibData->m_decompressedCache, AZ::SystemAllocator, m_decompressionCachePerStream, m_CompressedDataBufferAlignment); } - } // namespace IO -} // namespace AZ + + ReleaseDataBuffer(); + + // last step reset strream compressor data. + stream->SetCompressorData(nullptr); + return result; + } + + //========================================================================= + // AcquireDataBuffer + // [2/27/2013] + //========================================================================= + void CompressorZLib::AcquireDataBuffer() + { + if (m_compressedDataBuffer == nullptr) + { + AZ_Assert(m_compressedDataBufferUseCount == 0, "Buffer usecount should be 0 if the buffer is NULL"); + m_compressedDataBuffer = reinterpret_cast(azmalloc(m_compressedDataBufferSize, m_CompressedDataBufferAlignment, AZ::SystemAllocator, "CompressorZLib")); + m_lastReadStream = nullptr; // reset the cache info in the m_dataBuffer + } + ++m_compressedDataBufferUseCount; + } + + //========================================================================= + // ReleaseDataBuffer + // [2/27/2013] + //========================================================================= + void CompressorZLib::ReleaseDataBuffer() + { + --m_compressedDataBufferUseCount; + if (m_compressedDataBufferUseCount == 0) + { + AZ_Assert(m_compressedDataBuffer != nullptr, "Invalid data buffer! We should have a non null pointer!"); + azfree(m_compressedDataBuffer, AZ::SystemAllocator, m_compressedDataBufferSize, m_CompressedDataBufferAlignment); + m_compressedDataBuffer = nullptr; + m_lastReadStream = nullptr; // reset the cache info in the m_dataBuffer + } + } +} // namespace AZ::IO #endif // #if !defined(AZCORE_EXCLUDE_ZLIB) diff --git a/Code/Framework/AzCore/AzCore/IO/CompressorZStd.cpp b/Code/Framework/AzCore/AzCore/IO/CompressorZStd.cpp index b1631d2d22..91f380d73b 100644 --- a/Code/Framework/AzCore/AzCore/IO/CompressorZStd.cpp +++ b/Code/Framework/AzCore/AzCore/IO/CompressorZStd.cpp @@ -14,478 +14,475 @@ #include #include -namespace AZ +namespace AZ::IO { - namespace IO + CompressorZStd::CompressorZStd(unsigned int decompressionCachePerStream, unsigned int dataBufferSize) + : m_compressedDataBufferSize(dataBufferSize) + , m_decompressionCachePerStream(decompressionCachePerStream) + { - CompressorZStd::CompressorZStd(unsigned int decompressionCachePerStream, unsigned int dataBufferSize) - : m_compressedDataBufferSize(dataBufferSize) - , m_decompressionCachePerStream(decompressionCachePerStream) + AZ_Assert((dataBufferSize % (32 * 1024)) == 0, "Data buffer size %d must be multiple of 32 KB.", dataBufferSize); + AZ_Assert((decompressionCachePerStream % (32 * 1024)) == 0, "Decompress cache size %d must be multiple of 32 KB.", decompressionCachePerStream); + } + CompressorZStd::~CompressorZStd() + { + AZ_Warning("IO", m_compressedDataBufferUseCount == 0, "CompressorZStd has it's data buffer still referenced, it means that %d compressed streams have NOT closed. Freeing data...", m_compressedDataBufferUseCount); + while (m_compressedDataBufferUseCount) { - AZ_Assert((dataBufferSize % (32 * 1024)) == 0, "Data buffer size %d must be multiple of 32 KB.", dataBufferSize); - AZ_Assert((decompressionCachePerStream % (32 * 1024)) == 0, "Decompress cache size %d must be multiple of 32 KB.", decompressionCachePerStream); + ReleaseDataBuffer(); } + } - CompressorZStd::~CompressorZStd() + AZ::u32 CompressorZStd::TypeId() + { + return AZ_CRC("ZStd", 0x72fd505e); + } + + bool CompressorZStd::ReadHeaderAndData(CompressorStream* stream, AZ::u8* data, unsigned int dataSize) + { + if (stream->GetCompressorData() != nullptr) // we already have compressor data { - AZ_Warning("IO", m_compressedDataBufferUseCount == 0, "CompressorZStd has it's data buffer still referenced, it means that %d compressed streams have NOT closed. Freeing data...", m_compressedDataBufferUseCount); - while (m_compressedDataBufferUseCount) - { - ReleaseDataBuffer(); - } - } - - AZ::u32 CompressorZStd::TypeId() - { - return AZ_CRC("ZStd", 0x72fd505e); - } - - bool CompressorZStd::ReadHeaderAndData(CompressorStream* stream, AZ::u8* data, unsigned int dataSize) - { - if (stream->GetCompressorData() != nullptr) // we already have compressor data - { - return false; - } - - // Read the ZStd header should be after the default compression header... - // We should not be in this function otherwise. - if (dataSize < sizeof(CompressorZStdHeader) + sizeof(ZStd::Header)) - { - AZ_Error("CompressorZStd", false, "We did not read enough data, we have only %d bytes left in the buffer and we need %d.", dataSize, sizeof(CompressorZStdHeader) + sizeof(ZStd::Header)); - return false; - } - - AcquireDataBuffer(); - - CompressorZStdHeader* hdr = reinterpret_cast(data); - dataSize -= sizeof(CompressorZStdHeader); - data += sizeof(CompressorZStdHeader); - - AZStd::unique_ptr zstdData = AZStd::make_unique(); - zstdData->m_compressor = this; - zstdData->m_uncompressedSize = 0; - zstdData->m_zstdHeader = *reinterpret_cast(data); - dataSize -= sizeof(zstdData->m_zstdHeader); - data += sizeof(zstdData->m_zstdHeader); - zstdData->m_decompressNextOffset = sizeof(CompressorHeader) + sizeof(CompressorZStdHeader) + sizeof(ZStd::Header); // start after the headers - - AZ_Error("CompressorZStd", hdr->m_numSeekPoints > 0, "We should have at least one seek point for the entire stream."); - - // go the end of the file and read all sync points. - SizeType compressedFileEnd = stream->GetLength(); - if (compressedFileEnd == 0) - { - return false; - } - - zstdData->m_seekPoints.resize(hdr->m_numSeekPoints); - SizeType dataToRead = sizeof(CompressorZStdSeekPoint) * static_cast(hdr->m_numSeekPoints); - SizeType seekPointOffset = compressedFileEnd - dataToRead; - - if (seekPointOffset > compressedFileEnd) - { - AZ_Error("CompressorZStd", false, "We have an invalid archive, this is impossible."); - return false; - } - - GenericStream* baseStream = stream->GetWrappedStream(); - if (baseStream->ReadAtOffset(dataToRead, zstdData->m_seekPoints.data(), seekPointOffset) != dataToRead) - { - return false; - } - - if (m_decompressionCachePerStream) - { - zstdData->m_decompressedCache = reinterpret_cast(azmalloc(m_decompressionCachePerStream, m_CompressedDataBufferAlignment, AZ::SystemAllocator, "CompressorZStd")); - } - - zstdData->m_decompressLastOffset = seekPointOffset; // set the start address of the seek points as the last valid read address for the compressed stream. - - zstdData->m_zstd.StartDecompressor(); - - stream->SetCompressorData(zstdData.release()); - - return true; - } - - bool CompressorZStd::WriteHeaderAndData(CompressorStream* stream) - { - if (!Compressor::WriteHeaderAndData(stream)) - { - return false; - } - - CompressorZStdData* compressorData = static_cast(stream->GetCompressorData()); - CompressorZStdHeader header; - header.m_numSeekPoints = aznumeric_caster(compressorData->m_seekPoints.size()); - GenericStream* baseStream = stream->GetWrappedStream(); - if (baseStream->WriteAtOffset(sizeof(header), &header, sizeof(CompressorHeader)) == sizeof(header)) - { - return true; - } - return false; } - inline CompressorZStd::SizeType CompressorZStd::FillFromDecompressCache(CompressorZStdData* zstdData, void*& buffer, SizeType& byteSize, SizeType& offset) + // Read the ZStd header should be after the default compression header... + // We should not be in this function otherwise. + if (dataSize < sizeof(CompressorZStdHeader) + sizeof(ZStd::Header)) { - SizeType firstOffsetInCache = zstdData->m_decompressedCacheOffset; - SizeType lastOffsetInCache = firstOffsetInCache + zstdData->m_decompressedCacheDataSize; - SizeType firstDataOffset = offset; - SizeType lastDataOffset = offset + byteSize; - SizeType numCopied = 0; - if (firstOffsetInCache < lastDataOffset && lastOffsetInCache >= firstDataOffset) // check if there is data in the cache - { - size_t copyOffsetStart = 0; - size_t copyOffsetEnd = zstdData->m_decompressedCacheDataSize; - - size_t bufferCopyOffset = 0; - - if (firstOffsetInCache < firstDataOffset) - { - copyOffsetStart = aznumeric_caster(firstDataOffset - firstOffsetInCache); - } - else - { - bufferCopyOffset = aznumeric_caster(firstOffsetInCache - firstDataOffset); - } - - if (lastOffsetInCache >= lastDataOffset) - { - copyOffsetEnd -= static_cast(lastOffsetInCache - lastDataOffset); - } - else if (bufferCopyOffset > 0) // the cache block is in the middle of the data, we can't use it (since we need to split buffer request into 2) - { - return 0; - } - - numCopied = copyOffsetEnd - copyOffsetStart; - memcpy(static_cast(buffer) + bufferCopyOffset, zstdData->m_decompressedCache + copyOffsetStart, static_cast(numCopied)); - - // adjust pointers and sizes - byteSize -= numCopied; - if (bufferCopyOffset == 0) - { - // copied in the start - buffer = reinterpret_cast(buffer) + numCopied; - offset += numCopied; - } - } - - return numCopied; + AZ_Error("CompressorZStd", false, "We did not read enough data, we have only %d bytes left in the buffer and we need %d.", dataSize, sizeof(CompressorZStdHeader) + sizeof(ZStd::Header)); + return false; } - inline CompressorZStd::SizeType CompressorZStd::FillCompressedBuffer(CompressorStream* stream) + AcquireDataBuffer(); + + CompressorZStdHeader* hdr = reinterpret_cast(data); + dataSize -= sizeof(CompressorZStdHeader); + data += sizeof(CompressorZStdHeader); + + AZStd::unique_ptr zstdData = AZStd::make_unique(); + zstdData->m_compressor = this; + zstdData->m_uncompressedSize = 0; + zstdData->m_zstdHeader = *reinterpret_cast(data); + dataSize -= sizeof(zstdData->m_zstdHeader); + data += sizeof(zstdData->m_zstdHeader); + zstdData->m_decompressNextOffset = sizeof(CompressorHeader) + sizeof(CompressorZStdHeader) + sizeof(ZStd::Header); // start after the headers + + AZ_Error("CompressorZStd", hdr->m_numSeekPoints > 0, "We should have at least one seek point for the entire stream."); + + // go the end of the file and read all sync points. + SizeType compressedFileEnd = stream->GetLength(); + if (compressedFileEnd == 0) { - CompressorZStdData* zstdData = static_cast(stream->GetCompressorData()); - SizeType dataFromBuffer = 0; - if (stream == m_lastReadStream) // if the buffer is filled with data from the current stream, try to reuse - { - if (zstdData->m_decompressNextOffset > m_lastReadStreamOffset) - { - SizeType offsetInCache = zstdData->m_decompressNextOffset - m_lastReadStreamOffset; - if (offsetInCache < m_lastReadStreamSize) // last check if there is data overlap - { - // copy the usable part at the start of the buffer - SizeType toMove = m_lastReadStreamSize - offsetInCache; - memcpy(m_compressedDataBuffer, &m_compressedDataBuffer[static_cast(offsetInCache)], static_cast(toMove)); - dataFromBuffer += toMove; - } - } - } - - SizeType toReadFromStream = m_compressedDataBufferSize - dataFromBuffer; - SizeType readOffset = zstdData->m_decompressNextOffset + dataFromBuffer; - if (readOffset + toReadFromStream > zstdData->m_decompressLastOffset) - { - // don't read past the end - AZ_Assert(readOffset <= zstdData->m_decompressLastOffset, "Read offset should always be before the end of stream."); - toReadFromStream = zstdData->m_decompressLastOffset - readOffset; - } - - SizeType numReadFromStream = 0; - if (toReadFromStream) // if we did not reuse the whole buffer, read some data from the stream - { - GenericStream* baseStream = stream->GetWrappedStream(); - numReadFromStream = baseStream->ReadAtOffset(toReadFromStream, &m_compressedDataBuffer[static_cast(dataFromBuffer)], readOffset); - } - - // update what's actually in the read data buffer. - m_lastReadStream = stream; - m_lastReadStreamOffset = zstdData->m_decompressNextOffset; - m_lastReadStreamSize = dataFromBuffer + numReadFromStream; - return m_lastReadStreamSize; + return false; } - struct ZStdCompareUpper + zstdData->m_seekPoints.resize(hdr->m_numSeekPoints); + SizeType dataToRead = sizeof(CompressorZStdSeekPoint) * static_cast(hdr->m_numSeekPoints); + SizeType seekPointOffset = compressedFileEnd - dataToRead; + + if (seekPointOffset > compressedFileEnd) { - bool operator()(const AZ::u64& offset, const CompressorZStdSeekPoint& sp) const - { - return offset < sp.m_uncompressedOffset; - } - }; + AZ_Error("CompressorZStd", false, "We have an invalid archive, this is impossible."); + return false; + } - CompressorZStd::SizeType CompressorZStd::Read(CompressorStream* stream, SizeType byteSize, SizeType offset, void* buffer) + GenericStream* baseStream = stream->GetWrappedStream(); + if (baseStream->ReadAtOffset(dataToRead, zstdData->m_seekPoints.data(), seekPointOffset) != dataToRead) { - AZ_Assert(stream->GetCompressorData(), "This stream doesn't have decompression enabled."); - CompressorZStdData* zstdData = static_cast(stream->GetCompressorData()); - AZ_Assert(!zstdData->m_zstd.IsCompressorStarted(), "You can't read/decompress while writing a compressed stream %s."); + return false; + } - // check if the request can be finished from the decompressed cache - SizeType numRead = FillFromDecompressCache(zstdData, buffer, byteSize, offset); - if (byteSize == 0) // are we done + if (m_decompressionCachePerStream) + { + zstdData->m_decompressedCache = reinterpret_cast(azmalloc(m_decompressionCachePerStream, m_CompressedDataBufferAlignment, AZ::SystemAllocator, "CompressorZStd")); + } + + zstdData->m_decompressLastOffset = seekPointOffset; // set the start address of the seek points as the last valid read address for the compressed stream. + + zstdData->m_zstd.StartDecompressor(); + + stream->SetCompressorData(zstdData.release()); + + return true; + } + + bool CompressorZStd::WriteHeaderAndData(CompressorStream* stream) + { + if (!Compressor::WriteHeaderAndData(stream)) + { + return false; + } + + CompressorZStdData* compressorData = static_cast(stream->GetCompressorData()); + CompressorZStdHeader header; + header.m_numSeekPoints = aznumeric_caster(compressorData->m_seekPoints.size()); + GenericStream* baseStream = stream->GetWrappedStream(); + if (baseStream->WriteAtOffset(sizeof(header), &header, sizeof(CompressorHeader)) == sizeof(header)) + { + return true; + } + + return false; + } + + inline CompressorZStd::SizeType CompressorZStd::FillFromDecompressCache(CompressorZStdData* zstdData, void*& buffer, SizeType& byteSize, SizeType& offset) + { + SizeType firstOffsetInCache = zstdData->m_decompressedCacheOffset; + SizeType lastOffsetInCache = firstOffsetInCache + zstdData->m_decompressedCacheDataSize; + SizeType firstDataOffset = offset; + SizeType lastDataOffset = offset + byteSize; + SizeType numCopied = 0; + if (firstOffsetInCache < lastDataOffset && lastOffsetInCache >= firstDataOffset) // check if there is data in the cache + { + size_t copyOffsetStart = 0; + size_t copyOffsetEnd = zstdData->m_decompressedCacheDataSize; + + size_t bufferCopyOffset = 0; + + if (firstOffsetInCache < firstDataOffset) { - return numRead; + copyOffsetStart = aznumeric_caster(firstDataOffset - firstOffsetInCache); + } + else + { + bufferCopyOffset = aznumeric_caster(firstOffsetInCache - firstDataOffset); } - // find the best seek point for current offset - CompressorZStdData::SeekPointArray::iterator it = AZStd::upper_bound(zstdData->m_seekPoints.begin(), zstdData->m_seekPoints.end(), offset, ZStdCompareUpper()); - AZ_Assert(it != zstdData->m_seekPoints.begin(), "This should be impossible, we should always have a valid seek point at 0 offset."); - const CompressorZStdSeekPoint& bestSeekPoint = *(--it); // get the previous (so it includes the current offset) - - // if read is continuous continue with decompression - bool isJumpToSeekPoint = false; - SizeType lastOffsetInCache = zstdData->m_decompressedCacheOffset + zstdData->m_decompressedCacheDataSize; - if (bestSeekPoint.m_uncompressedOffset > lastOffsetInCache) // if the best seek point is forward, jump forward to it. + if (lastOffsetInCache >= lastDataOffset) { - isJumpToSeekPoint = true; + copyOffsetEnd -= static_cast(lastOffsetInCache - lastDataOffset); } - else if (offset < lastOffsetInCache) // if the seek point is in the past and the requested offset is not in the cache jump back to it. + else if (bufferCopyOffset > 0) // the cache block is in the middle of the data, we can't use it (since we need to split buffer request into 2) { - isJumpToSeekPoint = true; + return 0; } - if (isJumpToSeekPoint) - { - zstdData->m_decompressNextOffset = bestSeekPoint.m_compressedOffset; // set next read point - zstdData->m_decompressedCacheOffset = bestSeekPoint.m_uncompressedOffset; // set uncompressed offset - zstdData->m_decompressedCacheDataSize = 0; // invalidate the cache - zstdData->m_zstd.ResetDecompressor(&zstdData->m_zstdHeader); // reset decompressor and setup the header. - } + numCopied = copyOffsetEnd - copyOffsetStart; + memcpy(static_cast(buffer) + bufferCopyOffset, zstdData->m_decompressedCache + copyOffsetStart, static_cast(numCopied)); - // decompress and move forward until the request is done - while (byteSize > 0) + // adjust pointers and sizes + byteSize -= numCopied; + if (bufferCopyOffset == 0) { - // fill buffer with compressed data - SizeType compressedDataSize = FillCompressedBuffer(stream); - if (compressedDataSize == 0) + // copied in the start + buffer = reinterpret_cast(buffer) + numCopied; + offset += numCopied; + } + } + + return numCopied; + } + + inline CompressorZStd::SizeType CompressorZStd::FillCompressedBuffer(CompressorStream* stream) + { + CompressorZStdData* zstdData = static_cast(stream->GetCompressorData()); + SizeType dataFromBuffer = 0; + if (stream == m_lastReadStream) // if the buffer is filled with data from the current stream, try to reuse + { + if (zstdData->m_decompressNextOffset > m_lastReadStreamOffset) + { + SizeType offsetInCache = zstdData->m_decompressNextOffset - m_lastReadStreamOffset; + if (offsetInCache < m_lastReadStreamSize) // last check if there is data overlap { - return numRead; // we are done reading and obviously we did not managed to read all data + // copy the usable part at the start of the buffer + SizeType toMove = m_lastReadStreamSize - offsetInCache; + memcpy(m_compressedDataBuffer, &m_compressedDataBuffer[static_cast(offsetInCache)], static_cast(toMove)); + dataFromBuffer += toMove; } - unsigned int processedCompressedData = 0; - while (byteSize > 0 && processedCompressedData < compressedDataSize) // decompressed data either until we are done with the request (byteSize == 0) or we need to fill the compression buffer again. - { - // if we have data in the cache move to the next offset, we always move forward by default. - zstdData->m_decompressedCacheOffset += zstdData->m_decompressedCacheDataSize; - - // decompress in the cache buffer - u32 availDecompressedCacheSize = m_decompressionCachePerStream; // reset buffer size - size_t nextBlockSize; - unsigned int processed = zstdData->m_zstd.Decompress(&m_compressedDataBuffer[processedCompressedData], - static_cast(compressedDataSize) - processedCompressedData, - zstdData->m_decompressedCache, - availDecompressedCacheSize, - &nextBlockSize); - zstdData->m_decompressedCacheDataSize = m_decompressionCachePerStream - availDecompressedCacheSize; - if (processed == 0) - { - break; // we processed everything we could, load more compressed data. - } - processedCompressedData += processed; - // fill what we can from the cache - numRead += FillFromDecompressCache(zstdData, buffer, byteSize, offset); - } - // update next read position the the compressed stream - zstdData->m_decompressNextOffset += processedCompressedData; } + } + + SizeType toReadFromStream = m_compressedDataBufferSize - dataFromBuffer; + SizeType readOffset = zstdData->m_decompressNextOffset + dataFromBuffer; + if (readOffset + toReadFromStream > zstdData->m_decompressLastOffset) + { + // don't read past the end + AZ_Assert(readOffset <= zstdData->m_decompressLastOffset, "Read offset should always be before the end of stream."); + toReadFromStream = zstdData->m_decompressLastOffset - readOffset; + } + + SizeType numReadFromStream = 0; + if (toReadFromStream) // if we did not reuse the whole buffer, read some data from the stream + { + GenericStream* baseStream = stream->GetWrappedStream(); + numReadFromStream = baseStream->ReadAtOffset(toReadFromStream, &m_compressedDataBuffer[static_cast(dataFromBuffer)], readOffset); + } + + // update what's actually in the read data buffer. + m_lastReadStream = stream; + m_lastReadStreamOffset = zstdData->m_decompressNextOffset; + m_lastReadStreamSize = dataFromBuffer + numReadFromStream; + return m_lastReadStreamSize; + } + + struct ZStdCompareUpper + { + bool operator()(const AZ::u64& offset, const CompressorZStdSeekPoint& sp) const + { + return offset < sp.m_uncompressedOffset; + } + }; + + CompressorZStd::SizeType CompressorZStd::Read(CompressorStream* stream, SizeType byteSize, SizeType offset, void* buffer) + { + AZ_Assert(stream->GetCompressorData(), "This stream doesn't have decompression enabled."); + CompressorZStdData* zstdData = static_cast(stream->GetCompressorData()); + AZ_Assert(!zstdData->m_zstd.IsCompressorStarted(), "You can't read/decompress while writing a compressed stream %s."); + + // check if the request can be finished from the decompressed cache + SizeType numRead = FillFromDecompressCache(zstdData, buffer, byteSize, offset); + if (byteSize == 0) // are we done + { return numRead; } - CompressorZStd::SizeType CompressorZStd::Write(CompressorStream* stream, SizeType byteSize, const void* data, SizeType offset) + // find the best seek point for current offset + CompressorZStdData::SeekPointArray::iterator it = AZStd::upper_bound(zstdData->m_seekPoints.begin(), zstdData->m_seekPoints.end(), offset, ZStdCompareUpper()); + AZ_Assert(it != zstdData->m_seekPoints.begin(), "This should be impossible, we should always have a valid seek point at 0 offset."); + const CompressorZStdSeekPoint& bestSeekPoint = *(--it); // get the previous (so it includes the current offset) + + // if read is continuous continue with decompression + bool isJumpToSeekPoint = false; + SizeType lastOffsetInCache = zstdData->m_decompressedCacheOffset + zstdData->m_decompressedCacheDataSize; + if (bestSeekPoint.m_uncompressedOffset > lastOffsetInCache) // if the best seek point is forward, jump forward to it. { - AZ_UNUSED(offset); + isJumpToSeekPoint = true; + } + else if (offset < lastOffsetInCache) // if the seek point is in the past and the requested offset is not in the cache jump back to it. + { + isJumpToSeekPoint = true; + } - AZ_Assert(stream && stream->GetCompressorData(), "This stream doesn't have compression enabled. Call Stream::WriteCompressed after you create the file."); - AZ_Assert(offset == SizeType(-1) || offset == stream->GetCurPos(), "We can write compressed data only at the end of the stream."); + if (isJumpToSeekPoint) + { + zstdData->m_decompressNextOffset = bestSeekPoint.m_compressedOffset; // set next read point + zstdData->m_decompressedCacheOffset = bestSeekPoint.m_uncompressedOffset; // set uncompressed offset + zstdData->m_decompressedCacheDataSize = 0; // invalidate the cache + zstdData->m_zstd.ResetDecompressor(&zstdData->m_zstdHeader); // reset decompressor and setup the header. + } - m_lastReadStream = nullptr; // invalidate last read position, otherwise m_dataBuffer will be corrupted (as we are about to write in it). - - CompressorZStdData* zstdData = static_cast(stream->GetCompressorData()); - AZ_Assert(!zstdData->m_zstd.IsDecompressorStarted(), "You can't write while reading/decompressing a compressed stream."); - - const u8* bytes = reinterpret_cast(data); - unsigned int dataToCompress = aznumeric_caster(byteSize); - while (dataToCompress != 0) + // decompress and move forward until the request is done + while (byteSize > 0) + { + // fill buffer with compressed data + SizeType compressedDataSize = FillCompressedBuffer(stream); + if (compressedDataSize == 0) { - unsigned int oldDataToCompress = dataToCompress; - unsigned int compressedSize = zstdData->m_zstd.Compress(bytes, dataToCompress, m_compressedDataBuffer, m_compressedDataBufferSize); - if (compressedSize) - { - GenericStream* baseStream = stream->GetWrappedStream(); - SizeType numWritten = baseStream->Write(compressedSize, m_compressedDataBuffer); - if (numWritten != compressedSize) - { - return numWritten; // error we could not write all data - } - } - bytes += oldDataToCompress - dataToCompress; + return numRead; // we are done reading and obviously we did not managed to read all data } - zstdData->m_uncompressedSize += byteSize; - - if (zstdData->m_autoSeekSize > 0) + unsigned int processedCompressedData = 0; + while (byteSize > 0 && processedCompressedData < compressedDataSize) // decompressed data either until we are done with the request (byteSize == 0) or we need to fill the compression buffer again. { - // insert a seek point if needed. - if (zstdData->m_seekPoints.empty()) + // if we have data in the cache move to the next offset, we always move forward by default. + zstdData->m_decompressedCacheOffset += zstdData->m_decompressedCacheDataSize; + + // decompress in the cache buffer + u32 availDecompressedCacheSize = m_decompressionCachePerStream; // reset buffer size + size_t nextBlockSize; + unsigned int processed = zstdData->m_zstd.Decompress(&m_compressedDataBuffer[processedCompressedData], + static_cast(compressedDataSize) - processedCompressedData, + zstdData->m_decompressedCache, + availDecompressedCacheSize, + &nextBlockSize); + zstdData->m_decompressedCacheDataSize = m_decompressionCachePerStream - availDecompressedCacheSize; + if (processed == 0) { - if (zstdData->m_uncompressedSize >= zstdData->m_autoSeekSize) - { - WriteSeekPoint(stream); - } + break; // we processed everything we could, load more compressed data. } - else if ((zstdData->m_uncompressedSize - zstdData->m_seekPoints.back().m_uncompressedOffset) > zstdData->m_autoSeekSize) + processedCompressedData += processed; + // fill what we can from the cache + numRead += FillFromDecompressCache(zstdData, buffer, byteSize, offset); + } + // update next read position the the compressed stream + zstdData->m_decompressNextOffset += processedCompressedData; + } + return numRead; + } + + CompressorZStd::SizeType CompressorZStd::Write(CompressorStream* stream, SizeType byteSize, const void* data, SizeType offset) + { + AZ_UNUSED(offset); + + AZ_Assert(stream && stream->GetCompressorData(), "This stream doesn't have compression enabled. Call Stream::WriteCompressed after you create the file."); + AZ_Assert(offset == SizeType(-1) || offset == stream->GetCurPos(), "We can write compressed data only at the end of the stream."); + + m_lastReadStream = nullptr; // invalidate last read position, otherwise m_dataBuffer will be corrupted (as we are about to write in it). + + CompressorZStdData* zstdData = static_cast(stream->GetCompressorData()); + AZ_Assert(!zstdData->m_zstd.IsDecompressorStarted(), "You can't write while reading/decompressing a compressed stream."); + + const u8* bytes = reinterpret_cast(data); + unsigned int dataToCompress = aznumeric_caster(byteSize); + while (dataToCompress != 0) + { + unsigned int oldDataToCompress = dataToCompress; + unsigned int compressedSize = zstdData->m_zstd.Compress(bytes, dataToCompress, m_compressedDataBuffer, m_compressedDataBufferSize); + if (compressedSize) + { + GenericStream* baseStream = stream->GetWrappedStream(); + SizeType numWritten = baseStream->Write(compressedSize, m_compressedDataBuffer); + if (numWritten != compressedSize) + { + return numWritten; // error we could not write all data + } + } + bytes += oldDataToCompress - dataToCompress; + } + zstdData->m_uncompressedSize += byteSize; + + if (zstdData->m_autoSeekSize > 0) + { + // insert a seek point if needed. + if (zstdData->m_seekPoints.empty()) + { + if (zstdData->m_uncompressedSize >= zstdData->m_autoSeekSize) { WriteSeekPoint(stream); } } - return byteSize; + else if ((zstdData->m_uncompressedSize - zstdData->m_seekPoints.back().m_uncompressedOffset) > zstdData->m_autoSeekSize) + { + WriteSeekPoint(stream); + } } + return byteSize; + } - bool CompressorZStd::WriteSeekPoint(CompressorStream* stream) + bool CompressorZStd::WriteSeekPoint(CompressorStream* stream) + { + AZ_Assert(stream && stream->GetCompressorData(), "This stream doesn't have compression enabled. Call Stream::WriteCompressed after you create the file."); + CompressorZStdData* zstdData = static_cast(stream->GetCompressorData()); + + m_lastReadStream = nullptr; // invalidate last read position, otherwise m_dataBuffer will be corrupted (as we are about to write in it). + + unsigned int compressedSize; + unsigned int dataToCompress = 0; + do { - AZ_Assert(stream && stream->GetCompressorData(), "This stream doesn't have compression enabled. Call Stream::WriteCompressed after you create the file."); - CompressorZStdData* zstdData = static_cast(stream->GetCompressorData()); + compressedSize = zstdData->m_zstd.Compress(nullptr, dataToCompress, m_compressedDataBuffer, m_compressedDataBufferSize, ZStd::FT_FULL_FLUSH); + if (compressedSize) + { + GenericStream* baseStream = stream->GetWrappedStream(); + SizeType numWritten = baseStream->Write(compressedSize, m_compressedDataBuffer); + if (numWritten != compressedSize) + { + return false; // error we wrote less than than requested! + } + } + } while (dataToCompress != 0); + CompressorZStdSeekPoint sp; + sp.m_compressedOffset = stream->GetLength(); + sp.m_uncompressedOffset = zstdData->m_uncompressedSize; + zstdData->m_seekPoints.push_back(sp); + return true; + } + + bool CompressorZStd::StartCompressor(CompressorStream* stream, int compressionLevel, SizeType autoSeekDataSize) + { + AZ_Assert(stream && !stream->GetCompressorData(), "Stream has compressor already enabled."); + + AcquireDataBuffer(); + + CompressorZStdData* zstdData = aznew CompressorZStdData; + zstdData->m_compressor = this; + zstdData->m_zstdHeader = 0; // not used for compression + zstdData->m_uncompressedSize = 0; + zstdData->m_autoSeekSize = autoSeekDataSize; + compressionLevel = AZ::GetClamp(compressionLevel, 1, 9); // remap to zlib levels + + zstdData->m_zstd.StartCompressor(compressionLevel); + + stream->SetCompressorData(zstdData); + + if (WriteHeaderAndData(stream)) + { + // add the first and always present seek point at the start of the compressed stream + CompressorZStdSeekPoint sp; + sp.m_compressedOffset = sizeof(CompressorHeader) + sizeof(CompressorZStdHeader) + sizeof(zstdData->m_zstdHeader); + sp.m_uncompressedOffset = 0; + zstdData->m_seekPoints.push_back(sp); + return true; + } + return false; + } + + bool CompressorZStd::Close(CompressorStream* stream) + { + AZ_Assert(stream->IsOpen(), "Stream is not open to be closed."); + + CompressorZStdData* zstdData = static_cast(stream->GetCompressorData()); + GenericStream* baseStream = stream->GetWrappedStream(); + + bool result = true; + if (zstdData->m_zstd.IsCompressorStarted()) + { m_lastReadStream = nullptr; // invalidate last read position, otherwise m_dataBuffer will be corrupted (as we are about to write in it). + // flush all compressed data unsigned int compressedSize; unsigned int dataToCompress = 0; do { - compressedSize = zstdData->m_zstd.Compress(nullptr, dataToCompress, m_compressedDataBuffer, m_compressedDataBufferSize, ZStd::FT_FULL_FLUSH); + compressedSize = zstdData->m_zstd.Compress(nullptr, dataToCompress, m_compressedDataBuffer, m_compressedDataBufferSize, ZStd::FT_FINISH); if (compressedSize) { - GenericStream* baseStream = stream->GetWrappedStream(); - SizeType numWritten = baseStream->Write(compressedSize, m_compressedDataBuffer); - if (numWritten != compressedSize) - { - return false; // error we wrote less than than requested! - } + baseStream->Write(compressedSize, m_compressedDataBuffer); } } while (dataToCompress != 0); - CompressorZStdSeekPoint sp; - sp.m_compressedOffset = stream->GetLength(); - sp.m_uncompressedOffset = zstdData->m_uncompressedSize; - zstdData->m_seekPoints.push_back(sp); - return true; + result = WriteHeaderAndData(stream); + if (result) + { + SizeType dataToWrite = zstdData->m_seekPoints.size() * sizeof(CompressorZStdSeekPoint); + baseStream->Seek(0U, GenericStream::SeekMode::ST_SEEK_END); + result = (baseStream->Write(dataToWrite, zstdData->m_seekPoints.data()) == dataToWrite); + } } - - bool CompressorZStd::StartCompressor(CompressorStream* stream, int compressionLevel, SizeType autoSeekDataSize) + else { - AZ_Assert(stream && !stream->GetCompressorData(), "Stream has compressor already enabled."); - - AcquireDataBuffer(); - - CompressorZStdData* zstdData = aznew CompressorZStdData; - zstdData->m_compressor = this; - zstdData->m_zstdHeader = 0; // not used for compression - zstdData->m_uncompressedSize = 0; - zstdData->m_autoSeekSize = autoSeekDataSize; - compressionLevel = AZ::GetClamp(compressionLevel, 1, 9); // remap to zlib levels - - zstdData->m_zstd.StartCompressor(compressionLevel); - - stream->SetCompressorData(zstdData); - - if (WriteHeaderAndData(stream)) + if (m_lastReadStream == stream) { - // add the first and always present seek point at the start of the compressed stream - CompressorZStdSeekPoint sp; - sp.m_compressedOffset = sizeof(CompressorHeader) + sizeof(CompressorZStdHeader) + sizeof(zstdData->m_zstdHeader); - sp.m_uncompressedOffset = 0; - zstdData->m_seekPoints.push_back(sp); - return true; + m_lastReadStream = nullptr; // invalidate the data in m_dataBuffer if it was from the current stream. } - return false; } - bool CompressorZStd::Close(CompressorStream* stream) + // if we have decompressor cache delete it + if (zstdData->m_decompressedCache) { - AZ_Assert(stream->IsOpen(), "Stream is not open to be closed."); - - CompressorZStdData* zstdData = static_cast(stream->GetCompressorData()); - GenericStream* baseStream = stream->GetWrappedStream(); - - bool result = true; - if (zstdData->m_zstd.IsCompressorStarted()) - { - m_lastReadStream = nullptr; // invalidate last read position, otherwise m_dataBuffer will be corrupted (as we are about to write in it). - - // flush all compressed data - unsigned int compressedSize; - unsigned int dataToCompress = 0; - do - { - compressedSize = zstdData->m_zstd.Compress(nullptr, dataToCompress, m_compressedDataBuffer, m_compressedDataBufferSize, ZStd::FT_FINISH); - if (compressedSize) - { - baseStream->Write(compressedSize, m_compressedDataBuffer); - } - } while (dataToCompress != 0); - - result = WriteHeaderAndData(stream); - if (result) - { - SizeType dataToWrite = zstdData->m_seekPoints.size() * sizeof(CompressorZStdSeekPoint); - baseStream->Seek(0U, GenericStream::SeekMode::ST_SEEK_END); - result = (baseStream->Write(dataToWrite, zstdData->m_seekPoints.data()) == dataToWrite); - } - } - else - { - if (m_lastReadStream == stream) - { - m_lastReadStream = nullptr; // invalidate the data in m_dataBuffer if it was from the current stream. - } - } - - // if we have decompressor cache delete it - if (zstdData->m_decompressedCache) - { - azfree(zstdData->m_decompressedCache, AZ::SystemAllocator, m_decompressionCachePerStream, m_CompressedDataBufferAlignment); - } - - ReleaseDataBuffer(); - - // last step reset strream compressor data. - stream->SetCompressorData(nullptr); - return result; + azfree(zstdData->m_decompressedCache, AZ::SystemAllocator, m_decompressionCachePerStream, m_CompressedDataBufferAlignment); } - void CompressorZStd::AcquireDataBuffer() + ReleaseDataBuffer(); + + // last step reset strream compressor data. + stream->SetCompressorData(nullptr); + return result; + } + + void CompressorZStd::AcquireDataBuffer() + { + if (m_compressedDataBuffer == nullptr) { - if (m_compressedDataBuffer == nullptr) - { - AZ_Assert(m_compressedDataBufferUseCount == 0, "Buffer usecount should be 0 if the buffer is NULL"); - m_compressedDataBuffer = reinterpret_cast(azmalloc(m_compressedDataBufferSize, m_CompressedDataBufferAlignment, AZ::SystemAllocator, "CompressorZStd")); - m_lastReadStream = nullptr; // reset the cache info in the m_dataBuffer - } - ++m_compressedDataBufferUseCount; + AZ_Assert(m_compressedDataBufferUseCount == 0, "Buffer usecount should be 0 if the buffer is NULL"); + m_compressedDataBuffer = reinterpret_cast(azmalloc(m_compressedDataBufferSize, m_CompressedDataBufferAlignment, AZ::SystemAllocator, "CompressorZStd")); + m_lastReadStream = nullptr; // reset the cache info in the m_dataBuffer } + ++m_compressedDataBufferUseCount; + } - void CompressorZStd::ReleaseDataBuffer() + void CompressorZStd::ReleaseDataBuffer() + { + --m_compressedDataBufferUseCount; + if (m_compressedDataBufferUseCount == 0) { - --m_compressedDataBufferUseCount; - if (m_compressedDataBufferUseCount == 0) - { - AZ_Assert(m_compressedDataBuffer, "Invalid data buffer. We should have a non null pointer."); - azfree(m_compressedDataBuffer, AZ::SystemAllocator, m_compressedDataBufferSize, m_CompressedDataBufferAlignment); - m_compressedDataBuffer = nullptr; - m_lastReadStream = nullptr; // reset the cache info in the m_dataBuffer - } + AZ_Assert(m_compressedDataBuffer, "Invalid data buffer. We should have a non null pointer."); + azfree(m_compressedDataBuffer, AZ::SystemAllocator, m_compressedDataBufferSize, m_CompressedDataBufferAlignment); + m_compressedDataBuffer = nullptr; + m_lastReadStream = nullptr; // reset the cache info in the m_dataBuffer } - } // namespace IO -} // namespace AZ + } +} // namespace AZ::IO #endif // #if !defined(AZCORE_EXCLUDE_ZSTD) diff --git a/Code/Framework/AzCore/AzCore/IO/FileIO.cpp b/Code/Framework/AzCore/AzCore/IO/FileIO.cpp index 837aca8d84..3522187131 100644 --- a/Code/Framework/AzCore/AzCore/IO/FileIO.cpp +++ b/Code/Framework/AzCore/AzCore/IO/FileIO.cpp @@ -21,485 +21,482 @@ # define SEEK_END 2 /* Set file pointer to EOF plus "offset" */ #endif -namespace AZ +namespace AZ::IO { - namespace IO + static EnvironmentVariable g_fileIOInstance; + static EnvironmentVariable g_directFileIOInstance; + static const char* s_EngineFileIOName = "EngineFileIO"; + static const char* s_DirectFileIOName = "DirectFileIO"; + + FileIOBase* FileIOBase::GetInstance() { - static EnvironmentVariable g_fileIOInstance; - static EnvironmentVariable g_directFileIOInstance; - static const char* s_EngineFileIOName = "EngineFileIO"; - static const char* s_DirectFileIOName = "DirectFileIO"; - - FileIOBase* FileIOBase::GetInstance() + if (!g_fileIOInstance) { - if (!g_fileIOInstance) - { - g_fileIOInstance = Environment::FindVariable(s_EngineFileIOName); - } - - return g_fileIOInstance ? (*g_fileIOInstance) : nullptr; + g_fileIOInstance = Environment::FindVariable(s_EngineFileIOName); } - void FileIOBase::SetInstance(FileIOBase* instance) + return g_fileIOInstance ? (*g_fileIOInstance) : nullptr; + } + + void FileIOBase::SetInstance(FileIOBase* instance) + { + if (!g_fileIOInstance) { - if (!g_fileIOInstance) - { - g_fileIOInstance = Environment::CreateVariable(s_EngineFileIOName); - (*g_fileIOInstance) = nullptr; - } - - // at this point we're guaranteed to have g_fileIOInstance. Its value might be null. - - if ((instance) && (g_fileIOInstance) && (*g_fileIOInstance)) - { - AZ_Error("FileIO", false, "FileIOBase::SetInstance was called without first destroying the old instance and setting it to nullptr"); - } - - (*g_fileIOInstance) = instance; + g_fileIOInstance = Environment::CreateVariable(s_EngineFileIOName); + (*g_fileIOInstance) = nullptr; } - FileIOBase* FileIOBase::GetDirectInstance() + // at this point we're guaranteed to have g_fileIOInstance. Its value might be null. + + if ((instance) && (g_fileIOInstance) && (*g_fileIOInstance)) { - if (!g_directFileIOInstance) - { - g_directFileIOInstance = Environment::FindVariable(s_DirectFileIOName); - } - - // for backwards compatibilty, return the regular instance if this is not attached - if (!g_directFileIOInstance) - { - return GetInstance(); - } - - return g_directFileIOInstance ? (*g_directFileIOInstance) : nullptr; + AZ_Error("FileIO", false, "FileIOBase::SetInstance was called without first destroying the old instance and setting it to nullptr"); } - void FileIOBase::SetDirectInstance(FileIOBase* instance) + (*g_fileIOInstance) = instance; + } + + FileIOBase* FileIOBase::GetDirectInstance() + { + if (!g_directFileIOInstance) { - if (!g_directFileIOInstance) - { - g_directFileIOInstance = Environment::CreateVariable(s_DirectFileIOName); - (*g_directFileIOInstance) = nullptr; - } - - // at this point we're guaranteed to have g_directFileIOInstance. Its value might be null. - - if ((instance) && (g_directFileIOInstance) && (*g_directFileIOInstance)) - { - AZ_Error("FileIO", false, "FileIOBase::SetDirectInstance was called without first destroying the old instance and setting it to nullptr"); - } - - (*g_directFileIOInstance) = instance; + g_directFileIOInstance = Environment::FindVariable(s_DirectFileIOName); } - AZStd::optional FileIOBase::ConvertToAlias(const AZ::IO::PathView& path) const + // for backwards compatibilty, return the regular instance if this is not attached + if (!g_directFileIOInstance) { - AZ::IO::FixedMaxPath convertedPath; - if (ConvertToAlias(convertedPath, path)) - { - return convertedPath; - } - - return AZStd::nullopt; + return GetInstance(); } - AZStd::optional FileIOBase::ResolvePath(const AZ::IO::PathView& path) const - { - AZ::IO::FixedMaxPath resolvedPath; - if (ResolvePath(resolvedPath, path)) - { - return resolvedPath; - } + return g_directFileIOInstance ? (*g_directFileIOInstance) : nullptr; + } - return AZStd::nullopt; + void FileIOBase::SetDirectInstance(FileIOBase* instance) + { + if (!g_directFileIOInstance) + { + g_directFileIOInstance = Environment::CreateVariable(s_DirectFileIOName); + (*g_directFileIOInstance) = nullptr; } - SeekType GetSeekTypeFromFSeekMode(int mode) - { - switch (mode) - { - case SEEK_SET: - return SeekType::SeekFromStart; - case SEEK_CUR: - return SeekType::SeekFromCurrent; - case SEEK_END: - return SeekType::SeekFromEnd; - } + // at this point we're guaranteed to have g_directFileIOInstance. Its value might be null. - // Must have some default, hitting here means some random int mode + if ((instance) && (g_directFileIOInstance) && (*g_directFileIOInstance)) + { + AZ_Error("FileIO", false, "FileIOBase::SetDirectInstance was called without first destroying the old instance and setting it to nullptr"); + } + + (*g_directFileIOInstance) = instance; + } + + AZStd::optional FileIOBase::ConvertToAlias(const AZ::IO::PathView& path) const + { + AZ::IO::FixedMaxPath convertedPath; + if (ConvertToAlias(convertedPath, path)) + { + return convertedPath; + } + + return AZStd::nullopt; + } + + AZStd::optional FileIOBase::ResolvePath(const AZ::IO::PathView& path) const + { + AZ::IO::FixedMaxPath resolvedPath; + if (ResolvePath(resolvedPath, path)) + { + return resolvedPath; + } + + return AZStd::nullopt; + } + + SeekType GetSeekTypeFromFSeekMode(int mode) + { + switch (mode) + { + case SEEK_SET: return SeekType::SeekFromStart; + case SEEK_CUR: + return SeekType::SeekFromCurrent; + case SEEK_END: + return SeekType::SeekFromEnd; } - int GetFSeekModeFromSeekType(SeekType type) - { - switch (type) - { - case SeekType::SeekFromStart: - return SEEK_SET; - case SeekType::SeekFromCurrent: - return SEEK_CUR; - case SeekType::SeekFromEnd: - return SEEK_END; - } + // Must have some default, hitting here means some random int mode + return SeekType::SeekFromStart; + } + int GetFSeekModeFromSeekType(SeekType type) + { + switch (type) + { + case SeekType::SeekFromStart: return SEEK_SET; + case SeekType::SeekFromCurrent: + return SEEK_CUR; + case SeekType::SeekFromEnd: + return SEEK_END; } - void UpdateOpenModeForReading(OpenMode& openMode) + return SEEK_SET; + } + + void UpdateOpenModeForReading(OpenMode& openMode) + { + if (AnyFlag(openMode & OpenMode::ModeRead)) { - if (AnyFlag(openMode & OpenMode::ModeRead)) + if (AnyFlag(openMode & OpenMode::ModeText)) { - if (AnyFlag(openMode & OpenMode::ModeText)) - { - OpenMode extraModes = openMode & (OpenMode::ModeUpdate | OpenMode::ModeAppend); - openMode = OpenMode::ModeRead | OpenMode::ModeBinary | extraModes; - } - else if (!AnyFlag(openMode & OpenMode::ModeBinary)) - { - // if you haven't supplied any flag, supply binary - openMode = openMode | OpenMode::ModeBinary; - } + OpenMode extraModes = openMode & (OpenMode::ModeUpdate | OpenMode::ModeAppend); + openMode = OpenMode::ModeRead | OpenMode::ModeBinary | extraModes; + } + else if (!AnyFlag(openMode & OpenMode::ModeBinary)) + { + // if you haven't supplied any flag, supply binary + openMode = openMode | OpenMode::ModeBinary; } } + } - OpenMode GetOpenModeFromStringMode(const char* mode) + OpenMode GetOpenModeFromStringMode(const char* mode) + { + OpenMode openMode = OpenMode::Invalid; + + if (strstr(mode, "w")) { - OpenMode openMode = OpenMode::Invalid; - - if (strstr(mode, "w")) - { - openMode |= OpenMode::ModeWrite; - } - - if (strstr(mode, "r")) - { - openMode |= OpenMode::ModeRead; - } - - if (strstr(mode, "a")) - { - openMode |= OpenMode::ModeAppend; - } - - if (strstr(mode, "b")) - { - openMode |= OpenMode::ModeBinary; - } - - if (strstr(mode, "t")) - { - openMode |= OpenMode::ModeText; - } - - if (strstr(mode, "+")) - { - openMode |= OpenMode::ModeUpdate; - } - - UpdateOpenModeForReading(openMode); - - return openMode; + openMode |= OpenMode::ModeWrite; } - const char* GetStringModeFromOpenMode(OpenMode mode) + if (strstr(mode, "r")) { - UpdateOpenModeForReading(mode); - // Append is highest priority, followed by write and then read - // APPEND - if (AnyFlag(mode & OpenMode::ModeAppend)) + openMode |= OpenMode::ModeRead; + } + + if (strstr(mode, "a")) + { + openMode |= OpenMode::ModeAppend; + } + + if (strstr(mode, "b")) + { + openMode |= OpenMode::ModeBinary; + } + + if (strstr(mode, "t")) + { + openMode |= OpenMode::ModeText; + } + + if (strstr(mode, "+")) + { + openMode |= OpenMode::ModeUpdate; + } + + UpdateOpenModeForReading(openMode); + + return openMode; + } + + const char* GetStringModeFromOpenMode(OpenMode mode) + { + UpdateOpenModeForReading(mode); + // Append is highest priority, followed by write and then read + // APPEND + if (AnyFlag(mode & OpenMode::ModeAppend)) + { + if (AnyFlag(mode & OpenMode::ModeUpdate)) { - if (AnyFlag(mode & OpenMode::ModeUpdate)) - { - if (AnyFlag(mode & OpenMode::ModeBinary)) - { - return "a+b"; - } - if (AnyFlag(mode & OpenMode::ModeText)) - { - return "a+t"; - } - return "a+"; - } if (AnyFlag(mode & OpenMode::ModeBinary)) { - return "ab"; + return "a+b"; } if (AnyFlag(mode & OpenMode::ModeText)) { - return "at"; + return "a+t"; } - return "a"; + return "a+"; } - - // WRITE - if (AnyFlag(mode & OpenMode::ModeWrite)) + if (AnyFlag(mode & OpenMode::ModeBinary)) + { + return "ab"; + } + if (AnyFlag(mode & OpenMode::ModeText)) + { + return "at"; + } + return "a"; + } + + // WRITE + if (AnyFlag(mode & OpenMode::ModeWrite)) + { + if (AnyFlag(mode & OpenMode::ModeUpdate)) { - if (AnyFlag(mode & OpenMode::ModeUpdate)) - { - if (AnyFlag(mode & OpenMode::ModeBinary)) - { - return "w+b"; - } - if (AnyFlag(mode & OpenMode::ModeText)) - { - return "w+t"; - } - return "w+"; - } if (AnyFlag(mode & OpenMode::ModeBinary)) { - return "wb"; + return "w+b"; } if (AnyFlag(mode & OpenMode::ModeText)) { - return "wt"; + return "w+t"; } - return "w"; + return "w+"; } - - // READ - if (AnyFlag(mode & OpenMode::ModeRead)) + if (AnyFlag(mode & OpenMode::ModeBinary)) + { + return "wb"; + } + if (AnyFlag(mode & OpenMode::ModeText)) + { + return "wt"; + } + return "w"; + } + + // READ + if (AnyFlag(mode & OpenMode::ModeRead)) + { + if (AnyFlag(mode & OpenMode::ModeUpdate)) { - if (AnyFlag(mode & OpenMode::ModeUpdate)) - { - if (AnyFlag(mode & OpenMode::ModeBinary)) - { - return "r+b"; - } - if (AnyFlag(mode & OpenMode::ModeText)) - { - return "r+t"; - } - return "r+"; - } if (AnyFlag(mode & OpenMode::ModeBinary)) { - return "rb"; + return "r+b"; } if (AnyFlag(mode & OpenMode::ModeText)) { - return "rt"; + return "r+t"; } - return "r"; + return "r+"; } - - // Bad open mode passed in - AZ_Error("FileIO", false, "A bad open mode was sent to GetStringModeFromOpenMode()"); - return ""; + if (AnyFlag(mode & OpenMode::ModeBinary)) + { + return "rb"; + } + if (AnyFlag(mode & OpenMode::ModeText)) + { + return "rt"; + } + return "r"; } - bool NameMatchesFilter(const char* name, const char* filter) + // Bad open mode passed in + AZ_Error("FileIO", false, "A bad open mode was sent to GetStringModeFromOpenMode()"); + return ""; + } + + bool NameMatchesFilter(const char* name, const char* filter) + { + return AZStd::wildcard_match(filter, name); + } + + FileIOStream::FileIOStream() + : m_handle(InvalidHandle) + , m_mode(OpenMode::Invalid) + , m_ownsHandle(true) + { + + } + + FileIOStream::FileIOStream(HandleType fileHandle, AZ::IO::OpenMode mode, bool ownsHandle) + : m_handle(fileHandle) + , m_mode(mode) + , m_ownsHandle(ownsHandle) + { + + FileIOBase* fileIO = FileIOBase::GetInstance(); + AZ_Assert(fileIO, "FileIO is not initialized."); + AZStd::array resolvedPath{ {0} }; + fileIO->GetFilename(m_handle, resolvedPath.data(), resolvedPath.size() - 1); + m_filename = resolvedPath.data(); + } + + FileIOStream::FileIOStream(const char* path, AZ::IO::OpenMode mode, bool errorOnFailure) + : m_handle(InvalidHandle) + , m_mode(mode) + , m_errorOnFailure(errorOnFailure) + { + Open(path, mode); + } + + FileIOStream::~FileIOStream() + { + if (m_ownsHandle) { - return AZStd::wildcard_match(filter, name); + Close(); } + } - FileIOStream::FileIOStream() - : m_handle(InvalidHandle) - , m_mode(OpenMode::Invalid) - , m_ownsHandle(true) + bool FileIOStream::Open(const char* path, OpenMode mode) + { + AZ_Assert(FileIOBase::GetInstance(), "FileIO is not initialized."); + FileIOBase* fileIO = FileIOBase::GetInstance(); + + Close(); + + const Result result = fileIO->Open(path, mode, m_handle); + m_ownsHandle = IsOpen(); + m_mode = mode; + + if (IsOpen()) { - - } - - FileIOStream::FileIOStream(HandleType fileHandle, AZ::IO::OpenMode mode, bool ownsHandle) - : m_handle(fileHandle) - , m_mode(mode) - , m_ownsHandle(ownsHandle) - { - - FileIOBase* fileIO = FileIOBase::GetInstance(); - AZ_Assert(fileIO, "FileIO is not initialized."); + // Not using supplied path parameter as it may be unresolved AZStd::array resolvedPath{ {0} }; fileIO->GetFilename(m_handle, resolvedPath.data(), resolvedPath.size() - 1); m_filename = resolvedPath.data(); } - - FileIOStream::FileIOStream(const char* path, AZ::IO::OpenMode mode, bool errorOnFailure) - : m_handle(InvalidHandle) - , m_mode(mode) - , m_errorOnFailure(errorOnFailure) + else { - Open(path, mode); + // remember the file name so you can try again with ReOpen + m_filename = path; } - FileIOStream::~FileIOStream() - { - if (m_ownsHandle) - { - Close(); - } - } + AZ_PROFILE_INTERVAL_START_COLORED(AzCore, &m_filename, 0xff0000ff, "FileIO: %s", m_filename.c_str()); + return result; + } - bool FileIOStream::Open(const char* path, OpenMode mode) + bool FileIOStream::ReOpen() + { + Close(); + return (m_mode != OpenMode::Invalid) ? Open(m_filename.data(), m_mode) : false; + } + + void FileIOStream::Close() + { + if (m_handle != InvalidHandle) { AZ_Assert(FileIOBase::GetInstance(), "FileIO is not initialized."); - FileIOBase* fileIO = FileIOBase::GetInstance(); - Close(); - - const Result result = fileIO->Open(path, mode, m_handle); - m_ownsHandle = IsOpen(); - m_mode = mode; - - if (IsOpen()) - { - // Not using supplied path parameter as it may be unresolved - AZStd::array resolvedPath{ {0} }; - fileIO->GetFilename(m_handle, resolvedPath.data(), resolvedPath.size() - 1); - m_filename = resolvedPath.data(); - } - else - { - // remember the file name so you can try again with ReOpen - m_filename = path; - } - - AZ_PROFILE_INTERVAL_START_COLORED(AzCore, &m_filename, 0xff0000ff, "FileIO: %s", m_filename.c_str()); - return result; + FileIOBase::GetInstance()->Close(m_handle); + m_handle = InvalidHandle; + m_ownsHandle = false; + AZ_PROFILE_INTERVAL_END(AzCore, &m_filename); } + } - bool FileIOStream::ReOpen() + bool FileIOStream::IsOpen() const + { + return (m_handle != InvalidHandle); + } + + /*! + \brief Retrieves underlying FileIO Handle from file stream + \return HandleType + */ + HandleType FileIOStream::GetHandle() const + { + return m_handle; + } + + /*! + \brief Retrieves filename + \return const char* + */ + const char* FileIOStream::GetFilename() const + { + return m_filename.data(); + } + + /*! + \brief Retrieves OpenMode flags used to open this file + \return OpenMode + */ + AZ::IO::OpenMode FileIOStream::GetModeFlags() const + { + return m_mode; + } + + bool FileIOStream::CanSeek() const + { + return true; + } + + bool FileIOStream::CanRead() const + { + return (m_mode & (OpenMode::ModeRead | OpenMode::ModeUpdate)) != OpenMode::Invalid; + } + + bool FileIOStream::CanWrite() const + { + return (m_mode & (OpenMode::ModeWrite | OpenMode::ModeAppend | OpenMode::ModeUpdate)) != OpenMode::Invalid; + } + + void FileIOStream::Seek(OffsetType bytes, SeekMode mode) + { + AZ_Assert(FileIOBase::GetInstance(), "FileIO is not initialized."); + AZ_Assert(IsOpen(), "Cannot seek on a FileIOStream that is not open."); + + SeekType seekType = SeekType::SeekFromCurrent; + switch (mode) { - Close(); - return (m_mode != OpenMode::Invalid) ? Open(m_filename.data(), m_mode) : false; + case GenericStream::ST_SEEK_BEGIN: + seekType = SeekType::SeekFromStart; + break; + case GenericStream::ST_SEEK_CUR: + seekType = SeekType::SeekFromCurrent; + break; + case GenericStream::ST_SEEK_END: + seekType = SeekType::SeekFromEnd; + break; + default: + seekType = SeekType::SeekFromCurrent; + break; } - void FileIOStream::Close() + const Result result = FileIOBase::GetInstance()->Seek(m_handle, static_cast(bytes), seekType); + (void)result; + AZ_Error("FileIOStream", result.GetResultCode() == ResultCode::Success, "Seek failed."); + } + + SizeType FileIOStream::Read(SizeType bytes, void* oBuffer) + { + AZ_Assert(FileIOBase::GetInstance(), "FileIO is not initialized."); + AZ_Assert(IsOpen(), "Cannot read from a FileIOStream that is not open."); + + AZ::u64 bytesRead = 0; + const Result result = FileIOBase::GetInstance()->Read(m_handle, oBuffer, bytes, m_errorOnFailure, &bytesRead); + (void)result; + AZ_Error("FileIOStream", result.GetResultCode() == ResultCode::Success, "Read failed in file %s.", m_filename.empty() ? "NULL" : m_filename.c_str()); + return static_cast(bytesRead); + } + + SizeType FileIOStream::Write(SizeType bytes, const void* iBuffer) + { + AZ_Assert(FileIOBase::GetInstance(), "FileIO is not initialized."); + AZ_Assert(IsOpen(), "Cannot write to a FileIOStream that is not open."); + + AZ::u64 bytesWritten = 0; + const Result result = FileIOBase::GetInstance()->Write(m_handle, iBuffer, bytes, &bytesWritten); + (void)result; + AZ_Error("FileIOStream", result.GetResultCode() == ResultCode::Success, "Write failed."); + return static_cast(bytesWritten); + } + + SizeType FileIOStream::GetCurPos() const + { + AZ_Assert(FileIOBase::GetInstance(), "FileIO is not initialized."); + AZ_Assert(IsOpen(), "Cannot use a FileIOStream that is not open."); + + AZ::u64 currentPosition = 0; + const Result result = FileIOBase::GetInstance()->Tell(m_handle, currentPosition); + (void)result; + AZ_Error("FileIOStream", result.GetResultCode() == ResultCode::Success, "GetCurPos failed."); + return static_cast(currentPosition); + } + + SizeType FileIOStream::GetLength() const + { + AZ_Assert(FileIOBase::GetInstance(), "FileIO is not initialized."); + AZ_Assert(IsOpen(), "Cannot use a FileIOStream that is not open."); + + SizeType fileLengthBytes = 0; + if (!FileIOBase::GetInstance()->Size(m_handle, fileLengthBytes)) { - if (m_handle != InvalidHandle) - { - AZ_Assert(FileIOBase::GetInstance(), "FileIO is not initialized."); - - FileIOBase::GetInstance()->Close(m_handle); - m_handle = InvalidHandle; - m_ownsHandle = false; - AZ_PROFILE_INTERVAL_END(AzCore, &m_filename); - } + AZ_Error("FileIOStream", false, "GetLength failed."); } - bool FileIOStream::IsOpen() const - { - return (m_handle != InvalidHandle); - } + return fileLengthBytes; + } - /*! - \brief Retrieves underlying FileIO Handle from file stream - \return HandleType - */ - HandleType FileIOStream::GetHandle() const - { - return m_handle; - } - - /*! - \brief Retrieves filename - \return const char* - */ - const char* FileIOStream::GetFilename() const - { - return m_filename.data(); - } - - /*! - \brief Retrieves OpenMode flags used to open this file - \return OpenMode - */ - AZ::IO::OpenMode FileIOStream::GetModeFlags() const - { - return m_mode; - } - - bool FileIOStream::CanSeek() const - { - return true; - } - - bool FileIOStream::CanRead() const - { - return (m_mode & (OpenMode::ModeRead | OpenMode::ModeUpdate)) != OpenMode::Invalid; - } - - bool FileIOStream::CanWrite() const - { - return (m_mode & (OpenMode::ModeWrite | OpenMode::ModeAppend | OpenMode::ModeUpdate)) != OpenMode::Invalid; - } - - void FileIOStream::Seek(OffsetType bytes, SeekMode mode) - { - AZ_Assert(FileIOBase::GetInstance(), "FileIO is not initialized."); - AZ_Assert(IsOpen(), "Cannot seek on a FileIOStream that is not open."); - - SeekType seekType = SeekType::SeekFromCurrent; - switch (mode) - { - case GenericStream::ST_SEEK_BEGIN: - seekType = SeekType::SeekFromStart; - break; - case GenericStream::ST_SEEK_CUR: - seekType = SeekType::SeekFromCurrent; - break; - case GenericStream::ST_SEEK_END: - seekType = SeekType::SeekFromEnd; - break; - default: - seekType = SeekType::SeekFromCurrent; - break; - } - - const Result result = FileIOBase::GetInstance()->Seek(m_handle, static_cast(bytes), seekType); - (void)result; - AZ_Error("FileIOStream", result.GetResultCode() == ResultCode::Success, "Seek failed."); - } - - SizeType FileIOStream::Read(SizeType bytes, void* oBuffer) - { - AZ_Assert(FileIOBase::GetInstance(), "FileIO is not initialized."); - AZ_Assert(IsOpen(), "Cannot read from a FileIOStream that is not open."); - - AZ::u64 bytesRead = 0; - const Result result = FileIOBase::GetInstance()->Read(m_handle, oBuffer, bytes, m_errorOnFailure, &bytesRead); - (void)result; - AZ_Error("FileIOStream", result.GetResultCode() == ResultCode::Success, "Read failed in file %s.", m_filename.empty() ? "NULL" : m_filename.c_str()); - return static_cast(bytesRead); - } - - SizeType FileIOStream::Write(SizeType bytes, const void* iBuffer) - { - AZ_Assert(FileIOBase::GetInstance(), "FileIO is not initialized."); - AZ_Assert(IsOpen(), "Cannot write to a FileIOStream that is not open."); - - AZ::u64 bytesWritten = 0; - const Result result = FileIOBase::GetInstance()->Write(m_handle, iBuffer, bytes, &bytesWritten); - (void)result; - AZ_Error("FileIOStream", result.GetResultCode() == ResultCode::Success, "Write failed."); - return static_cast(bytesWritten); - } - - SizeType FileIOStream::GetCurPos() const - { - AZ_Assert(FileIOBase::GetInstance(), "FileIO is not initialized."); - AZ_Assert(IsOpen(), "Cannot use a FileIOStream that is not open."); - - AZ::u64 currentPosition = 0; - const Result result = FileIOBase::GetInstance()->Tell(m_handle, currentPosition); - (void)result; - AZ_Error("FileIOStream", result.GetResultCode() == ResultCode::Success, "GetCurPos failed."); - return static_cast(currentPosition); - } - - SizeType FileIOStream::GetLength() const - { - AZ_Assert(FileIOBase::GetInstance(), "FileIO is not initialized."); - AZ_Assert(IsOpen(), "Cannot use a FileIOStream that is not open."); - - SizeType fileLengthBytes = 0; - if (!FileIOBase::GetInstance()->Size(m_handle, fileLengthBytes)) - { - AZ_Error("FileIOStream", false, "GetLength failed."); - } - - return fileLengthBytes; - } - - } // namespace IO -} // namespace AZ +} // namespace AZ::IO diff --git a/Code/Framework/AzCore/AzCore/IO/IOUtils.cpp b/Code/Framework/AzCore/AzCore/IO/IOUtils.cpp index 1e3a31f8ab..222a36ef70 100644 --- a/Code/Framework/AzCore/AzCore/IO/IOUtils.cpp +++ b/Code/Framework/AzCore/AzCore/IO/IOUtils.cpp @@ -10,69 +10,64 @@ #include #include /// this_thread sleep_for. -namespace AZ +namespace AZ::IO { - namespace IO - { - int TranslateOpenModeToSystemFileMode(const char* path, OpenMode mode) + int TranslateOpenModeToSystemFileMode(const char* path, OpenMode mode) + { + int systemFileMode = 0; + bool read = AnyFlag(mode & OpenMode::ModeRead) || AnyFlag(mode & OpenMode::ModeUpdate); + bool write = AnyFlag(mode & OpenMode::ModeWrite) || AnyFlag(mode & OpenMode::ModeUpdate) || AnyFlag(mode & OpenMode::ModeAppend); + if (write) { - int systemFileMode = 0; - bool read = AnyFlag(mode & OpenMode::ModeRead) || AnyFlag(mode & OpenMode::ModeUpdate); - bool write = AnyFlag(mode & OpenMode::ModeWrite) || AnyFlag(mode & OpenMode::ModeUpdate) || AnyFlag(mode & OpenMode::ModeAppend); - if (write) + // If writing the file, create the file in all cases (except r+) + if (!SystemFile::Exists(path) && !(AnyFlag(mode & OpenMode::ModeRead) && AnyFlag(mode & OpenMode::ModeUpdate))) { - // If writing the file, create the file in all cases (except r+) - if (!SystemFile::Exists(path) && !(AnyFlag(mode & OpenMode::ModeRead) && AnyFlag(mode & OpenMode::ModeUpdate))) - { - // LocalFileIO creates by default - systemFileMode |= SystemFile::SF_OPEN_CREATE; - } - - if (AnyFlag(mode & OpenMode::ModeCreatePath)) - { - systemFileMode |= SystemFile::SF_OPEN_CREATE_PATH; - } - - // If appending, append. - if (AnyFlag(mode & OpenMode::ModeAppend)) - { - systemFileMode |= SystemFile::SF_OPEN_APPEND; - } - // If writing and not appending, empty the file - else if (AnyFlag(mode & OpenMode::ModeWrite)) - { - systemFileMode |= SystemFile::SF_OPEN_TRUNCATE; - } - - // If reading, set read/write, otherwise just write - if (read) - { - systemFileMode |= SystemFile::SF_OPEN_READ_WRITE; - } - else - { - systemFileMode |= SystemFile::SF_OPEN_WRITE_ONLY; - } - } - else if (read) - { - systemFileMode |= SystemFile::SF_OPEN_READ_ONLY; + // LocalFileIO creates by default + systemFileMode |= SystemFile::SF_OPEN_CREATE; } - return systemFileMode; + if (AnyFlag(mode & OpenMode::ModeCreatePath)) + { + systemFileMode |= SystemFile::SF_OPEN_CREATE_PATH; + } + + // If appending, append. + if (AnyFlag(mode & OpenMode::ModeAppend)) + { + systemFileMode |= SystemFile::SF_OPEN_APPEND; + } + // If writing and not appending, empty the file + else if (AnyFlag(mode & OpenMode::ModeWrite)) + { + systemFileMode |= SystemFile::SF_OPEN_TRUNCATE; + } + + // If reading, set read/write, otherwise just write + if (read) + { + systemFileMode |= SystemFile::SF_OPEN_READ_WRITE; + } + else + { + systemFileMode |= SystemFile::SF_OPEN_WRITE_ONLY; + } + } + else if (read) + { + systemFileMode |= SystemFile::SF_OPEN_READ_ONLY; } - bool RetryOpenStream(FileIOStream& stream, int numRetries, int delayBetweenRetry) + return systemFileMode; + } + + bool RetryOpenStream(FileIOStream& stream, int numRetries, int delayBetweenRetry) + { + while ((!stream.IsOpen()) && (numRetries > 0)) { - while ((!stream.IsOpen()) && (numRetries > 0)) - { - numRetries--; - AZStd::this_thread::sleep_for(AZStd::chrono::milliseconds(delayBetweenRetry)); - stream.ReOpen(); - } - return stream.IsOpen(); + numRetries--; + AZStd::this_thread::sleep_for(AZStd::chrono::milliseconds(delayBetweenRetry)); + stream.ReOpen(); } - } // namespace IO -} // namespace AZ - - + return stream.IsOpen(); + } +} // namespace AZ::IO diff --git a/Code/Framework/AzCore/AzCore/IO/Streamer/BlockCache.cpp b/Code/Framework/AzCore/AzCore/IO/Streamer/BlockCache.cpp index e838324408..6c873f3050 100644 --- a/Code/Framework/AzCore/AzCore/IO/Streamer/BlockCache.cpp +++ b/Code/Framework/AzCore/AzCore/IO/Streamer/BlockCache.cpp @@ -15,736 +15,733 @@ #include #include -namespace AZ +namespace AZ::IO { - namespace IO + AZStd::shared_ptr BlockCacheConfig::AddStreamStackEntry( + const HardwareInformation& hardware, AZStd::shared_ptr parent) { - AZStd::shared_ptr BlockCacheConfig::AddStreamStackEntry( - const HardwareInformation& hardware, AZStd::shared_ptr parent) + size_t blockSize; + switch (m_blockSize) { - size_t blockSize; - switch (m_blockSize) - { - case BlockSize::MaxTransfer: - blockSize = hardware.m_maxTransfer; - break; - case BlockSize::MemoryAlignment: - blockSize = hardware.m_maxPhysicalSectorSize; - break; - case BlockSize::SizeAlignment: - blockSize = hardware.m_maxLogicalSectorSize; - break; - default: - blockSize = m_blockSize; - break; - } - - u32 cacheSize = static_cast(m_cacheSizeMib * 1_mib); - if (blockSize * 2 > cacheSize) - { - AZ_Warning("Streamer", false, "Size (%u) for BlockCache isn't big enough to hold at least two cache blocks of size (%zu). " - "The cache size will be increased to fit 2 cache blocks.", cacheSize, blockSize); - cacheSize = aznumeric_caster(blockSize * 2); - } - - auto stackEntry = AZStd::make_shared( - cacheSize, aznumeric_cast(blockSize), aznumeric_cast(hardware.m_maxPhysicalSectorSize), false); - stackEntry->SetNext(AZStd::move(parent)); - return stackEntry; + case BlockSize::MaxTransfer: + blockSize = hardware.m_maxTransfer; + break; + case BlockSize::MemoryAlignment: + blockSize = hardware.m_maxPhysicalSectorSize; + break; + case BlockSize::SizeAlignment: + blockSize = hardware.m_maxLogicalSectorSize; + break; + default: + blockSize = m_blockSize; + break; } - void BlockCacheConfig::Reflect(AZ::ReflectContext* context) + u32 cacheSize = static_cast(m_cacheSizeMib * 1_mib); + if (blockSize * 2 > cacheSize) { - if (auto serializeContext = azrtti_cast(context); serializeContext != nullptr) - { - serializeContext->Enum() - ->Version(1) - ->Value("MaxTransfer", BlockSize::MaxTransfer) - ->Value("MemoryAlignment", BlockSize::MemoryAlignment) - ->Value("SizeAlignment", BlockSize::SizeAlignment); - - serializeContext->Class() - ->Version(1) - ->Field("CacheSizeMib", &BlockCacheConfig::m_cacheSizeMib) - ->Field("BlockSize", &BlockCacheConfig::m_blockSize); - } + AZ_Warning("Streamer", false, "Size (%u) for BlockCache isn't big enough to hold at least two cache blocks of size (%zu). " + "The cache size will be increased to fit 2 cache blocks.", cacheSize, blockSize); + cacheSize = aznumeric_caster(blockSize * 2); } - static constexpr char CacheHitRateName[] = "Cache hit rate"; - static constexpr char CacheableName[] = "Cacheable"; + auto stackEntry = AZStd::make_shared( + cacheSize, aznumeric_cast(blockSize), aznumeric_cast(hardware.m_maxPhysicalSectorSize), false); + stackEntry->SetNext(AZStd::move(parent)); + return stackEntry; + } - void BlockCache::Section::Prefix(const Section& section) + void BlockCacheConfig::Reflect(AZ::ReflectContext* context) + { + if (auto serializeContext = azrtti_cast(context); serializeContext != nullptr) { - AZ_Assert(section.m_used, "Trying to prefix an unused section"); - AZ_Assert(!m_wait && !section.m_wait, "Can't merge two section that are already waiting for data to be loaded."); + serializeContext->Enum() + ->Version(1) + ->Value("MaxTransfer", BlockSize::MaxTransfer) + ->Value("MemoryAlignment", BlockSize::MemoryAlignment) + ->Value("SizeAlignment", BlockSize::SizeAlignment); - if (m_used) + serializeContext->Class() + ->Version(1) + ->Field("CacheSizeMib", &BlockCacheConfig::m_cacheSizeMib) + ->Field("BlockSize", &BlockCacheConfig::m_blockSize); + } + } + + static constexpr char CacheHitRateName[] = "Cache hit rate"; + static constexpr char CacheableName[] = "Cacheable"; + + void BlockCache::Section::Prefix(const Section& section) + { + AZ_Assert(section.m_used, "Trying to prefix an unused section"); + AZ_Assert(!m_wait && !section.m_wait, "Can't merge two section that are already waiting for data to be loaded."); + + if (m_used) + { + AZ_Assert(m_blockOffset == 0, "Unable to add a block cache to this one as this block requires an offset upon completion."); + + AZ_Assert(section.m_readOffset < m_readOffset, "The block that's being merged needs to come before this block."); + m_readOffset = section.m_readOffset + section.m_blockOffset; // Remove any alignment that might have been added. + m_readSize += section.m_readSize - section.m_blockOffset; + + AZ_Assert(section.m_output < m_output, "The block that's being merged needs to come before this block."); + m_output = section.m_output; + m_copySize += section.m_copySize; + } + else + { + m_used = true; + m_readOffset = section.m_readOffset + section.m_blockOffset; + m_readSize = section.m_readSize - section.m_blockOffset; + m_output = section.m_output; + m_copySize = section.m_copySize; + } + m_blockOffset = 0; // Two merged sections do not support caching. + } + + BlockCache::BlockCache(u64 cacheSize, u32 blockSize, u32 alignment, bool onlyEpilogWrites) + : StreamStackEntry("Block cache") + , m_alignment(alignment) + , m_onlyEpilogWrites(onlyEpilogWrites) + { + AZ_Assert(IStreamerTypes::IsPowerOf2(alignment), "Alignment needs to be a power of 2."); + AZ_Assert(IStreamerTypes::IsAlignedTo(blockSize, alignment), "Block size needs to be a multiple of the alignment."); + + m_numBlocks = aznumeric_caster(cacheSize / blockSize); + m_cacheSize = cacheSize - (cacheSize % blockSize); // Only use the amount needed for the cache. + m_blockSize = blockSize; + if (m_numBlocks == 1) + { + m_onlyEpilogWrites = true; + } + + m_cache = reinterpret_cast(AZ::AllocatorInstance::Get().Allocate( + m_cacheSize, alignment, 0, "AZ::IO::Streamer BlockCache", __FILE__, __LINE__)); + m_cachedPaths = AZStd::unique_ptr(new RequestPath[m_numBlocks]); + m_cachedOffsets = AZStd::unique_ptr(new u64[m_numBlocks]); + m_blockLastTouched = AZStd::unique_ptr(new TimePoint[m_numBlocks]); + m_inFlightRequests = AZStd::unique_ptr(new FileRequest*[m_numBlocks]); + + ResetCache(); + } + + BlockCache::~BlockCache() + { + AZ::AllocatorInstance::Get().DeAllocate(m_cache, m_cacheSize, m_alignment); + } + + void BlockCache::QueueRequest(FileRequest* request) + { + AZ_Assert(request, "QueueRequest was provided a null request."); + + AZStd::visit([this, request](auto&& args) + { + using Command = AZStd::decay_t; + if constexpr (AZStd::is_same_v) { - AZ_Assert(m_blockOffset == 0, "Unable to add a block cache to this one as this block requires an offset upon completion."); - - AZ_Assert(section.m_readOffset < m_readOffset, "The block that's being merged needs to come before this block."); - m_readOffset = section.m_readOffset + section.m_blockOffset; // Remove any alignment that might have been added. - m_readSize += section.m_readSize - section.m_blockOffset; - - AZ_Assert(section.m_output < m_output, "The block that's being merged needs to come before this block."); - m_output = section.m_output; - m_copySize += section.m_copySize; + ReadFile(request, args); + return; } else { - m_used = true; - m_readOffset = section.m_readOffset + section.m_blockOffset; - m_readSize = section.m_readSize - section.m_blockOffset; - m_output = section.m_output; - m_copySize = section.m_copySize; - } - m_blockOffset = 0; // Two merged sections do not support caching. - } - - BlockCache::BlockCache(u64 cacheSize, u32 blockSize, u32 alignment, bool onlyEpilogWrites) - : StreamStackEntry("Block cache") - , m_alignment(alignment) - , m_onlyEpilogWrites(onlyEpilogWrites) - { - AZ_Assert(IStreamerTypes::IsPowerOf2(alignment), "Alignment needs to be a power of 2."); - AZ_Assert(IStreamerTypes::IsAlignedTo(blockSize, alignment), "Block size needs to be a multiple of the alignment."); - - m_numBlocks = aznumeric_caster(cacheSize / blockSize); - m_cacheSize = cacheSize - (cacheSize % blockSize); // Only use the amount needed for the cache. - m_blockSize = blockSize; - if (m_numBlocks == 1) - { - m_onlyEpilogWrites = true; - } - - m_cache = reinterpret_cast(AZ::AllocatorInstance::Get().Allocate( - m_cacheSize, alignment, 0, "AZ::IO::Streamer BlockCache", __FILE__, __LINE__)); - m_cachedPaths = AZStd::unique_ptr(new RequestPath[m_numBlocks]); - m_cachedOffsets = AZStd::unique_ptr(new u64[m_numBlocks]); - m_blockLastTouched = AZStd::unique_ptr(new TimePoint[m_numBlocks]); - m_inFlightRequests = AZStd::unique_ptr(new FileRequest*[m_numBlocks]); - - ResetCache(); - } - - BlockCache::~BlockCache() - { - AZ::AllocatorInstance::Get().DeAllocate(m_cache, m_cacheSize, m_alignment); - } - - void BlockCache::QueueRequest(FileRequest* request) - { - AZ_Assert(request, "QueueRequest was provided a null request."); - - AZStd::visit([this, request](auto&& args) - { - using Command = AZStd::decay_t; - if constexpr (AZStd::is_same_v) + if constexpr (AZStd::is_same_v) { - ReadFile(request, args); + FlushCache(args.m_path); + } + else if constexpr (AZStd::is_same_v) + { + FlushEntireCache(); + } + StreamStackEntry::QueueRequest(request); + } + }, request->GetCommand()); + } + + bool BlockCache::ExecuteRequests() + { + size_t delayedCount = m_delayedSections.size(); + + bool delayedRequestProcessed = false; + for (size_t i = 0; i < delayedCount; ++i) + { + Section& delayed = m_delayedSections.front(); + AZ_Assert(delayed.m_parent, "Delayed section doesn't have a reference to the original request."); + auto data = AZStd::get_if(&delayed.m_parent->GetCommand()); + AZ_Assert(data, "A request in the delayed queue of the BlockCache didn't have a parent with read data."); + // This call can add the same section to the back of the queue if there's not + // enough space. Because of this the entry needs to be removed from the delayed + // list no matter what the result is of ServiceFromCache. + if (ServiceFromCache(delayed.m_parent, delayed, data->m_path, data->m_sharedRead) != CacheResult::Delayed) + { + delayedRequestProcessed = true; + } + m_delayedSections.pop_front(); + } + bool nextResult = StreamStackEntry::ExecuteRequests(); + return nextResult || delayedRequestProcessed; + } + + void BlockCache::UpdateStatus(Status& status) const + { + StreamStackEntry::UpdateStatus(status); + s32 numAvailableSlots = CalculateAvailableRequestSlots(); + status.m_numAvailableSlots = AZStd::min(status.m_numAvailableSlots, numAvailableSlots); + status.m_isIdle = status.m_isIdle && + static_cast(numAvailableSlots) == m_numBlocks && + m_delayedSections.empty(); + } + + void BlockCache::UpdateCompletionEstimates(AZStd::chrono::system_clock::time_point now, AZStd::vector& internalPending, + StreamerContext::PreparedQueue::iterator pendingBegin, StreamerContext::PreparedQueue::iterator pendingEnd) + { + // Have the stack downstream estimate the completion time for the requests that are waiting for a slot to execute in. + AddDelayedRequests(internalPending); + + StreamStackEntry::UpdateCompletionEstimates(now, internalPending, pendingBegin, pendingEnd); + + // The in-flight requests don't have to be updated because the subdivided request will bubble up in order so the final + // write will be the latest completion time. Requests that have a wait on another request though will need to be update + // as the estimation of the in-flight request needs to be copied to the wait request to get an accurate prediction. + UpdatePendingRequestEstimations(); + + // Technically here the wait commands for the delayed sections should be updated as well, but it's the parent that's interesting, + // not the wait so don't waste cycles updating the wait. + } + + void BlockCache::AddDelayedRequests(AZStd::vector& internalPending) + { + for (auto& section : m_delayedSections) + { + internalPending.push_back(section.m_parent); + } + } + + void BlockCache::UpdatePendingRequestEstimations() + { + for (auto it : m_pendingRequests) + { + Section& section = it.second; + AZ_Assert(section.m_cacheBlockIndex != s_fileNotCached, "An in-flight cache section doesn't have a cache block associated with it."); + AZ_Assert(m_inFlightRequests[section.m_cacheBlockIndex], + "Cache block %i is reported as being in-flight but has no request.", section.m_cacheBlockIndex); + if (section.m_wait) + { + AZ_Assert(section.m_parent, "A cache section with a wait request pending is missing a parent to wait on."); + auto largestTime = AZStd::max(section.m_parent->GetEstimatedCompletion(), it.first->GetEstimatedCompletion()); + section.m_wait->SetEstimatedCompletion(largestTime); + } + } + } + + void BlockCache::ReadFile(FileRequest* request, FileRequest::ReadData& data) + { + if (!m_next) + { + request->SetStatus(IStreamerTypes::RequestStatus::Failed); + m_context->MarkRequestAsCompleted(request); + return; + } + + auto continueReadFile = [this, request](FileRequest& fileSizeRequest) + { + AZ_PROFILE_FUNCTION(AzCore); + AZ_Assert(m_numMetaDataRetrievalInProgress > 0, + "More requests have completed meta data retrieval in the Block Cache than were requested."); + m_numMetaDataRetrievalInProgress--; + if (fileSizeRequest.GetStatus() == IStreamerTypes::RequestStatus::Completed) + { + auto& requestInfo = AZStd::get(fileSizeRequest.GetCommand()); + if (requestInfo.m_found) + { + ContinueReadFile(request, requestInfo.m_fileSize); return; } - else + } + // Couldn't find the file size so don't try to split and pass the request to the next entry in the stack. + StreamStackEntry::QueueRequest(request); + }; + m_numMetaDataRetrievalInProgress++; + FileRequest* fileSizeRequest = m_context->GetNewInternalRequest(); + fileSizeRequest->CreateFileMetaDataRetrieval(data.m_path); + fileSizeRequest->SetCompletionCallback(AZStd::move(continueReadFile)); + StreamStackEntry::QueueRequest(fileSizeRequest); + } + void BlockCache::ContinueReadFile(FileRequest* request, u64 fileLength) + { + Section prolog; + Section main; + Section epilog; + + auto& data = AZStd::get(request->GetCommand()); + + if (!SplitRequest(prolog, main, epilog, data.m_path, fileLength, data.m_offset, data.m_size, + reinterpret_cast(data.m_output))) + { + m_context->MarkRequestAsCompleted(request); + return; + } + + if (prolog.m_used || epilog.m_used) + { + m_cacheableStat.PushSample(1.0); + Statistic::PlotImmediate(m_name, CacheableName, m_cacheableStat.GetMostRecentSample()); + } + else + { + // Nothing to cache so simply forward the call to the next entry in the stack for direct reading. + m_cacheableStat.PushSample(0.0); + Statistic::PlotImmediate(m_name, CacheableName, m_cacheableStat.GetMostRecentSample()); + m_next->QueueRequest(request); + return; + } + + bool fullyCached = true; + if (prolog.m_used) + { + if (m_onlyEpilogWrites && (main.m_used || epilog.m_used)) + { + // Only the epilog is allowed to write to the cache, but a previous read could + // still have cached the prolog, so check the cache and use the data if it's there + // otherwise merge the section with the main section to have the data read. + if (ReadFromCache(request, prolog, data.m_path) == CacheResult::CacheMiss) { - if constexpr (AZStd::is_same_v) - { - FlushCache(args.m_path); - } - else if constexpr (AZStd::is_same_v) - { - FlushEntireCache(); - } - StreamStackEntry::QueueRequest(request); - } - }, request->GetCommand()); - } - - bool BlockCache::ExecuteRequests() - { - size_t delayedCount = m_delayedSections.size(); - - bool delayedRequestProcessed = false; - for (size_t i = 0; i < delayedCount; ++i) - { - Section& delayed = m_delayedSections.front(); - AZ_Assert(delayed.m_parent, "Delayed section doesn't have a reference to the original request."); - auto data = AZStd::get_if(&delayed.m_parent->GetCommand()); - AZ_Assert(data, "A request in the delayed queue of the BlockCache didn't have a parent with read data."); - // This call can add the same section to the back of the queue if there's not - // enough space. Because of this the entry needs to be removed from the delayed - // list no matter what the result is of ServiceFromCache. - if (ServiceFromCache(delayed.m_parent, delayed, data->m_path, data->m_sharedRead) != CacheResult::Delayed) - { - delayedRequestProcessed = true; - } - m_delayedSections.pop_front(); - } - bool nextResult = StreamStackEntry::ExecuteRequests(); - return nextResult || delayedRequestProcessed; - } - - void BlockCache::UpdateStatus(Status& status) const - { - StreamStackEntry::UpdateStatus(status); - s32 numAvailableSlots = CalculateAvailableRequestSlots(); - status.m_numAvailableSlots = AZStd::min(status.m_numAvailableSlots, numAvailableSlots); - status.m_isIdle = status.m_isIdle && - static_cast(numAvailableSlots) == m_numBlocks && - m_delayedSections.empty(); - } - - void BlockCache::UpdateCompletionEstimates(AZStd::chrono::system_clock::time_point now, AZStd::vector& internalPending, - StreamerContext::PreparedQueue::iterator pendingBegin, StreamerContext::PreparedQueue::iterator pendingEnd) - { - // Have the stack downstream estimate the completion time for the requests that are waiting for a slot to execute in. - AddDelayedRequests(internalPending); - - StreamStackEntry::UpdateCompletionEstimates(now, internalPending, pendingBegin, pendingEnd); - - // The in-flight requests don't have to be updated because the subdivided request will bubble up in order so the final - // write will be the latest completion time. Requests that have a wait on another request though will need to be update - // as the estimation of the in-flight request needs to be copied to the wait request to get an accurate prediction. - UpdatePendingRequestEstimations(); - - // Technically here the wait commands for the delayed sections should be updated as well, but it's the parent that's interesting, - // not the wait so don't waste cycles updating the wait. - } - - void BlockCache::AddDelayedRequests(AZStd::vector& internalPending) - { - for (auto& section : m_delayedSections) - { - internalPending.push_back(section.m_parent); - } - } - - void BlockCache::UpdatePendingRequestEstimations() - { - for (auto it : m_pendingRequests) - { - Section& section = it.second; - AZ_Assert(section.m_cacheBlockIndex != s_fileNotCached, "An in-flight cache section doesn't have a cache block associated with it."); - AZ_Assert(m_inFlightRequests[section.m_cacheBlockIndex], - "Cache block %i is reported as being in-flight but has no request.", section.m_cacheBlockIndex); - if (section.m_wait) - { - AZ_Assert(section.m_parent, "A cache section with a wait request pending is missing a parent to wait on."); - auto largestTime = AZStd::max(section.m_parent->GetEstimatedCompletion(), it.first->GetEstimatedCompletion()); - section.m_wait->SetEstimatedCompletion(largestTime); - } - } - } - - void BlockCache::ReadFile(FileRequest* request, FileRequest::ReadData& data) - { - if (!m_next) - { - request->SetStatus(IStreamerTypes::RequestStatus::Failed); - m_context->MarkRequestAsCompleted(request); - return; - } - - auto continueReadFile = [this, request](FileRequest& fileSizeRequest) - { - AZ_PROFILE_FUNCTION(AzCore); - AZ_Assert(m_numMetaDataRetrievalInProgress > 0, - "More requests have completed meta data retrieval in the Block Cache than were requested."); - m_numMetaDataRetrievalInProgress--; - if (fileSizeRequest.GetStatus() == IStreamerTypes::RequestStatus::Completed) - { - auto& requestInfo = AZStd::get(fileSizeRequest.GetCommand()); - if (requestInfo.m_found) - { - ContinueReadFile(request, requestInfo.m_fileSize); - return; - } - } - // Couldn't find the file size so don't try to split and pass the request to the next entry in the stack. - StreamStackEntry::QueueRequest(request); - }; - m_numMetaDataRetrievalInProgress++; - FileRequest* fileSizeRequest = m_context->GetNewInternalRequest(); - fileSizeRequest->CreateFileMetaDataRetrieval(data.m_path); - fileSizeRequest->SetCompletionCallback(AZStd::move(continueReadFile)); - StreamStackEntry::QueueRequest(fileSizeRequest); - } - void BlockCache::ContinueReadFile(FileRequest* request, u64 fileLength) - { - Section prolog; - Section main; - Section epilog; - - auto& data = AZStd::get(request->GetCommand()); - - if (!SplitRequest(prolog, main, epilog, data.m_path, fileLength, data.m_offset, data.m_size, - reinterpret_cast(data.m_output))) - { - m_context->MarkRequestAsCompleted(request); - return; - } - - if (prolog.m_used || epilog.m_used) - { - m_cacheableStat.PushSample(1.0); - Statistic::PlotImmediate(m_name, CacheableName, m_cacheableStat.GetMostRecentSample()); - } - else - { - // Nothing to cache so simply forward the call to the next entry in the stack for direct reading. - m_cacheableStat.PushSample(0.0); - Statistic::PlotImmediate(m_name, CacheableName, m_cacheableStat.GetMostRecentSample()); - m_next->QueueRequest(request); - return; - } - - bool fullyCached = true; - if (prolog.m_used) - { - if (m_onlyEpilogWrites && (main.m_used || epilog.m_used)) - { - // Only the epilog is allowed to write to the cache, but a previous read could - // still have cached the prolog, so check the cache and use the data if it's there - // otherwise merge the section with the main section to have the data read. - if (ReadFromCache(request, prolog, data.m_path) == CacheResult::CacheMiss) - { - // The data isn't cached so put the prolog in front of the main section - // so it's read in one read request. If main wasn't used, prefixing the prolog - // will cause it to be filled in and used. - main.Prefix(prolog); - m_hitRateStat.PushSample(0.0); - Statistic::PlotImmediate(m_name, CacheHitRateName, m_hitRateStat.GetMostRecentSample()); - } - else - { - m_hitRateStat.PushSample(1.0); - Statistic::PlotImmediate(m_name, CacheHitRateName, m_hitRateStat.GetMostRecentSample()); - } + // The data isn't cached so put the prolog in front of the main section + // so it's read in one read request. If main wasn't used, prefixing the prolog + // will cause it to be filled in and used. + main.Prefix(prolog); + m_hitRateStat.PushSample(0.0); + Statistic::PlotImmediate(m_name, CacheHitRateName, m_hitRateStat.GetMostRecentSample()); } else { - // If m_onlyEpilogWrites is set but main and epilog are not filled in, it means that - // the request was so small it fits in one cache block, in which case the prolog and - // epilog are practically the same. Or this code is reached because both prolog and - // epilog are allowed to write. - bool readFromCache = (ServiceFromCache(request, prolog, data.m_path, data.m_sharedRead) == CacheResult::ReadFromCache); - fullyCached = readFromCache && fullyCached; - - m_hitRateStat.PushSample(readFromCache ? 1.0 : 0.0); + m_hitRateStat.PushSample(1.0); Statistic::PlotImmediate(m_name, CacheHitRateName, m_hitRateStat.GetMostRecentSample()); } } - - if (main.m_used) + else { - FileRequest* mainRequest = m_context->GetNewInternalRequest(); - // No need for a callback as there's nothing to do after the read has been completed. - mainRequest->CreateRead(request, main.m_output, main.m_readSize, data.m_path, - main.m_readOffset, main.m_readSize, data.m_sharedRead); - m_next->QueueRequest(mainRequest); - fullyCached = false; - } - - if (epilog.m_used) - { - bool readFromCache = (ServiceFromCache(request, epilog, data.m_path, data.m_sharedRead) == CacheResult::ReadFromCache); + // If m_onlyEpilogWrites is set but main and epilog are not filled in, it means that + // the request was so small it fits in one cache block, in which case the prolog and + // epilog are practically the same. Or this code is reached because both prolog and + // epilog are allowed to write. + bool readFromCache = (ServiceFromCache(request, prolog, data.m_path, data.m_sharedRead) == CacheResult::ReadFromCache); fullyCached = readFromCache && fullyCached; m_hitRateStat.PushSample(readFromCache ? 1.0 : 0.0); Statistic::PlotImmediate(m_name, CacheHitRateName, m_hitRateStat.GetMostRecentSample()); } + } - if (fullyCached) + if (main.m_used) + { + FileRequest* mainRequest = m_context->GetNewInternalRequest(); + // No need for a callback as there's nothing to do after the read has been completed. + mainRequest->CreateRead(request, main.m_output, main.m_readSize, data.m_path, + main.m_readOffset, main.m_readSize, data.m_sharedRead); + m_next->QueueRequest(mainRequest); + fullyCached = false; + } + + if (epilog.m_used) + { + bool readFromCache = (ServiceFromCache(request, epilog, data.m_path, data.m_sharedRead) == CacheResult::ReadFromCache); + fullyCached = readFromCache && fullyCached; + + m_hitRateStat.PushSample(readFromCache ? 1.0 : 0.0); + Statistic::PlotImmediate(m_name, CacheHitRateName, m_hitRateStat.GetMostRecentSample()); + } + + if (fullyCached) + { + request->SetStatus(IStreamerTypes::RequestStatus::Completed); + m_context->MarkRequestAsCompleted(request); + } + } + + void BlockCache::FlushCache(const RequestPath& filePath) + { + for (u32 i = 0; i < m_numBlocks; ++i) + { + if (m_cachedPaths[i] == filePath) { - request->SetStatus(IStreamerTypes::RequestStatus::Completed); - m_context->MarkRequestAsCompleted(request); + ResetCacheEntry(i); } } + } - void BlockCache::FlushCache(const RequestPath& filePath) + void BlockCache::FlushEntireCache() + { + ResetCache(); + } + + void BlockCache::CollectStatistics(AZStd::vector& statistics) const + { + statistics.push_back(Statistic::CreatePercentage(m_name, CacheHitRateName, CalculateHitRatePercentage())); + statistics.push_back(Statistic::CreatePercentage(m_name, CacheableName, CalculateCacheableRatePercentage())); + statistics.push_back(Statistic::CreateInteger(m_name, "Available slots", CalculateAvailableRequestSlots())); + + StreamStackEntry::CollectStatistics(statistics); + } + + double BlockCache::CalculateHitRatePercentage() const + { + return m_hitRateStat.GetAverage(); + } + + double BlockCache::CalculateCacheableRatePercentage() const + { + return m_cacheableStat.GetAverage(); + } + + s32 BlockCache::CalculateAvailableRequestSlots() const + { + return aznumeric_cast(m_numBlocks) - m_numInFlightRequests - m_numMetaDataRetrievalInProgress - + aznumeric_cast(m_delayedSections.size()); + } + + BlockCache::CacheResult BlockCache::ReadFromCache(FileRequest* request, Section& section, const RequestPath& filePath) + { + u32 cacheLocation = FindInCache(filePath, section.m_readOffset); + if (cacheLocation != s_fileNotCached) { - for (u32 i = 0; i < m_numBlocks; ++i) - { - if (m_cachedPaths[i] == filePath) - { - ResetCacheEntry(i); - } - } + return ReadFromCache(request, section, cacheLocation); } - - void BlockCache::FlushEntireCache() + else { - ResetCache(); + return CacheResult::CacheMiss; } + } - void BlockCache::CollectStatistics(AZStd::vector& statistics) const + BlockCache::CacheResult BlockCache::ReadFromCache(FileRequest* request, Section& section, u32 cacheBlock) + { + if (!IsCacheBlockInFlight(cacheBlock)) { - statistics.push_back(Statistic::CreatePercentage(m_name, CacheHitRateName, CalculateHitRatePercentage())); - statistics.push_back(Statistic::CreatePercentage(m_name, CacheableName, CalculateCacheableRatePercentage())); - statistics.push_back(Statistic::CreateInteger(m_name, "Available slots", CalculateAvailableRequestSlots())); - - StreamStackEntry::CollectStatistics(statistics); + TouchBlock(cacheBlock); + memcpy(section.m_output, GetCacheBlockData(cacheBlock) + section.m_blockOffset, section.m_copySize); + return CacheResult::ReadFromCache; } - - double BlockCache::CalculateHitRatePercentage() const + else { - return m_hitRateStat.GetAverage(); + AZ_Assert(section.m_wait == nullptr, "A wait request has to be set on a block cache section, but one has already been assigned."); + FileRequest* wait = m_context->GetNewInternalRequest(); + wait->CreateWait(request); + section.m_cacheBlockIndex = cacheBlock; + section.m_parent = request; + section.m_wait = wait; + m_pendingRequests.emplace(m_inFlightRequests[cacheBlock], section); + return CacheResult::Queued; } + } - double BlockCache::CalculateCacheableRatePercentage() const - { - return m_cacheableStat.GetAverage(); - } + BlockCache::CacheResult BlockCache::ServiceFromCache( + FileRequest* request, Section& section, const RequestPath& filePath, bool sharedRead) + { + AZ_Assert(m_next, "ServiceFromCache in BlockCache was called when the cache doesn't have a way to read files."); - s32 BlockCache::CalculateAvailableRequestSlots() const + u32 cacheLocation = FindInCache(filePath, section.m_readOffset); + if (cacheLocation == s_fileNotCached) { - return aznumeric_cast(m_numBlocks) - m_numInFlightRequests - m_numMetaDataRetrievalInProgress - - aznumeric_cast(m_delayedSections.size()); - } + m_hitRateStat.PushSample(0.0); + Statistic::PlotImmediate(m_name, CacheHitRateName, m_hitRateStat.GetMostRecentSample()); - BlockCache::CacheResult BlockCache::ReadFromCache(FileRequest* request, Section& section, const RequestPath& filePath) - { - u32 cacheLocation = FindInCache(filePath, section.m_readOffset); + section.m_parent = request; + cacheLocation = RecycleOldestBlock(filePath, section.m_readOffset); if (cacheLocation != s_fileNotCached) { - return ReadFromCache(request, section, cacheLocation); - } - else - { - return CacheResult::CacheMiss; - } - } + FileRequest* readRequest = m_context->GetNewInternalRequest(); + readRequest->CreateRead(request, GetCacheBlockData(cacheLocation), m_blockSize, filePath, section.m_readOffset, + section.m_readSize, sharedRead); + readRequest->SetCompletionCallback([this](FileRequest& request) + { + AZ_PROFILE_FUNCTION(AzCore); + CompleteRead(request); + }); + section.m_cacheBlockIndex = cacheLocation; + m_inFlightRequests[cacheLocation] = readRequest; + m_numInFlightRequests++; - BlockCache::CacheResult BlockCache::ReadFromCache(FileRequest* request, Section& section, u32 cacheBlock) - { - if (!IsCacheBlockInFlight(cacheBlock)) - { - TouchBlock(cacheBlock); - memcpy(section.m_output, GetCacheBlockData(cacheBlock) + section.m_blockOffset, section.m_copySize); - return CacheResult::ReadFromCache; - } - else - { - AZ_Assert(section.m_wait == nullptr, "A wait request has to be set on a block cache section, but one has already been assigned."); - FileRequest* wait = m_context->GetNewInternalRequest(); - wait->CreateWait(request); - section.m_cacheBlockIndex = cacheBlock; - section.m_parent = request; - section.m_wait = wait; - m_pendingRequests.emplace(m_inFlightRequests[cacheBlock], section); + // If set, this is the wait added by the delay. + if (section.m_wait) + { + m_context->MarkRequestAsCompleted(section.m_wait); + section.m_wait = nullptr; + } + + m_pendingRequests.emplace(readRequest, section); + m_next->QueueRequest(readRequest); return CacheResult::Queued; } - } - - BlockCache::CacheResult BlockCache::ServiceFromCache( - FileRequest* request, Section& section, const RequestPath& filePath, bool sharedRead) - { - AZ_Assert(m_next, "ServiceFromCache in BlockCache was called when the cache doesn't have a way to read files."); - - u32 cacheLocation = FindInCache(filePath, section.m_readOffset); - if (cacheLocation == s_fileNotCached) - { - m_hitRateStat.PushSample(0.0); - Statistic::PlotImmediate(m_name, CacheHitRateName, m_hitRateStat.GetMostRecentSample()); - - section.m_parent = request; - cacheLocation = RecycleOldestBlock(filePath, section.m_readOffset); - if (cacheLocation != s_fileNotCached) - { - FileRequest* readRequest = m_context->GetNewInternalRequest(); - readRequest->CreateRead(request, GetCacheBlockData(cacheLocation), m_blockSize, filePath, section.m_readOffset, - section.m_readSize, sharedRead); - readRequest->SetCompletionCallback([this](FileRequest& request) - { - AZ_PROFILE_FUNCTION(AzCore); - CompleteRead(request); - }); - section.m_cacheBlockIndex = cacheLocation; - m_inFlightRequests[cacheLocation] = readRequest; - m_numInFlightRequests++; - - // If set, this is the wait added by the delay. - if (section.m_wait) - { - m_context->MarkRequestAsCompleted(section.m_wait); - section.m_wait = nullptr; - } - - m_pendingRequests.emplace(readRequest, section); - m_next->QueueRequest(readRequest); - return CacheResult::Queued; - } - else - { - // There's no more space in the cache to store this request to. This is because there are more in-flight requests than - // there are slots in the cache. Delay the request until there's a slot available but add a wait for the section to - // make sure the request can't complete if some parts are read. - if (!section.m_wait) - { - section.m_wait = m_context->GetNewInternalRequest(); - section.m_wait->CreateWait(request); - } - m_delayedSections.push_back(section); - return CacheResult::Delayed; - } - } else { - // If set, this is the wait added by the delay when the cache was full. - if (section.m_wait) + // There's no more space in the cache to store this request to. This is because there are more in-flight requests than + // there are slots in the cache. Delay the request until there's a slot available but add a wait for the section to + // make sure the request can't complete if some parts are read. + if (!section.m_wait) { - m_context->MarkRequestAsCompleted(section.m_wait); - section.m_wait = nullptr; + section.m_wait = m_context->GetNewInternalRequest(); + section.m_wait->CreateWait(request); } - - m_hitRateStat.PushSample(1.0); - Statistic::PlotImmediate(m_name, CacheHitRateName, m_hitRateStat.GetMostRecentSample()); - - return ReadFromCache(request, section, cacheLocation); + m_delayedSections.push_back(section); + return CacheResult::Delayed; } } - - void BlockCache::CompleteRead(FileRequest& request) + else { - auto requestInfo = m_pendingRequests.equal_range(&request); - AZ_Assert(requestInfo.first != requestInfo.second, "Block cache was asked to complete a file request it never queued."); - - IStreamerTypes::RequestStatus requestStatus = request.GetStatus(); - bool requestWasSuccessful = requestStatus == IStreamerTypes::RequestStatus::Completed; - u32 cacheBlockIndex = requestInfo.first->second.m_cacheBlockIndex; - - for (auto it = requestInfo.first; it != requestInfo.second; ++it) + // If set, this is the wait added by the delay when the cache was full. + if (section.m_wait) { - Section& section = it->second; - AZ_Assert(section.m_cacheBlockIndex == cacheBlockIndex, - "Section associated with the file request is referencing the incorrect cache block (%u vs %u).", cacheBlockIndex, section.m_cacheBlockIndex); - if (section.m_wait) - { - section.m_wait->SetStatus(requestStatus); - m_context->MarkRequestAsCompleted(section.m_wait); - section.m_wait = nullptr; - } + m_context->MarkRequestAsCompleted(section.m_wait); + section.m_wait = nullptr; + } - if (requestWasSuccessful) - { - memcpy(section.m_output, GetCacheBlockData(cacheBlockIndex) + section.m_blockOffset, section.m_copySize); - } + m_hitRateStat.PushSample(1.0); + Statistic::PlotImmediate(m_name, CacheHitRateName, m_hitRateStat.GetMostRecentSample()); + + return ReadFromCache(request, section, cacheLocation); + } + } + + void BlockCache::CompleteRead(FileRequest& request) + { + auto requestInfo = m_pendingRequests.equal_range(&request); + AZ_Assert(requestInfo.first != requestInfo.second, "Block cache was asked to complete a file request it never queued."); + + IStreamerTypes::RequestStatus requestStatus = request.GetStatus(); + bool requestWasSuccessful = requestStatus == IStreamerTypes::RequestStatus::Completed; + u32 cacheBlockIndex = requestInfo.first->second.m_cacheBlockIndex; + + for (auto it = requestInfo.first; it != requestInfo.second; ++it) + { + Section& section = it->second; + AZ_Assert(section.m_cacheBlockIndex == cacheBlockIndex, + "Section associated with the file request is referencing the incorrect cache block (%u vs %u).", cacheBlockIndex, section.m_cacheBlockIndex); + if (section.m_wait) + { + section.m_wait->SetStatus(requestStatus); + m_context->MarkRequestAsCompleted(section.m_wait); + section.m_wait = nullptr; } if (requestWasSuccessful) { - TouchBlock(cacheBlockIndex); - m_inFlightRequests[cacheBlockIndex] = nullptr; + memcpy(section.m_output, GetCacheBlockData(cacheBlockIndex) + section.m_blockOffset, section.m_copySize); } - else - { - ResetCacheEntry(cacheBlockIndex); - } - AZ_Assert(m_numInFlightRequests > 0, "Clearing out an in-flight request, but there shouldn't be any in flight according to records."); - m_numInFlightRequests--; - m_pendingRequests.erase(&request); } - bool BlockCache::SplitRequest(Section& prolog, Section& main, Section& epilog, - [[maybe_unused]] const RequestPath& filePath, u64 fileLength, - u64 offset, u64 size, u8* buffer) const + if (requestWasSuccessful) { - AZ_Assert(offset + size <= fileLength, "File at path '%s' is being read past the end of the file.", filePath.GetRelativePath()); + TouchBlock(cacheBlockIndex); + m_inFlightRequests[cacheBlockIndex] = nullptr; + } + else + { + ResetCacheEntry(cacheBlockIndex); + } + AZ_Assert(m_numInFlightRequests > 0, "Clearing out an in-flight request, but there shouldn't be any in flight according to records."); + m_numInFlightRequests--; + m_pendingRequests.erase(&request); + } - // - // Prolog - // This looks at the request and sees if there's anything in front of the file that should be cached. This also - // deals with the situation where the entire file request fits inside the cache which could mean there's data - // left after the file as well that could be cached. - // - u64 roundedOffsetStart = AZ_SIZE_ALIGN_DOWN(offset, aznumeric_cast(m_blockSize)); - - u64 blockReadSizeStart = AZStd::min(fileLength - roundedOffsetStart, aznumeric_cast(m_blockSize)); - // Check if the request is on the left edge of the cache block, which means there's nothing in front of it - // that could be cached. - if (roundedOffsetStart == offset) - { - if (offset + size >= fileLength) - { - // The entire (remainder) of the file is read so there's nothing to cache - main.m_readOffset = offset; - main.m_readSize = size; - main.m_output = buffer; - main.m_used = true; - return true; - } - else if (size < blockReadSizeStart) - { - // The entire request fits inside a single cache block, but there's more file to read. - prolog.m_readOffset = offset; - prolog.m_readSize = blockReadSizeStart; - prolog.m_blockOffset = 0; - prolog.m_output = buffer; - prolog.m_copySize = size; - prolog.m_used = true; - return true; - } - // In any other case it means that the entire block would be read so caching has no effect. - } - else - { - // There is a portion of the file before that's not requested so always cache this block. - const u64 blockOffset = offset - roundedOffsetStart; - prolog.m_readOffset = roundedOffsetStart; - prolog.m_blockOffset = blockOffset; - prolog.m_output = buffer; - prolog.m_used = true; + bool BlockCache::SplitRequest(Section& prolog, Section& main, Section& epilog, + [[maybe_unused]] const RequestPath& filePath, u64 fileLength, + u64 offset, u64 size, u8* buffer) const + { + AZ_Assert(offset + size <= fileLength, "File at path '%s' is being read past the end of the file.", filePath.GetRelativePath()); - const bool isEntirelyInCache = blockOffset + size <= blockReadSizeStart; - if (isEntirelyInCache) - { - // The read size is already clamped to the file size above when blockReadSizeStart is set. - AZ_Assert(roundedOffsetStart + blockReadSizeStart <= fileLength, - "Read size in block cache was set to %llu but this is beyond the file length of %llu.", - roundedOffsetStart + blockReadSizeStart, fileLength); - prolog.m_readSize = blockReadSizeStart; - prolog.m_copySize = size; + // + // Prolog + // This looks at the request and sees if there's anything in front of the file that should be cached. This also + // deals with the situation where the entire file request fits inside the cache which could mean there's data + // left after the file as well that could be cached. + // + u64 roundedOffsetStart = AZ_SIZE_ALIGN_DOWN(offset, aznumeric_cast(m_blockSize)); - // There won't be anything else coming after this so continue reading. - return true; - } - else - { - prolog.m_readSize = blockReadSizeStart; - prolog.m_copySize = blockReadSizeStart - blockOffset; - } - } - - - // - // Epilog - // Since the prolog already takes care of the situation where the file fits entirely in the cache the epilog is - // much simpler as it only has to look at the case where there is more file after the request to read for caching. - // - u64 roundedOffsetEnd = AZ_SIZE_ALIGN_DOWN(offset + size, aznumeric_cast(m_blockSize)); - u64 copySize = offset + size - roundedOffsetEnd; - u64 blockReadSizeEnd = m_blockSize; - if ((roundedOffsetEnd + blockReadSizeEnd) > fileLength) + u64 blockReadSizeStart = AZStd::min(fileLength - roundedOffsetStart, aznumeric_cast(m_blockSize)); + // Check if the request is on the left edge of the cache block, which means there's nothing in front of it + // that could be cached. + if (roundedOffsetStart == offset) + { + if (offset + size >= fileLength) { - blockReadSizeEnd = fileLength - roundedOffsetEnd; - } - - // If the read doesn't align with the edge of the cache - if (copySize != 0 && copySize < blockReadSizeEnd) - { - epilog.m_readOffset = roundedOffsetEnd; - epilog.m_readSize = blockReadSizeEnd; - epilog.m_blockOffset = 0; - epilog.m_output = buffer + (roundedOffsetEnd - offset); - epilog.m_copySize = copySize; - epilog.m_used = true; - } - - // - // Main - // If this point is reached there's potentially a block between the prolog and epilog that can be directly read. - // - u64 adjustedOffset = offset; - if (prolog.m_used) - { - adjustedOffset += prolog.m_copySize; - size -= prolog.m_copySize; - } - if (epilog.m_used) - { - size -= epilog.m_copySize; - } - AZ_Assert(IStreamerTypes::IsAlignedTo(adjustedOffset, m_blockSize), - "The adjustments made by the prolog should guarantee the offset is aligned to a cache block."); - if (size != 0) - { - main.m_readOffset = adjustedOffset; + // The entire (remainder) of the file is read so there's nothing to cache + main.m_readOffset = offset; main.m_readSize = size; - main.m_output = buffer + (adjustedOffset - offset); + main.m_output = buffer; main.m_used = true; + return true; } - - return true; - } - - u8* BlockCache::GetCacheBlockData(u32 index) - { - AZ_Assert(index < m_numBlocks, "Index for touch a cache entry in the BlockCache is out of bounds."); - return m_cache + (index * m_blockSize); - } - - void BlockCache::TouchBlock(u32 index) - { - AZ_Assert(index < m_numBlocks, "Index for touch a cache entry in the BlockCache is out of bounds."); - m_blockLastTouched[index] = AZStd::chrono::high_resolution_clock::now(); - } - - u32 BlockCache::RecycleOldestBlock(const RequestPath& filePath, u64 offset) - { - AZ_Assert((offset & (m_blockSize - 1)) == 0, "The offset used to recycle a block cache needs to be a multiple of the block size."); - - // Find the oldest cache block. - TimePoint oldest = m_blockLastTouched[0]; - u32 oldestIndex = 0; - for (u32 i = 1; i < m_numBlocks; ++i) + else if (size < blockReadSizeStart) { - if (m_blockLastTouched[i] < oldest && !m_inFlightRequests[i]) - { - oldest = m_blockLastTouched[i]; - oldestIndex = i; - } + // The entire request fits inside a single cache block, but there's more file to read. + prolog.m_readOffset = offset; + prolog.m_readSize = blockReadSizeStart; + prolog.m_blockOffset = 0; + prolog.m_output = buffer; + prolog.m_copySize = size; + prolog.m_used = true; + return true; } + // In any other case it means that the entire block would be read so caching has no effect. + } + else + { + // There is a portion of the file before that's not requested so always cache this block. + const u64 blockOffset = offset - roundedOffsetStart; + prolog.m_readOffset = roundedOffsetStart; + prolog.m_blockOffset = blockOffset; + prolog.m_output = buffer; + prolog.m_used = true; - if (!IsCacheBlockInFlight(oldestIndex)) + const bool isEntirelyInCache = blockOffset + size <= blockReadSizeStart; + if (isEntirelyInCache) { - // Recycle the block. - m_cachedPaths[oldestIndex] = filePath; - m_cachedOffsets[oldestIndex] = offset; - TouchBlock(oldestIndex); - return oldestIndex; + // The read size is already clamped to the file size above when blockReadSizeStart is set. + AZ_Assert(roundedOffsetStart + blockReadSizeStart <= fileLength, + "Read size in block cache was set to %llu but this is beyond the file length of %llu.", + roundedOffsetStart + blockReadSizeStart, fileLength); + prolog.m_readSize = blockReadSizeStart; + prolog.m_copySize = size; + + // There won't be anything else coming after this so continue reading. + return true; } else { - return s_fileNotCached; + prolog.m_readSize = blockReadSizeStart; + prolog.m_copySize = blockReadSizeStart - blockOffset; } } - u32 BlockCache::FindInCache(const RequestPath& filePath, u64 offset) const - { - AZ_Assert((offset & (m_blockSize - 1)) == 0, "The offset used to find a block in the block cache needs to be a multiple of the block size."); - for (u32 i = 0; i < m_numBlocks; ++i) - { - if (m_cachedPaths[i] == filePath && m_cachedOffsets[i] == offset) - { - return i; - } - } + // + // Epilog + // Since the prolog already takes care of the situation where the file fits entirely in the cache the epilog is + // much simpler as it only has to look at the case where there is more file after the request to read for caching. + // + u64 roundedOffsetEnd = AZ_SIZE_ALIGN_DOWN(offset + size, aznumeric_cast(m_blockSize)); + u64 copySize = offset + size - roundedOffsetEnd; + u64 blockReadSizeEnd = m_blockSize; + if ((roundedOffsetEnd + blockReadSizeEnd) > fileLength) + { + blockReadSizeEnd = fileLength - roundedOffsetEnd; + } + + // If the read doesn't align with the edge of the cache + if (copySize != 0 && copySize < blockReadSizeEnd) + { + epilog.m_readOffset = roundedOffsetEnd; + epilog.m_readSize = blockReadSizeEnd; + epilog.m_blockOffset = 0; + epilog.m_output = buffer + (roundedOffsetEnd - offset); + epilog.m_copySize = copySize; + epilog.m_used = true; + } + + // + // Main + // If this point is reached there's potentially a block between the prolog and epilog that can be directly read. + // + u64 adjustedOffset = offset; + if (prolog.m_used) + { + adjustedOffset += prolog.m_copySize; + size -= prolog.m_copySize; + } + if (epilog.m_used) + { + size -= epilog.m_copySize; + } + AZ_Assert(IStreamerTypes::IsAlignedTo(adjustedOffset, m_blockSize), + "The adjustments made by the prolog should guarantee the offset is aligned to a cache block."); + if (size != 0) + { + main.m_readOffset = adjustedOffset; + main.m_readSize = size; + main.m_output = buffer + (adjustedOffset - offset); + main.m_used = true; + } + + return true; + } + + u8* BlockCache::GetCacheBlockData(u32 index) + { + AZ_Assert(index < m_numBlocks, "Index for touch a cache entry in the BlockCache is out of bounds."); + return m_cache + (index * m_blockSize); + } + + void BlockCache::TouchBlock(u32 index) + { + AZ_Assert(index < m_numBlocks, "Index for touch a cache entry in the BlockCache is out of bounds."); + m_blockLastTouched[index] = AZStd::chrono::high_resolution_clock::now(); + } + + u32 BlockCache::RecycleOldestBlock(const RequestPath& filePath, u64 offset) + { + AZ_Assert((offset & (m_blockSize - 1)) == 0, "The offset used to recycle a block cache needs to be a multiple of the block size."); + + // Find the oldest cache block. + TimePoint oldest = m_blockLastTouched[0]; + u32 oldestIndex = 0; + for (u32 i = 1; i < m_numBlocks; ++i) + { + if (m_blockLastTouched[i] < oldest && !m_inFlightRequests[i]) + { + oldest = m_blockLastTouched[i]; + oldestIndex = i; + } + } + + if (!IsCacheBlockInFlight(oldestIndex)) + { + // Recycle the block. + m_cachedPaths[oldestIndex] = filePath; + m_cachedOffsets[oldestIndex] = offset; + TouchBlock(oldestIndex); + return oldestIndex; + } + else + { return s_fileNotCached; } + } - bool BlockCache::IsCacheBlockInFlight(u32 index) const + u32 BlockCache::FindInCache(const RequestPath& filePath, u64 offset) const + { + AZ_Assert((offset & (m_blockSize - 1)) == 0, "The offset used to find a block in the block cache needs to be a multiple of the block size."); + for (u32 i = 0; i < m_numBlocks; ++i) { - AZ_Assert(index < m_numBlocks, "Index for checking if a cache block is in flight is out of bounds."); - return m_inFlightRequests[index] != nullptr; - } - - void BlockCache::ResetCacheEntry(u32 index) - { - AZ_Assert(index < m_numBlocks, "Index for resetting a cache entry in the BlockCache is out of bounds."); - - m_cachedPaths[index].Clear(); - m_cachedOffsets[index] = 0; - m_blockLastTouched[index] = TimePoint::min(); - m_inFlightRequests[index] = nullptr; - } - - void BlockCache::ResetCache() - { - for (u32 i = 0; i < m_numBlocks; ++i) + if (m_cachedPaths[i] == filePath && m_cachedOffsets[i] == offset) { - ResetCacheEntry(i); + return i; } - m_numInFlightRequests = 0; } - } // namespace IO -} // namespace AZ + + return s_fileNotCached; + } + + bool BlockCache::IsCacheBlockInFlight(u32 index) const + { + AZ_Assert(index < m_numBlocks, "Index for checking if a cache block is in flight is out of bounds."); + return m_inFlightRequests[index] != nullptr; + } + + void BlockCache::ResetCacheEntry(u32 index) + { + AZ_Assert(index < m_numBlocks, "Index for resetting a cache entry in the BlockCache is out of bounds."); + + m_cachedPaths[index].Clear(); + m_cachedOffsets[index] = 0; + m_blockLastTouched[index] = TimePoint::min(); + m_inFlightRequests[index] = nullptr; + } + + void BlockCache::ResetCache() + { + for (u32 i = 0; i < m_numBlocks; ++i) + { + ResetCacheEntry(i); + } + m_numInFlightRequests = 0; + } +} // namespace AZ::IO diff --git a/Code/Framework/AzCore/AzCore/IO/Streamer/DedicatedCache.cpp b/Code/Framework/AzCore/AzCore/IO/Streamer/DedicatedCache.cpp index e0e512e21f..b80a1ea724 100644 --- a/Code/Framework/AzCore/AzCore/IO/Streamer/DedicatedCache.cpp +++ b/Code/Framework/AzCore/AzCore/IO/Streamer/DedicatedCache.cpp @@ -12,320 +12,317 @@ #include #include -namespace AZ +namespace AZ::IO { - namespace IO + AZStd::shared_ptr DedicatedCacheConfig::AddStreamStackEntry( + const HardwareInformation& hardware, AZStd::shared_ptr parent) { - AZStd::shared_ptr DedicatedCacheConfig::AddStreamStackEntry( - const HardwareInformation& hardware, AZStd::shared_ptr parent) + size_t blockSize; + switch (m_blockSize) { - size_t blockSize; - switch (m_blockSize) + case BlockCacheConfig::BlockSize::MaxTransfer: + blockSize = hardware.m_maxTransfer; + break; + case BlockCacheConfig::BlockSize::MemoryAlignment: + blockSize = hardware.m_maxPhysicalSectorSize; + break; + case BlockCacheConfig::BlockSize::SizeAlignment: + blockSize = hardware.m_maxLogicalSectorSize; + break; + default: + blockSize = m_blockSize; + break; + } + + u32 cacheSize = static_cast(m_cacheSizeMib * 1_mib); + if (blockSize > cacheSize) + { + AZ_Warning("Streamer", false, "Size (%u) for DedicatedCache isn't big enough to hold at least one cache blocks of size (%zu). " + "The cache size will be increased to fit one cache block.", cacheSize, blockSize); + cacheSize = aznumeric_caster(blockSize); + } + + auto stackEntry = AZStd::make_shared( + cacheSize, aznumeric_cast(blockSize), aznumeric_cast(hardware.m_maxPhysicalSectorSize), m_writeOnlyEpilog); + stackEntry->SetNext(AZStd::move(parent)); + return stackEntry; + } + + void DedicatedCacheConfig::Reflect(AZ::ReflectContext* context) + { + if (auto serializeContext = azrtti_cast(context); serializeContext != nullptr) + { + serializeContext->Class() + ->Version(1) + ->Field("CacheSizeMib", &DedicatedCacheConfig::m_cacheSizeMib) + ->Field("BlockSize", &DedicatedCacheConfig::m_blockSize) + ->Field("WriteOnlyEpilog", &DedicatedCacheConfig::m_writeOnlyEpilog); + } + } + + + + // + // DedicatedCache + // + + DedicatedCache::DedicatedCache(u64 cacheSize, u32 blockSize, u32 alignment, bool onlyEpilogWrites) + : StreamStackEntry("Dedicated cache") + , m_cacheSize(cacheSize) + , m_alignment(alignment) + , m_blockSize(blockSize) + , m_onlyEpilogWrites(onlyEpilogWrites) + { + } + + void DedicatedCache::SetNext(AZStd::shared_ptr next) + { + m_next = AZStd::move(next); + for (AZStd::unique_ptr& cache : m_cachedFileCaches) + { + cache->SetNext(m_next); + } + } + + void DedicatedCache::SetContext(StreamerContext& context) + { + StreamStackEntry::SetContext(context); + for (AZStd::unique_ptr& cache : m_cachedFileCaches) + { + cache->SetContext(context); + } + } + + void DedicatedCache::PrepareRequest(FileRequest* request) + { + AZ_Assert(request, "PrepareRequest was provided a null request."); + + // Claim the requests so other entries can't claim it and make updates. + AZStd::visit([this, request](auto&& args) + { + using Command = AZStd::decay_t; + if constexpr (AZStd::is_same_v) { - case BlockCacheConfig::BlockSize::MaxTransfer: - blockSize = hardware.m_maxTransfer; - break; - case BlockCacheConfig::BlockSize::MemoryAlignment: - blockSize = hardware.m_maxPhysicalSectorSize; - break; - case BlockCacheConfig::BlockSize::SizeAlignment: - blockSize = hardware.m_maxLogicalSectorSize; - break; - default: - blockSize = m_blockSize; - break; + args.m_range = FileRange::CreateRangeForEntireFile(); + m_context->PushPreparedRequest(request); } - - u32 cacheSize = static_cast(m_cacheSizeMib * 1_mib); - if (blockSize > cacheSize) + else if constexpr (AZStd::is_same_v) { - AZ_Warning("Streamer", false, "Size (%u) for DedicatedCache isn't big enough to hold at least one cache blocks of size (%zu). " - "The cache size will be increased to fit one cache block.", cacheSize, blockSize); - cacheSize = aznumeric_caster(blockSize); - } - - auto stackEntry = AZStd::make_shared( - cacheSize, aznumeric_cast(blockSize), aznumeric_cast(hardware.m_maxPhysicalSectorSize), m_writeOnlyEpilog); - stackEntry->SetNext(AZStd::move(parent)); - return stackEntry; - } - - void DedicatedCacheConfig::Reflect(AZ::ReflectContext* context) - { - if (auto serializeContext = azrtti_cast(context); serializeContext != nullptr) - { - serializeContext->Class() - ->Version(1) - ->Field("CacheSizeMib", &DedicatedCacheConfig::m_cacheSizeMib) - ->Field("BlockSize", &DedicatedCacheConfig::m_blockSize) - ->Field("WriteOnlyEpilog", &DedicatedCacheConfig::m_writeOnlyEpilog); - } - } - - - - // - // DedicatedCache - // - - DedicatedCache::DedicatedCache(u64 cacheSize, u32 blockSize, u32 alignment, bool onlyEpilogWrites) - : StreamStackEntry("Dedicated cache") - , m_cacheSize(cacheSize) - , m_alignment(alignment) - , m_blockSize(blockSize) - , m_onlyEpilogWrites(onlyEpilogWrites) - { - } - - void DedicatedCache::SetNext(AZStd::shared_ptr next) - { - m_next = AZStd::move(next); - for (AZStd::unique_ptr& cache : m_cachedFileCaches) - { - cache->SetNext(m_next); - } - } - - void DedicatedCache::SetContext(StreamerContext& context) - { - StreamStackEntry::SetContext(context); - for (AZStd::unique_ptr& cache : m_cachedFileCaches) - { - cache->SetContext(context); - } - } - - void DedicatedCache::PrepareRequest(FileRequest* request) - { - AZ_Assert(request, "PrepareRequest was provided a null request."); - - // Claim the requests so other entries can't claim it and make updates. - AZStd::visit([this, request](auto&& args) - { - using Command = AZStd::decay_t; - if constexpr (AZStd::is_same_v) - { - args.m_range = FileRange::CreateRangeForEntireFile(); - m_context->PushPreparedRequest(request); - } - else if constexpr (AZStd::is_same_v) - { - args.m_range = FileRange::CreateRangeForEntireFile(); - m_context->PushPreparedRequest(request); - } - else - { - StreamStackEntry::PrepareRequest(request); - } - }, request->GetCommand()); - } - - void DedicatedCache::QueueRequest(FileRequest* request) - { - AZ_Assert(request, "QueueRequest was provided a null request."); - - AZStd::visit([this, request](auto&& args) - { - using Command = AZStd::decay_t; - if constexpr (AZStd::is_same_v) - { - ReadFile(request, args); - return; - } - else if constexpr (AZStd::is_same_v) - { - CreateDedicatedCache(request, args); - return; - } - else if constexpr (AZStd::is_same_v) - { - DestroyDedicatedCache(request, args); - return; - } - else - { - if constexpr (AZStd::is_same_v) - { - FlushCache(args.m_path); - } - else if constexpr (AZStd::is_same_v) - { - FlushEntireCache(); - } - StreamStackEntry::QueueRequest(request); - } - }, request->GetCommand()); - } - - bool DedicatedCache::ExecuteRequests() - { - bool hasProcessedRequest = false; - for (AZStd::unique_ptr& cache : m_cachedFileCaches) - { - hasProcessedRequest = cache->ExecuteRequests() || hasProcessedRequest; - } - return StreamStackEntry::ExecuteRequests() || hasProcessedRequest; - } - - void DedicatedCache::UpdateStatus(Status& status) const - { - // Available slots are not updated because the dedicated caches are often - // small and specific to a tiny subset of files that are loaded. It would therefore - // return a small number of slots that would needlessly hamper streaming as it doesn't - // apply to the majority of files. - - bool isIdle = true; - for (auto& cache : m_cachedFileCaches) - { - Status blockStatus; - cache->UpdateStatus(blockStatus); - isIdle = isIdle && blockStatus.m_isIdle; - } - status.m_isIdle = status.m_isIdle && isIdle; - StreamStackEntry::UpdateStatus(status); - } - - void DedicatedCache::UpdateCompletionEstimates(AZStd::chrono::system_clock::time_point now, - AZStd::vector& internalPending, StreamerContext::PreparedQueue::iterator pendingBegin, - StreamerContext::PreparedQueue::iterator pendingEnd) - { - for (auto& cache : m_cachedFileCaches) - { - cache->AddDelayedRequests(internalPending); - } - - StreamStackEntry::UpdateCompletionEstimates(now, internalPending, pendingBegin, pendingEnd); - - for (auto& cache : m_cachedFileCaches) - { - cache->UpdatePendingRequestEstimations(); - } - } - - void DedicatedCache::ReadFile(FileRequest* request, FileRequest::ReadData& data) - { - size_t index = FindCache(data.m_path, data.m_offset); - if (index == s_fileNotFound) - { - m_usagePercentageStat.PushSample(0.0); - if (m_next) - { - m_next->QueueRequest(request); - } + args.m_range = FileRange::CreateRangeForEntireFile(); + m_context->PushPreparedRequest(request); } else { - m_usagePercentageStat.PushSample(1.0); - BlockCache& cache = *m_cachedFileCaches[index]; - cache.QueueRequest(request); -#if AZ_STREAMER_ADD_EXTRA_PROFILING_INFO - m_overallHitRateStat.PushSample(cache.CalculateHitRatePercentage()); - m_overallCacheableRateStat.PushSample(cache.CalculateCacheableRatePercentage()); -#endif + StreamStackEntry::PrepareRequest(request); } - } + }, request->GetCommand()); + } - void DedicatedCache::FlushCache(const RequestPath& filePath) + void DedicatedCache::QueueRequest(FileRequest* request) + { + AZ_Assert(request, "QueueRequest was provided a null request."); + + AZStd::visit([this, request](auto&& args) { - size_t count = m_cachedFileNames.size(); - for (size_t i = 0; i < count; ++i) + using Command = AZStd::decay_t; + if constexpr (AZStd::is_same_v) { - if (m_cachedFileNames[i] == filePath) - { - // Flush the entire block cache as it's entirely dedicated to the found file. - m_cachedFileCaches[i]->FlushEntireCache(); - } + ReadFile(request, args); + return; } - } - - void DedicatedCache::FlushEntireCache() - { - for (AZStd::unique_ptr& cache : m_cachedFileCaches) + else if constexpr (AZStd::is_same_v) { - cache->FlushEntireCache(); + CreateDedicatedCache(request, args); + return; } - } - - void DedicatedCache::CollectStatistics(AZStd::vector& statistics) const - { - statistics.push_back(Statistic::CreatePercentage(m_name, "Reads from dedicated cache", m_usagePercentageStat.GetAverage())); -#if AZ_STREAMER_ADD_EXTRA_PROFILING_INFO - statistics.push_back(Statistic::CreatePercentage(m_name, "Overall cacheable rate", m_overallCacheableRateStat.GetAverage())); - statistics.push_back(Statistic::CreatePercentage(m_name, "Overall hit rate", m_overallHitRateStat.GetAverage())); -#endif - statistics.push_back(Statistic::CreateInteger(m_name, "Num dedicated caches", aznumeric_caster(m_cachedFileNames.size()))); - StreamStackEntry::CollectStatistics(statistics); - } - - void DedicatedCache::CreateDedicatedCache(FileRequest* request, FileRequest::CreateDedicatedCacheData& data) - { - size_t index = FindCache(data.m_path, data.m_range); - if (index == s_fileNotFound) + else if constexpr (AZStd::is_same_v) { - index = m_cachedFileCaches.size(); - m_cachedFileNames.push_back(data.m_path); - m_cachedFileRanges.push_back(data.m_range); - m_cachedFileCaches.push_back(AZStd::make_unique(m_cacheSize, m_blockSize, m_alignment, m_onlyEpilogWrites)); - m_cachedFileCaches[index]->SetNext(m_next); - m_cachedFileCaches[index]->SetContext(*m_context); - m_cachedFileRefCounts.push_back(1); + DestroyDedicatedCache(request, args); + return; } else { - ++m_cachedFileRefCounts[index]; + if constexpr (AZStd::is_same_v) + { + FlushCache(args.m_path); + } + else if constexpr (AZStd::is_same_v) + { + FlushEntireCache(); + } + StreamStackEntry::QueueRequest(request); } - request->SetStatus(IStreamerTypes::RequestStatus::Completed); - m_context->MarkRequestAsCompleted(request); + }, request->GetCommand()); + } + + bool DedicatedCache::ExecuteRequests() + { + bool hasProcessedRequest = false; + for (AZStd::unique_ptr& cache : m_cachedFileCaches) + { + hasProcessedRequest = cache->ExecuteRequests() || hasProcessedRequest; + } + return StreamStackEntry::ExecuteRequests() || hasProcessedRequest; + } + + void DedicatedCache::UpdateStatus(Status& status) const + { + // Available slots are not updated because the dedicated caches are often + // small and specific to a tiny subset of files that are loaded. It would therefore + // return a small number of slots that would needlessly hamper streaming as it doesn't + // apply to the majority of files. + + bool isIdle = true; + for (auto& cache : m_cachedFileCaches) + { + Status blockStatus; + cache->UpdateStatus(blockStatus); + isIdle = isIdle && blockStatus.m_isIdle; + } + status.m_isIdle = status.m_isIdle && isIdle; + StreamStackEntry::UpdateStatus(status); + } + + void DedicatedCache::UpdateCompletionEstimates(AZStd::chrono::system_clock::time_point now, + AZStd::vector& internalPending, StreamerContext::PreparedQueue::iterator pendingBegin, + StreamerContext::PreparedQueue::iterator pendingEnd) + { + for (auto& cache : m_cachedFileCaches) + { + cache->AddDelayedRequests(internalPending); } - void DedicatedCache::DestroyDedicatedCache(FileRequest* request, FileRequest::DestroyDedicatedCacheData& data) - { - size_t index = FindCache(data.m_path, data.m_range); - if (index != s_fileNotFound) - { - if (m_cachedFileRefCounts[index] > 0) - { - --m_cachedFileRefCounts[index]; - if (m_cachedFileRefCounts[index] == 0) - { - m_cachedFileNames.erase(m_cachedFileNames.begin() + index); - m_cachedFileRanges.erase(m_cachedFileRanges.begin() + index); - m_cachedFileCaches.erase(m_cachedFileCaches.begin() + index); - m_cachedFileRefCounts.erase(m_cachedFileRefCounts.begin() + index); - } - request->SetStatus(IStreamerTypes::RequestStatus::Completed); - m_context->MarkRequestAsCompleted(request); - return; - } - } - request->SetStatus(IStreamerTypes::RequestStatus::Failed); - m_context->MarkRequestAsCompleted(request); - } + StreamStackEntry::UpdateCompletionEstimates(now, internalPending, pendingBegin, pendingEnd); - size_t DedicatedCache::FindCache(const RequestPath& filename, FileRange range) + for (auto& cache : m_cachedFileCaches) { - size_t count = m_cachedFileNames.size(); - for (size_t i = 0; i < count; ++i) - { - if (m_cachedFileNames[i] == filename && m_cachedFileRanges[i] == range) - { - return i; - } - } - return s_fileNotFound; + cache->UpdatePendingRequestEstimations(); } + } - size_t DedicatedCache::FindCache(const RequestPath& filename, u64 offset) + void DedicatedCache::ReadFile(FileRequest* request, FileRequest::ReadData& data) + { + size_t index = FindCache(data.m_path, data.m_offset); + if (index == s_fileNotFound) { - size_t count = m_cachedFileNames.size(); - for (size_t i = 0; i < count; ++i) + m_usagePercentageStat.PushSample(0.0); + if (m_next) { - if (m_cachedFileNames[i] == filename && m_cachedFileRanges[i].IsInRange(offset)) - { - return i; - } + m_next->QueueRequest(request); } - return s_fileNotFound; } - } // namespace IO -} // namespace AZ + else + { + m_usagePercentageStat.PushSample(1.0); + BlockCache& cache = *m_cachedFileCaches[index]; + cache.QueueRequest(request); +#if AZ_STREAMER_ADD_EXTRA_PROFILING_INFO + m_overallHitRateStat.PushSample(cache.CalculateHitRatePercentage()); + m_overallCacheableRateStat.PushSample(cache.CalculateCacheableRatePercentage()); +#endif + } + } + + void DedicatedCache::FlushCache(const RequestPath& filePath) + { + size_t count = m_cachedFileNames.size(); + for (size_t i = 0; i < count; ++i) + { + if (m_cachedFileNames[i] == filePath) + { + // Flush the entire block cache as it's entirely dedicated to the found file. + m_cachedFileCaches[i]->FlushEntireCache(); + } + } + } + + void DedicatedCache::FlushEntireCache() + { + for (AZStd::unique_ptr& cache : m_cachedFileCaches) + { + cache->FlushEntireCache(); + } + } + + void DedicatedCache::CollectStatistics(AZStd::vector& statistics) const + { + statistics.push_back(Statistic::CreatePercentage(m_name, "Reads from dedicated cache", m_usagePercentageStat.GetAverage())); +#if AZ_STREAMER_ADD_EXTRA_PROFILING_INFO + statistics.push_back(Statistic::CreatePercentage(m_name, "Overall cacheable rate", m_overallCacheableRateStat.GetAverage())); + statistics.push_back(Statistic::CreatePercentage(m_name, "Overall hit rate", m_overallHitRateStat.GetAverage())); +#endif + statistics.push_back(Statistic::CreateInteger(m_name, "Num dedicated caches", aznumeric_caster(m_cachedFileNames.size()))); + StreamStackEntry::CollectStatistics(statistics); + } + + void DedicatedCache::CreateDedicatedCache(FileRequest* request, FileRequest::CreateDedicatedCacheData& data) + { + size_t index = FindCache(data.m_path, data.m_range); + if (index == s_fileNotFound) + { + index = m_cachedFileCaches.size(); + m_cachedFileNames.push_back(data.m_path); + m_cachedFileRanges.push_back(data.m_range); + m_cachedFileCaches.push_back(AZStd::make_unique(m_cacheSize, m_blockSize, m_alignment, m_onlyEpilogWrites)); + m_cachedFileCaches[index]->SetNext(m_next); + m_cachedFileCaches[index]->SetContext(*m_context); + m_cachedFileRefCounts.push_back(1); + } + else + { + ++m_cachedFileRefCounts[index]; + } + request->SetStatus(IStreamerTypes::RequestStatus::Completed); + m_context->MarkRequestAsCompleted(request); + } + + void DedicatedCache::DestroyDedicatedCache(FileRequest* request, FileRequest::DestroyDedicatedCacheData& data) + { + size_t index = FindCache(data.m_path, data.m_range); + if (index != s_fileNotFound) + { + if (m_cachedFileRefCounts[index] > 0) + { + --m_cachedFileRefCounts[index]; + if (m_cachedFileRefCounts[index] == 0) + { + m_cachedFileNames.erase(m_cachedFileNames.begin() + index); + m_cachedFileRanges.erase(m_cachedFileRanges.begin() + index); + m_cachedFileCaches.erase(m_cachedFileCaches.begin() + index); + m_cachedFileRefCounts.erase(m_cachedFileRefCounts.begin() + index); + } + request->SetStatus(IStreamerTypes::RequestStatus::Completed); + m_context->MarkRequestAsCompleted(request); + return; + } + } + request->SetStatus(IStreamerTypes::RequestStatus::Failed); + m_context->MarkRequestAsCompleted(request); + } + + size_t DedicatedCache::FindCache(const RequestPath& filename, FileRange range) + { + size_t count = m_cachedFileNames.size(); + for (size_t i = 0; i < count; ++i) + { + if (m_cachedFileNames[i] == filename && m_cachedFileRanges[i] == range) + { + return i; + } + } + return s_fileNotFound; + } + + size_t DedicatedCache::FindCache(const RequestPath& filename, u64 offset) + { + size_t count = m_cachedFileNames.size(); + for (size_t i = 0; i < count; ++i) + { + if (m_cachedFileNames[i] == filename && m_cachedFileRanges[i].IsInRange(offset)) + { + return i; + } + } + return s_fileNotFound; + } +} // namespace AZ::IO diff --git a/Code/Framework/AzCore/AzCore/IO/Streamer/FileRange.cpp b/Code/Framework/AzCore/AzCore/IO/Streamer/FileRange.cpp index df4f77b722..3a568d3f47 100644 --- a/Code/Framework/AzCore/AzCore/IO/Streamer/FileRange.cpp +++ b/Code/Framework/AzCore/AzCore/IO/Streamer/FileRange.cpp @@ -8,104 +8,101 @@ #include -namespace AZ +namespace AZ::IO { - namespace IO + FileRange FileRange::CreateRange(u64 offset, u64 size) { - FileRange FileRange::CreateRange(u64 offset, u64 size) - { - FileRange result; - result.m_hasOffsetEndSet = true; - result.m_isEntireFile = false; - result.m_offsetBegin = offset; - result.m_offsetEnd = offset + size; - return result; - } + FileRange result; + result.m_hasOffsetEndSet = true; + result.m_isEntireFile = false; + result.m_offsetBegin = offset; + result.m_offsetEnd = offset + size; + return result; + } - FileRange FileRange::CreateRangeForEntireFile() - { - FileRange result; - result.m_hasOffsetEndSet = false; - result.m_isEntireFile = true; - result.m_offsetBegin = 0; - result.m_offsetEnd = (static_cast(1) << 63) - 1; - return result; - } + FileRange FileRange::CreateRangeForEntireFile() + { + FileRange result; + result.m_hasOffsetEndSet = false; + result.m_isEntireFile = true; + result.m_offsetBegin = 0; + result.m_offsetEnd = (static_cast(1) << 63) - 1; + return result; + } - FileRange FileRange::CreateRangeForEntireFile(u64 fileSize) - { - FileRange result; - result.m_hasOffsetEndSet = true; - result.m_isEntireFile = true; - result.m_offsetBegin = 0; - result.m_offsetEnd = fileSize; - return result; - } + FileRange FileRange::CreateRangeForEntireFile(u64 fileSize) + { + FileRange result; + result.m_hasOffsetEndSet = true; + result.m_isEntireFile = true; + result.m_offsetBegin = 0; + result.m_offsetEnd = fileSize; + return result; + } - FileRange::FileRange() - : m_isEntireFile(false) - , m_offsetBegin(0) - , m_hasOffsetEndSet(false) - , m_offsetEnd(0) - { - } + FileRange::FileRange() + : m_isEntireFile(false) + , m_offsetBegin(0) + , m_hasOffsetEndSet(false) + , m_offsetEnd(0) + { + } - bool FileRange::operator==(const FileRange& rhs) const + bool FileRange::operator==(const FileRange& rhs) const + { + if (m_isEntireFile) { - if (m_isEntireFile) - { - return rhs.m_isEntireFile && m_offsetBegin == rhs.m_offsetBegin; - } - else - { - return m_offsetBegin == rhs.m_offsetBegin && m_offsetEnd == rhs.m_offsetEnd; - } + return rhs.m_isEntireFile && m_offsetBegin == rhs.m_offsetBegin; } + else + { + return m_offsetBegin == rhs.m_offsetBegin && m_offsetEnd == rhs.m_offsetEnd; + } + } - bool FileRange::operator!=(const FileRange& rhs) const + bool FileRange::operator!=(const FileRange& rhs) const + { + if (m_isEntireFile) { - if (m_isEntireFile) - { - return !rhs.m_isEntireFile || m_offsetBegin != rhs.m_offsetBegin; - } - else - { - return m_offsetBegin != rhs.m_offsetBegin || m_offsetEnd != rhs.m_offsetEnd; - } + return !rhs.m_isEntireFile || m_offsetBegin != rhs.m_offsetBegin; } + else + { + return m_offsetBegin != rhs.m_offsetBegin || m_offsetEnd != rhs.m_offsetEnd; + } + } - bool FileRange::IsEntireFile() const - { - return m_isEntireFile != 0; - } + bool FileRange::IsEntireFile() const + { + return m_isEntireFile != 0; + } - bool FileRange::IsSizeKnown() const - { - // m_hasOffsetEndSet being zero has the special meaning that the file size has not - // specifically been set yet. - return m_hasOffsetEndSet != 0; - } + bool FileRange::IsSizeKnown() const + { + // m_hasOffsetEndSet being zero has the special meaning that the file size has not + // specifically been set yet. + return m_hasOffsetEndSet != 0; + } - bool FileRange::IsInRange(u64 offset) const - { - return m_offsetBegin <= offset && offset < m_offsetEnd; - } + bool FileRange::IsInRange(u64 offset) const + { + return m_offsetBegin <= offset && offset < m_offsetEnd; + } - u64 FileRange::GetOffset() const - { - return m_offsetBegin; - } + u64 FileRange::GetOffset() const + { + return m_offsetBegin; + } - u64 FileRange::GetSize() const - { - AZ_Assert(m_hasOffsetEndSet, "Calling GetSize on a FileRange that doesn't have a size specified."); - return m_offsetEnd - m_offsetBegin; - } + u64 FileRange::GetSize() const + { + AZ_Assert(m_hasOffsetEndSet, "Calling GetSize on a FileRange that doesn't have a size specified."); + return m_offsetEnd - m_offsetBegin; + } - u64 FileRange::GetEndPoint() const - { - AZ_Assert(m_hasOffsetEndSet, "Calling GetEndPoint on a FileRange that doesn't have an end offset specified."); - return m_offsetEnd; - } - } // namespace IO -} // namesapce AZ + u64 FileRange::GetEndPoint() const + { + AZ_Assert(m_hasOffsetEndSet, "Calling GetEndPoint on a FileRange that doesn't have an end offset specified."); + return m_offsetEnd; + } +} // namespace AZ::IO diff --git a/Code/Framework/AzCore/AzCore/IO/Streamer/FileRequest.cpp b/Code/Framework/AzCore/AzCore/IO/Streamer/FileRequest.cpp index 7b9cde76d3..fc05b77b36 100644 --- a/Code/Framework/AzCore/AzCore/IO/Streamer/FileRequest.cpp +++ b/Code/Framework/AzCore/AzCore/IO/Streamer/FileRequest.cpp @@ -12,469 +12,466 @@ #include #include -namespace AZ +namespace AZ::IO { - namespace IO + // + // Command structures. + // + + FileRequest::ExternalRequestData::ExternalRequestData(FileRequestPtr&& request) + : m_request(AZStd::move(request)) + {} + + FileRequest::RequestPathStoreData::RequestPathStoreData(RequestPath path) + : m_path(AZStd::move(path)) + {} + + FileRequest::ReadRequestData::ReadRequestData(RequestPath path, void* output, u64 outputSize, u64 offset, u64 size, + AZStd::chrono::system_clock::time_point deadline, IStreamerTypes::Priority priority) + : m_path(AZStd::move(path)) + , m_allocator(nullptr) + , m_deadline(deadline) + , m_output(output) + , m_outputSize(outputSize) + , m_offset(offset) + , m_size(size) + , m_priority(priority) + , m_memoryType(IStreamerTypes::MemoryType::ReadWrite) // Only generic memory can be assigned externally. + {} + + FileRequest::ReadRequestData::ReadRequestData(RequestPath path, IStreamerTypes::RequestMemoryAllocator* allocator, + u64 offset, u64 size, AZStd::chrono::system_clock::time_point deadline, IStreamerTypes::Priority priority) + : m_path(AZStd::move(path)) + , m_allocator(allocator) + , m_deadline(deadline) + , m_output(nullptr) + , m_outputSize(0) + , m_offset(offset) + , m_size(size) + , m_priority(priority) + , m_memoryType(IStreamerTypes::MemoryType::ReadWrite) // Only generic memory can be assigned externally. + {} + + FileRequest::ReadRequestData::~ReadRequestData() { - // - // Command structures. - // - - FileRequest::ExternalRequestData::ExternalRequestData(FileRequestPtr&& request) - : m_request(AZStd::move(request)) - {} - - FileRequest::RequestPathStoreData::RequestPathStoreData(RequestPath path) - : m_path(AZStd::move(path)) - {} - - FileRequest::ReadRequestData::ReadRequestData(RequestPath path, void* output, u64 outputSize, u64 offset, u64 size, - AZStd::chrono::system_clock::time_point deadline, IStreamerTypes::Priority priority) - : m_path(AZStd::move(path)) - , m_allocator(nullptr) - , m_deadline(deadline) - , m_output(output) - , m_outputSize(outputSize) - , m_offset(offset) - , m_size(size) - , m_priority(priority) - , m_memoryType(IStreamerTypes::MemoryType::ReadWrite) // Only generic memory can be assigned externally. - {} - - FileRequest::ReadRequestData::ReadRequestData(RequestPath path, IStreamerTypes::RequestMemoryAllocator* allocator, - u64 offset, u64 size, AZStd::chrono::system_clock::time_point deadline, IStreamerTypes::Priority priority) - : m_path(AZStd::move(path)) - , m_allocator(allocator) - , m_deadline(deadline) - , m_output(nullptr) - , m_outputSize(0) - , m_offset(offset) - , m_size(size) - , m_priority(priority) - , m_memoryType(IStreamerTypes::MemoryType::ReadWrite) // Only generic memory can be assigned externally. - {} - - FileRequest::ReadRequestData::~ReadRequestData() + if (m_allocator != nullptr) { - if (m_allocator != nullptr) + if (m_output != nullptr) { - if (m_output != nullptr) - { - m_allocator->Release(m_output); - } - m_allocator->UnlockAllocator(); + m_allocator->Release(m_output); } + m_allocator->UnlockAllocator(); } + } - FileRequest::ReadData::ReadData(void* output, u64 outputSize, const RequestPath& path, u64 offset, u64 size, bool sharedRead) - : m_output(output) - , m_outputSize(outputSize) - , m_path(path) - , m_offset(offset) - , m_size(size) - , m_sharedRead(sharedRead) - {} + FileRequest::ReadData::ReadData(void* output, u64 outputSize, const RequestPath& path, u64 offset, u64 size, bool sharedRead) + : m_output(output) + , m_outputSize(outputSize) + , m_path(path) + , m_offset(offset) + , m_size(size) + , m_sharedRead(sharedRead) + {} - FileRequest::CompressedReadData::CompressedReadData(CompressionInfo&& compressionInfo, void* output, u64 readOffset, u64 readSize) - : m_compressionInfo(AZStd::move(compressionInfo)) - , m_output(output) - , m_readOffset(readOffset) - , m_readSize(readSize) - {} + FileRequest::CompressedReadData::CompressedReadData(CompressionInfo&& compressionInfo, void* output, u64 readOffset, u64 readSize) + : m_compressionInfo(AZStd::move(compressionInfo)) + , m_output(output) + , m_readOffset(readOffset) + , m_readSize(readSize) + {} - FileRequest::FileExistsCheckData::FileExistsCheckData(const RequestPath& path) - : m_path(path) - {} + FileRequest::FileExistsCheckData::FileExistsCheckData(const RequestPath& path) + : m_path(path) + {} - FileRequest::FileMetaDataRetrievalData::FileMetaDataRetrievalData(const RequestPath& path) - : m_path(path) - {} + FileRequest::FileMetaDataRetrievalData::FileMetaDataRetrievalData(const RequestPath& path) + : m_path(path) + {} - FileRequest::CancelData::CancelData(FileRequestPtr target) - : m_target(AZStd::move(target)) - {} + FileRequest::CancelData::CancelData(FileRequestPtr target) + : m_target(AZStd::move(target)) + {} - FileRequest::FlushData::FlushData(RequestPath path) - : m_path(AZStd::move(path)) - {} + FileRequest::FlushData::FlushData(RequestPath path) + : m_path(AZStd::move(path)) + {} - FileRequest::RescheduleData::RescheduleData(FileRequestPtr target, AZStd::chrono::system_clock::time_point newDeadline, - IStreamerTypes::Priority newPriority) - : m_target(AZStd::move(target)) - , m_newDeadline(newDeadline) - , m_newPriority(newPriority) - {} + FileRequest::RescheduleData::RescheduleData(FileRequestPtr target, AZStd::chrono::system_clock::time_point newDeadline, + IStreamerTypes::Priority newPriority) + : m_target(AZStd::move(target)) + , m_newDeadline(newDeadline) + , m_newPriority(newPriority) + {} - FileRequest::CreateDedicatedCacheData::CreateDedicatedCacheData(RequestPath path, const FileRange& range) - : m_path(AZStd::move(path)) - , m_range(range) - {} + FileRequest::CreateDedicatedCacheData::CreateDedicatedCacheData(RequestPath path, const FileRange& range) + : m_path(AZStd::move(path)) + , m_range(range) + {} - FileRequest::DestroyDedicatedCacheData::DestroyDedicatedCacheData(RequestPath path, const FileRange& range) - : m_path(AZStd::move(path)) - , m_range(range) - {} + FileRequest::DestroyDedicatedCacheData::DestroyDedicatedCacheData(RequestPath path, const FileRange& range) + : m_path(AZStd::move(path)) + , m_range(range) + {} - FileRequest::ReportData::ReportData(ReportType reportType) - : m_reportType(reportType) - {} + FileRequest::ReportData::ReportData(ReportType reportType) + : m_reportType(reportType) + {} - FileRequest::CustomData::CustomData(AZStd::any data, bool failWhenUnhandled) - : m_data(AZStd::move(data)) - , m_failWhenUnhandled(failWhenUnhandled) - {} + FileRequest::CustomData::CustomData(AZStd::any data, bool failWhenUnhandled) + : m_data(AZStd::move(data)) + , m_failWhenUnhandled(failWhenUnhandled) + {} - // - // FileRequest - // + // + // FileRequest + // - FileRequest::FileRequest(Usage usage) - : m_usage(usage) + FileRequest::FileRequest(Usage usage) + : m_usage(usage) + { + Reset(); + } + + FileRequest::~FileRequest() + { + Reset(); + } + + void FileRequest::CreateRequestLink(FileRequestPtr&& request) + { + AZ_Assert(AZStd::holds_alternative(m_command), + "Attempting to set FileRequest to 'RequestLink', but another task was already assigned."); + m_parent = request->m_request.m_parent; + request->m_request.m_parent = this; + m_dependencies++; + m_command.emplace(AZStd::move(request)); + } + + void FileRequest::CreateRequestPathStore(FileRequest* parent, RequestPath path) + { + AZ_Assert(AZStd::holds_alternative(m_command), + "Attempting to set FileRequest to 'CreateRequestPathStore', but another task was already assigned."); + m_command.emplace(AZStd::move(path)); + SetOptionalParent(parent); + } + + void FileRequest::CreateReadRequest(RequestPath path, void* output, u64 outputSize, u64 offset, u64 size, + AZStd::chrono::system_clock::time_point deadline, IStreamerTypes::Priority priority) + { + AZ_Assert(AZStd::holds_alternative(m_command), + "Attempting to set FileRequest to 'ReadRequest', but another task was already assigned."); + m_command.emplace(AZStd::move(path), output, outputSize, offset, size, deadline, priority); + } + + void FileRequest::CreateReadRequest(RequestPath path, IStreamerTypes::RequestMemoryAllocator* allocator, u64 offset, u64 size, + AZStd::chrono::system_clock::time_point deadline, IStreamerTypes::Priority priority) + { + AZ_Assert(AZStd::holds_alternative(m_command), + "Attempting to set FileRequest to 'ReadRequest', but another task was already assigned."); + m_command.emplace(AZStd::move(path), allocator, offset, size, deadline, priority); + } + + void FileRequest::CreateRead(FileRequest* parent, void* output, u64 outputSize, const RequestPath& path, + u64 offset, u64 size, bool sharedRead) + { + AZ_Assert(AZStd::holds_alternative(m_command), + "Attempting to set FileRequest to 'Read', but another task was already assigned."); + m_command.emplace(output, outputSize, AZStd::move(path), offset, size, sharedRead); + SetOptionalParent(parent); + } + + void FileRequest::CreateCompressedRead(FileRequest* parent, const CompressionInfo& compressionInfo, + void* output, u64 readOffset, u64 readSize) + { + CreateCompressedRead(parent, CompressionInfo(compressionInfo), output, readOffset, readSize); + } + + void FileRequest::CreateCompressedRead(FileRequest* parent, CompressionInfo&& compressionInfo, + void* output, u64 readOffset, u64 readSize) + { + AZ_Assert(AZStd::holds_alternative(m_command), + "Attempting to set FileRequest to 'CompressedRead', but another task was already assigned."); + m_command.emplace(AZStd::move(compressionInfo), output, readOffset, readSize); + SetOptionalParent(parent); + } + + void FileRequest::CreateWait(FileRequest* parent) + { + AZ_Assert(AZStd::holds_alternative(m_command), + "Attempting to set FileRequest to 'Wait', but another task was already assigned."); + m_command.emplace(); + SetOptionalParent(parent); + } + + void FileRequest::CreateFileExistsCheck(const RequestPath& path) + { + AZ_Assert(AZStd::holds_alternative(m_command), + "Attempting to set FileRequest to 'FileExistsCheck', but another task was already assigned."); + m_command.emplace(path); + } + + void FileRequest::CreateFileMetaDataRetrieval(const RequestPath& path) + { + AZ_Assert(AZStd::holds_alternative(m_command), + "Attempting to set FileRequest to 'FileMetaDataRetrieval', but another task was already assigned."); + m_command.emplace(path); + } + + void FileRequest::CreateCancel(FileRequestPtr target) + { + AZ_Assert(AZStd::holds_alternative(m_command), + "Attempting to set FileRequest to 'Cancel', but another task was already assigned."); + m_command.emplace(AZStd::move(target)); + } + + void FileRequest::CreateReschedule(FileRequestPtr target, AZStd::chrono::system_clock::time_point newDeadline, + IStreamerTypes::Priority newPriority) + { + AZ_Assert(AZStd::holds_alternative(m_command), + "Attempting to set FileRequest to 'Reschedule', but another task was already assigned."); + m_command.emplace(AZStd::move(target), newDeadline, newPriority); + } + + void FileRequest::CreateFlush(RequestPath path) + { + AZ_Assert(AZStd::holds_alternative(m_command), + "Attempting to set FileRequest to 'Flush', but another task was already assigned."); + m_command.emplace(AZStd::move(path)); + } + + void FileRequest::CreateFlushAll() + { + AZ_Assert(AZStd::holds_alternative(m_command), + "Attempting to set FileRequest to 'FlushAll', but another task was already assigned."); + m_command.emplace(); + } + + void FileRequest::CreateDedicatedCacheCreation(RequestPath path, const FileRange& range, FileRequest* parent) + { + AZ_Assert(AZStd::holds_alternative(m_command), + "Attempting to set FileRequest to 'CreateDedicateCache', but another task was already assigned."); + m_command.emplace(AZStd::move(path), range); + SetOptionalParent(parent); + } + + void FileRequest::CreateDedicatedCacheDestruction(RequestPath path, const FileRange& range, FileRequest* parent) + { + AZ_Assert(AZStd::holds_alternative(m_command), + "Attempting to set FileRequest to 'DestroyDedicateCache', but another task was already assigned."); + m_command.emplace(AZStd::move(path), range); + SetOptionalParent(parent); + } + + void FileRequest::CreateReport(ReportData::ReportType reportType) + { + AZ_Assert(AZStd::holds_alternative(m_command), + "Attempting to set FileRequest to 'Report', but another task was already assigned."); + m_command.emplace(reportType); + } + + void FileRequest::CreateCustom(AZStd::any data, bool failWhenUnhandled, FileRequest* parent) + { + AZ_Assert(AZStd::holds_alternative(m_command), + "Attempting to set FileRequest to 'Custom', but another task was already assigned."); + m_command.emplace(AZStd::move(data), failWhenUnhandled); + SetOptionalParent(parent); + } + + void FileRequest::SetCompletionCallback(OnCompletionCallback callback) + { + m_onCompletion = AZStd::move(callback); + } + + FileRequest::CommandVariant& FileRequest::GetCommand() + { + return m_command; + } + + const FileRequest::CommandVariant& FileRequest::GetCommand() const + { + return m_command; + } + + IStreamerTypes::RequestStatus FileRequest::GetStatus() const + { + return m_status; + } + + void FileRequest::SetStatus(IStreamerTypes::RequestStatus newStatus) + { + IStreamerTypes::RequestStatus currentStatus = m_status; + switch (newStatus) { - Reset(); - } - - FileRequest::~FileRequest() - { - Reset(); - } - - void FileRequest::CreateRequestLink(FileRequestPtr&& request) - { - AZ_Assert(AZStd::holds_alternative(m_command), - "Attempting to set FileRequest to 'RequestLink', but another task was already assigned."); - m_parent = request->m_request.m_parent; - request->m_request.m_parent = this; - m_dependencies++; - m_command.emplace(AZStd::move(request)); - } - - void FileRequest::CreateRequestPathStore(FileRequest* parent, RequestPath path) - { - AZ_Assert(AZStd::holds_alternative(m_command), - "Attempting to set FileRequest to 'CreateRequestPathStore', but another task was already assigned."); - m_command.emplace(AZStd::move(path)); - SetOptionalParent(parent); - } - - void FileRequest::CreateReadRequest(RequestPath path, void* output, u64 outputSize, u64 offset, u64 size, - AZStd::chrono::system_clock::time_point deadline, IStreamerTypes::Priority priority) - { - AZ_Assert(AZStd::holds_alternative(m_command), - "Attempting to set FileRequest to 'ReadRequest', but another task was already assigned."); - m_command.emplace(AZStd::move(path), output, outputSize, offset, size, deadline, priority); - } - - void FileRequest::CreateReadRequest(RequestPath path, IStreamerTypes::RequestMemoryAllocator* allocator, u64 offset, u64 size, - AZStd::chrono::system_clock::time_point deadline, IStreamerTypes::Priority priority) - { - AZ_Assert(AZStd::holds_alternative(m_command), - "Attempting to set FileRequest to 'ReadRequest', but another task was already assigned."); - m_command.emplace(AZStd::move(path), allocator, offset, size, deadline, priority); - } - - void FileRequest::CreateRead(FileRequest* parent, void* output, u64 outputSize, const RequestPath& path, - u64 offset, u64 size, bool sharedRead) - { - AZ_Assert(AZStd::holds_alternative(m_command), - "Attempting to set FileRequest to 'Read', but another task was already assigned."); - m_command.emplace(output, outputSize, AZStd::move(path), offset, size, sharedRead); - SetOptionalParent(parent); - } - - void FileRequest::CreateCompressedRead(FileRequest* parent, const CompressionInfo& compressionInfo, - void* output, u64 readOffset, u64 readSize) - { - CreateCompressedRead(parent, CompressionInfo(compressionInfo), output, readOffset, readSize); - } - - void FileRequest::CreateCompressedRead(FileRequest* parent, CompressionInfo&& compressionInfo, - void* output, u64 readOffset, u64 readSize) - { - AZ_Assert(AZStd::holds_alternative(m_command), - "Attempting to set FileRequest to 'CompressedRead', but another task was already assigned."); - m_command.emplace(AZStd::move(compressionInfo), output, readOffset, readSize); - SetOptionalParent(parent); - } - - void FileRequest::CreateWait(FileRequest* parent) - { - AZ_Assert(AZStd::holds_alternative(m_command), - "Attempting to set FileRequest to 'Wait', but another task was already assigned."); - m_command.emplace(); - SetOptionalParent(parent); - } - - void FileRequest::CreateFileExistsCheck(const RequestPath& path) - { - AZ_Assert(AZStd::holds_alternative(m_command), - "Attempting to set FileRequest to 'FileExistsCheck', but another task was already assigned."); - m_command.emplace(path); - } - - void FileRequest::CreateFileMetaDataRetrieval(const RequestPath& path) - { - AZ_Assert(AZStd::holds_alternative(m_command), - "Attempting to set FileRequest to 'FileMetaDataRetrieval', but another task was already assigned."); - m_command.emplace(path); - } - - void FileRequest::CreateCancel(FileRequestPtr target) - { - AZ_Assert(AZStd::holds_alternative(m_command), - "Attempting to set FileRequest to 'Cancel', but another task was already assigned."); - m_command.emplace(AZStd::move(target)); - } - - void FileRequest::CreateReschedule(FileRequestPtr target, AZStd::chrono::system_clock::time_point newDeadline, - IStreamerTypes::Priority newPriority) - { - AZ_Assert(AZStd::holds_alternative(m_command), - "Attempting to set FileRequest to 'Reschedule', but another task was already assigned."); - m_command.emplace(AZStd::move(target), newDeadline, newPriority); - } - - void FileRequest::CreateFlush(RequestPath path) - { - AZ_Assert(AZStd::holds_alternative(m_command), - "Attempting to set FileRequest to 'Flush', but another task was already assigned."); - m_command.emplace(AZStd::move(path)); - } - - void FileRequest::CreateFlushAll() - { - AZ_Assert(AZStd::holds_alternative(m_command), - "Attempting to set FileRequest to 'FlushAll', but another task was already assigned."); - m_command.emplace(); - } - - void FileRequest::CreateDedicatedCacheCreation(RequestPath path, const FileRange& range, FileRequest* parent) - { - AZ_Assert(AZStd::holds_alternative(m_command), - "Attempting to set FileRequest to 'CreateDedicateCache', but another task was already assigned."); - m_command.emplace(AZStd::move(path), range); - SetOptionalParent(parent); - } - - void FileRequest::CreateDedicatedCacheDestruction(RequestPath path, const FileRange& range, FileRequest* parent) - { - AZ_Assert(AZStd::holds_alternative(m_command), - "Attempting to set FileRequest to 'DestroyDedicateCache', but another task was already assigned."); - m_command.emplace(AZStd::move(path), range); - SetOptionalParent(parent); - } - - void FileRequest::CreateReport(ReportData::ReportType reportType) - { - AZ_Assert(AZStd::holds_alternative(m_command), - "Attempting to set FileRequest to 'Report', but another task was already assigned."); - m_command.emplace(reportType); - } - - void FileRequest::CreateCustom(AZStd::any data, bool failWhenUnhandled, FileRequest* parent) - { - AZ_Assert(AZStd::holds_alternative(m_command), - "Attempting to set FileRequest to 'Custom', but another task was already assigned."); - m_command.emplace(AZStd::move(data), failWhenUnhandled); - SetOptionalParent(parent); - } - - void FileRequest::SetCompletionCallback(OnCompletionCallback callback) - { - m_onCompletion = AZStd::move(callback); - } - - FileRequest::CommandVariant& FileRequest::GetCommand() - { - return m_command; - } - - const FileRequest::CommandVariant& FileRequest::GetCommand() const - { - return m_command; - } - - IStreamerTypes::RequestStatus FileRequest::GetStatus() const - { - return m_status; - } - - void FileRequest::SetStatus(IStreamerTypes::RequestStatus newStatus) - { - IStreamerTypes::RequestStatus currentStatus = m_status; - switch (newStatus) + case IStreamerTypes::RequestStatus::Pending: + [[fallthrough]]; + case IStreamerTypes::RequestStatus::Queued: + [[fallthrough]]; + case IStreamerTypes::RequestStatus::Processing: + if (currentStatus == IStreamerTypes::RequestStatus::Failed || + currentStatus == IStreamerTypes::RequestStatus::Canceled || + currentStatus == IStreamerTypes::RequestStatus::Completed) { - case IStreamerTypes::RequestStatus::Pending: - [[fallthrough]]; - case IStreamerTypes::RequestStatus::Queued: - [[fallthrough]]; - case IStreamerTypes::RequestStatus::Processing: - if (currentStatus == IStreamerTypes::RequestStatus::Failed || - currentStatus == IStreamerTypes::RequestStatus::Canceled || - currentStatus == IStreamerTypes::RequestStatus::Completed) - { - return; - } - break; - case IStreamerTypes::RequestStatus::Completed: - if (currentStatus == IStreamerTypes::RequestStatus::Failed || currentStatus == IStreamerTypes::RequestStatus::Canceled) - { - return; - } - break; - case IStreamerTypes::RequestStatus::Canceled: - if (currentStatus == IStreamerTypes::RequestStatus::Failed || currentStatus == IStreamerTypes::RequestStatus::Completed) - { - return; - } - break; - case IStreamerTypes::RequestStatus::Failed: - [[fallthrough]]; - default: - break; + return; } - m_status = newStatus; - } - - FileRequest* FileRequest::GetParent() - { - return m_parent; - } - - const FileRequest* FileRequest::GetParent() const - { - return m_parent; - } - - size_t FileRequest::GetNumDependencies() const - { - return m_dependencies; - } - - bool FileRequest::FailsWhenUnhandled() const - { - return AZStd::visit([](auto&& args) + break; + case IStreamerTypes::RequestStatus::Completed: + if (currentStatus == IStreamerTypes::RequestStatus::Failed || currentStatus == IStreamerTypes::RequestStatus::Canceled) { - using Command = AZStd::decay_t; - if constexpr (AZStd::is_same_v) - { - AZ_Assert(false, - "Request does not contain a valid command. It may have been reset already or was never assigned a command."); - return true; - } - else if constexpr (AZStd::is_same_v) - { - return args.m_failWhenUnhandled; - } - else - { - return Command::s_failWhenUnhandled; - } - }, m_command); - } - - void FileRequest::Reset() - { - m_command = AZStd::monostate{}; - m_onCompletion = &OnCompletionPlaceholder; - m_estimatedCompletion = AZStd::chrono::system_clock::time_point(); - m_parent = nullptr; - m_status = IStreamerTypes::RequestStatus::Pending; - m_dependencies = 0; - } - - void FileRequest::SetOptionalParent(FileRequest* parent) - { - if (parent) - { - m_parent = parent; - AZ_Assert(parent->m_dependencies < std::numeric_limitsm_dependencies)>::max(), - "A file request dependency was added, but the parent can't have any more dependencies."); - ++parent->m_dependencies; + return; } - } - - bool FileRequest::WorksOn(FileRequestPtr& request) const - { - const FileRequest* current = this; - while (current) + break; + case IStreamerTypes::RequestStatus::Canceled: + if (currentStatus == IStreamerTypes::RequestStatus::Failed || currentStatus == IStreamerTypes::RequestStatus::Completed) { - auto* link = AZStd::get_if(¤t->m_command); - if (!link) - { - current = current->m_parent; - } - else - { - return link->m_request == request; - } + return; } - return false; + break; + case IStreamerTypes::RequestStatus::Failed: + [[fallthrough]]; + default: + break; } + m_status = newStatus; + } - size_t FileRequest::GetPendingId() const - { - return m_pendingId; - } + FileRequest* FileRequest::GetParent() + { + return m_parent; + } - void FileRequest::SetEstimatedCompletion(AZStd::chrono::system_clock::time_point time) + const FileRequest* FileRequest::GetParent() const + { + return m_parent; + } + + size_t FileRequest::GetNumDependencies() const + { + return m_dependencies; + } + + bool FileRequest::FailsWhenUnhandled() const + { + return AZStd::visit([](auto&& args) { - FileRequest* current = this; - do + using Command = AZStd::decay_t; + if constexpr (AZStd::is_same_v) + { + AZ_Assert(false, + "Request does not contain a valid command. It may have been reset already or was never assigned a command."); + return true; + } + else if constexpr (AZStd::is_same_v) + { + return args.m_failWhenUnhandled; + } + else + { + return Command::s_failWhenUnhandled; + } + }, m_command); + } + + void FileRequest::Reset() + { + m_command = AZStd::monostate{}; + m_onCompletion = &OnCompletionPlaceholder; + m_estimatedCompletion = AZStd::chrono::system_clock::time_point(); + m_parent = nullptr; + m_status = IStreamerTypes::RequestStatus::Pending; + m_dependencies = 0; + } + + void FileRequest::SetOptionalParent(FileRequest* parent) + { + if (parent) + { + m_parent = parent; + AZ_Assert(parent->m_dependencies < std::numeric_limitsm_dependencies)>::max(), + "A file request dependency was added, but the parent can't have any more dependencies."); + ++parent->m_dependencies; + } + } + + bool FileRequest::WorksOn(FileRequestPtr& request) const + { + const FileRequest* current = this; + while (current) + { + auto* link = AZStd::get_if(¤t->m_command); + if (!link) { - current->m_estimatedCompletion = time; current = current->m_parent; - } while (current); - } - - AZStd::chrono::system_clock::time_point FileRequest::GetEstimatedCompletion() const - { - return m_estimatedCompletion; - } - - // - // ExternalFileRequest - // - - ExternalFileRequest::ExternalFileRequest(StreamerContext* owner) - : m_request(FileRequest::Usage::External) - , m_owner(owner) - { - } - - void ExternalFileRequest::add_ref() - { - m_refCount++; - } - - void ExternalFileRequest::release() - { - if (--m_refCount == 0) + } + else { - AZ_Assert(m_owner, "No owning context set for the file request."); - m_owner->RecycleRequest(this); + return link->m_request == request; } } + return false; + } - bool operator==(const FileRequestHandle& lhs, const FileRequestPtr& rhs) - { - return lhs.m_request == &rhs->m_request; - } + size_t FileRequest::GetPendingId() const + { + return m_pendingId; + } - bool operator==(const FileRequestPtr& lhs, const FileRequestHandle& rhs) + void FileRequest::SetEstimatedCompletion(AZStd::chrono::system_clock::time_point time) + { + FileRequest* current = this; + do { - return rhs == lhs; - } + current->m_estimatedCompletion = time; + current = current->m_parent; + } while (current); + } - bool operator!=(const FileRequestHandle& lhs, const FileRequestPtr& rhs) - { - return !(lhs == rhs); - } + AZStd::chrono::system_clock::time_point FileRequest::GetEstimatedCompletion() const + { + return m_estimatedCompletion; + } - bool operator!=(const FileRequestPtr& lhs, const FileRequestHandle& rhs) + // + // ExternalFileRequest + // + + ExternalFileRequest::ExternalFileRequest(StreamerContext* owner) + : m_request(FileRequest::Usage::External) + , m_owner(owner) + { + } + + void ExternalFileRequest::add_ref() + { + m_refCount++; + } + + void ExternalFileRequest::release() + { + if (--m_refCount == 0) { - return !(rhs == lhs); + AZ_Assert(m_owner, "No owning context set for the file request."); + m_owner->RecycleRequest(this); } - } // namespace IO -} // namespace AZ + } + + bool operator==(const FileRequestHandle& lhs, const FileRequestPtr& rhs) + { + return lhs.m_request == &rhs->m_request; + } + + bool operator==(const FileRequestPtr& lhs, const FileRequestHandle& rhs) + { + return rhs == lhs; + } + + bool operator!=(const FileRequestHandle& lhs, const FileRequestPtr& rhs) + { + return !(lhs == rhs); + } + + bool operator!=(const FileRequestPtr& lhs, const FileRequestHandle& rhs) + { + return !(rhs == lhs); + } +} // namespace AZ::IO diff --git a/Code/Framework/AzCore/AzCore/IO/Streamer/FullFileDecompressor.cpp b/Code/Framework/AzCore/AzCore/IO/Streamer/FullFileDecompressor.cpp index 797f96e53f..723a5d62c8 100644 --- a/Code/Framework/AzCore/AzCore/IO/Streamer/FullFileDecompressor.cpp +++ b/Code/Framework/AzCore/AzCore/IO/Streamer/FullFileDecompressor.cpp @@ -21,720 +21,717 @@ #include #include -namespace AZ +namespace AZ::IO { - namespace IO + AZStd::shared_ptr FullFileDecompressorConfig::AddStreamStackEntry( + const HardwareInformation& hardware, AZStd::shared_ptr parent) { - AZStd::shared_ptr FullFileDecompressorConfig::AddStreamStackEntry( - const HardwareInformation& hardware, AZStd::shared_ptr parent) - { - auto stackEntry = AZStd::make_shared( - m_maxNumReads, m_maxNumJobs, aznumeric_caster(hardware.m_maxPhysicalSectorSize)); - stackEntry->SetNext(AZStd::move(parent)); - return stackEntry; - } + auto stackEntry = AZStd::make_shared( + m_maxNumReads, m_maxNumJobs, aznumeric_caster(hardware.m_maxPhysicalSectorSize)); + stackEntry->SetNext(AZStd::move(parent)); + return stackEntry; + } - void FullFileDecompressorConfig::Reflect(AZ::ReflectContext* context) + void FullFileDecompressorConfig::Reflect(AZ::ReflectContext* context) + { + if (auto serializeContext = azrtti_cast(context); serializeContext != nullptr) { - if (auto serializeContext = azrtti_cast(context); serializeContext != nullptr) - { - serializeContext->Class() - ->Version(1) - ->Field("MaxNumReads", &FullFileDecompressorConfig::m_maxNumReads) - ->Field("MaxNumJobs", &FullFileDecompressorConfig::m_maxNumJobs); - } + serializeContext->Class() + ->Version(1) + ->Field("MaxNumReads", &FullFileDecompressorConfig::m_maxNumReads) + ->Field("MaxNumJobs", &FullFileDecompressorConfig::m_maxNumJobs); } + } #if AZ_STREAMER_ADD_EXTRA_PROFILING_INFO - static constexpr char DecompBoundName[] = "Decompression bound"; - static constexpr char ReadBoundName[] = "Read bound"; + static constexpr char DecompBoundName[] = "Decompression bound"; + static constexpr char ReadBoundName[] = "Read bound"; #endif // AZ_STREAMER_ADD_EXTRA_PROFILING_INFO - bool FullFileDecompressor::DecompressionInformation::IsProcessing() const - { - return !!m_compressedData; - } - - FullFileDecompressor::FullFileDecompressor(u32 maxNumReads, u32 maxNumJobs, u32 alignment) - : StreamStackEntry("Full file decompressor") - , m_maxNumReads(maxNumReads) - , m_maxNumJobs(maxNumJobs) - , m_alignment(alignment) - { - JobManagerDesc jobDesc; + bool FullFileDecompressor::DecompressionInformation::IsProcessing() const + { + return !!m_compressedData; + } + + FullFileDecompressor::FullFileDecompressor(u32 maxNumReads, u32 maxNumJobs, u32 alignment) + : StreamStackEntry("Full file decompressor") + , m_maxNumReads(maxNumReads) + , m_maxNumJobs(maxNumJobs) + , m_alignment(alignment) + { + JobManagerDesc jobDesc; jobDesc.m_jobManagerName = "Full File Decompressor"; - u32 numThreads = AZ::GetMin(maxNumJobs, AZStd::thread::hardware_concurrency()); - for (u32 i = 0; i < numThreads; ++i) - { - jobDesc.m_workerThreads.push_back(JobManagerThreadDesc()); - } - m_decompressionJobManager = AZStd::make_unique(jobDesc); - m_decompressionjobContext = AZStd::make_unique(*m_decompressionJobManager); + u32 numThreads = AZ::GetMin(maxNumJobs, AZStd::thread::hardware_concurrency()); + for (u32 i = 0; i < numThreads; ++i) + { + jobDesc.m_workerThreads.push_back(JobManagerThreadDesc()); + } + m_decompressionJobManager = AZStd::make_unique(jobDesc); + m_decompressionjobContext = AZStd::make_unique(*m_decompressionJobManager); - m_processingJobs = AZStd::make_unique(maxNumJobs); + m_processingJobs = AZStd::make_unique(maxNumJobs); - m_readBuffers = AZStd::make_unique(maxNumReads); - m_readRequests = AZStd::make_unique(maxNumReads); - m_readBufferStatus = AZStd::make_unique(maxNumReads); - for (u32 i = 0; i < maxNumReads; ++i) - { - m_readBufferStatus[i] = ReadBufferStatus::Unused; - } - - // Add initial dummy values to the stats to avoid division by zero later on and avoid needing branches. - m_bytesDecompressed.PushEntry(1); - m_decompressionDurationMicroSec.PushEntry(1); + m_readBuffers = AZStd::make_unique(maxNumReads); + m_readRequests = AZStd::make_unique(maxNumReads); + m_readBufferStatus = AZStd::make_unique(maxNumReads); + for (u32 i = 0; i < maxNumReads; ++i) + { + m_readBufferStatus[i] = ReadBufferStatus::Unused; } - void FullFileDecompressor::PrepareRequest(FileRequest* request) - { - AZ_Assert(request, "PrepareRequest was provided a null request."); + // Add initial dummy values to the stats to avoid division by zero later on and avoid needing branches. + m_bytesDecompressed.PushEntry(1); + m_decompressionDurationMicroSec.PushEntry(1); + } - AZStd::visit([this, request](auto&& args) + void FullFileDecompressor::PrepareRequest(FileRequest* request) + { + AZ_Assert(request, "PrepareRequest was provided a null request."); + + AZStd::visit([this, request](auto&& args) + { + using Command = AZStd::decay_t; + if constexpr (AZStd::is_same_v) { - using Command = AZStd::decay_t; - if constexpr (AZStd::is_same_v) - { - PrepareReadRequest(request, args); - } - else if constexpr (AZStd::is_same_v || - AZStd::is_same_v) - { - PrepareDedicatedCache(request, args.m_path); - } - else - { - StreamStackEntry::PrepareRequest(request); - } - }, request->GetCommand()); + PrepareReadRequest(request, args); + } + else if constexpr (AZStd::is_same_v || + AZStd::is_same_v) + { + PrepareDedicatedCache(request, args.m_path); + } + else + { + StreamStackEntry::PrepareRequest(request); + } + }, request->GetCommand()); + } + + void FullFileDecompressor::QueueRequest(FileRequest* request) + { + AZ_Assert(request, "QueueRequest was provided a null request."); + + AZStd::visit([this, request](auto&& args) + { + using Command = AZStd::decay_t; + if constexpr (AZStd::is_same_v) + { + m_pendingReads.push_back(request); + } + else if constexpr (AZStd::is_same_v) + { + m_pendingFileExistChecks.push_back(request); + } + else + { + StreamStackEntry::QueueRequest(request); + } + }, request->GetCommand()); + } + + bool FullFileDecompressor::ExecuteRequests() + { + bool result = false; + // First queue jobs as this might open up new read slots. + if (m_numInFlightReads > 0 && m_numRunningJobs < m_maxNumJobs) + { + result = StartDecompressions(); } - void FullFileDecompressor::QueueRequest(FileRequest* request) + // Queue as many new reads as possible. + while (!m_pendingReads.empty() && m_numInFlightReads < m_maxNumReads) { - AZ_Assert(request, "QueueRequest was provided a null request."); - - AZStd::visit([this, request](auto&& args) - { - using Command = AZStd::decay_t; - if constexpr (AZStd::is_same_v) - { - m_pendingReads.push_back(request); - } - else if constexpr (AZStd::is_same_v) - { - m_pendingFileExistChecks.push_back(request); - } - else - { - StreamStackEntry::QueueRequest(request); - } - }, request->GetCommand()); + StartArchiveRead(m_pendingReads.front()); + m_pendingReads.pop_front(); + result = true; } - bool FullFileDecompressor::ExecuteRequests() + // If nothing else happened and there is at least one pending file exist check request, run one of those. + if (!result && !m_pendingFileExistChecks.empty()) { - bool result = false; - // First queue jobs as this might open up new read slots. - if (m_numInFlightReads > 0 && m_numRunningJobs < m_maxNumJobs) - { - result = StartDecompressions(); - } - - // Queue as many new reads as possible. - while (!m_pendingReads.empty() && m_numInFlightReads < m_maxNumReads) - { - StartArchiveRead(m_pendingReads.front()); - m_pendingReads.pop_front(); - result = true; - } - - // If nothing else happened and there is at least one pending file exist check request, run one of those. - if (!result && !m_pendingFileExistChecks.empty()) - { - FileExistsCheck(m_pendingFileExistChecks.front()); - m_pendingFileExistChecks.pop_front(); - result = true; - } + FileExistsCheck(m_pendingFileExistChecks.front()); + m_pendingFileExistChecks.pop_front(); + result = true; + } #if AZ_STREAMER_ADD_EXTRA_PROFILING_INFO - bool allPendingDecompression = true; - bool allReading = true; - for (u32 i = 0; i < m_maxNumReads; ++i) - { - allPendingDecompression = - allPendingDecompression && (m_readBufferStatus[i] == ReadBufferStatus::PendingDecompression); - allReading = - allReading && (m_readBufferStatus[i] == ReadBufferStatus::ReadInFlight); - } + bool allPendingDecompression = true; + bool allReading = true; + for (u32 i = 0; i < m_maxNumReads; ++i) + { + allPendingDecompression = + allPendingDecompression && (m_readBufferStatus[i] == ReadBufferStatus::PendingDecompression); + allReading = + allReading && (m_readBufferStatus[i] == ReadBufferStatus::ReadInFlight); + } - m_decompressionBoundStat.PushSample(allPendingDecompression ? 1.0 : 0.0); - Statistic::PlotImmediate(m_name, DecompBoundName, m_decompressionBoundStat.GetMostRecentSample()); + m_decompressionBoundStat.PushSample(allPendingDecompression ? 1.0 : 0.0); + Statistic::PlotImmediate(m_name, DecompBoundName, m_decompressionBoundStat.GetMostRecentSample()); - m_readBoundStat.PushSample(allReading && (m_numRunningJobs < m_maxNumJobs) ? 1.0 : 0.0); - Statistic::PlotImmediate(m_name, ReadBoundName, m_readBoundStat.GetMostRecentSample()); + m_readBoundStat.PushSample(allReading && (m_numRunningJobs < m_maxNumJobs) ? 1.0 : 0.0); + Statistic::PlotImmediate(m_name, ReadBoundName, m_readBoundStat.GetMostRecentSample()); #endif - return StreamStackEntry::ExecuteRequests() || result; - } + return StreamStackEntry::ExecuteRequests() || result; + } - void FullFileDecompressor::UpdateStatus(Status& status) const + void FullFileDecompressor::UpdateStatus(Status& status) const + { + StreamStackEntry::UpdateStatus(status); + s32 numAvailableSlots = aznumeric_cast(m_maxNumReads - m_numInFlightReads); + status.m_numAvailableSlots = AZStd::min(status.m_numAvailableSlots, numAvailableSlots); + status.m_isIdle = status.m_isIdle && IsIdle(); + } + + void FullFileDecompressor::UpdateCompletionEstimates(AZStd::chrono::system_clock::time_point now, AZStd::vector& internalPending, + StreamerContext::PreparedQueue::iterator pendingBegin, StreamerContext::PreparedQueue::iterator pendingEnd) + { + // Create predictions for all pending requests. Some will be further processed after this. + AZStd::reverse_copy(m_pendingFileExistChecks.begin(), m_pendingFileExistChecks.end(), AZStd::back_inserter(internalPending)); + AZStd::reverse_copy(m_pendingReads.begin(), m_pendingReads.end(), AZStd::back_inserter(internalPending)); + + StreamStackEntry::UpdateCompletionEstimates(now, internalPending, pendingBegin, pendingEnd); + + double totalBytesDecompressed = aznumeric_caster(m_bytesDecompressed.GetTotal()); + double totalDecompressionDuration = aznumeric_caster(m_decompressionDurationMicroSec.GetTotal()); + AZStd::chrono::microseconds cumulativeDelay = AZStd::chrono::microseconds::max(); + + // Check the number of jobs that are processing. + for (u32 i = 0; i < m_maxNumJobs; ++i) { - StreamStackEntry::UpdateStatus(status); - s32 numAvailableSlots = aznumeric_cast(m_maxNumReads - m_numInFlightReads); - status.m_numAvailableSlots = AZStd::min(status.m_numAvailableSlots, numAvailableSlots); - status.m_isIdle = status.m_isIdle && IsIdle(); - } - - void FullFileDecompressor::UpdateCompletionEstimates(AZStd::chrono::system_clock::time_point now, AZStd::vector& internalPending, - StreamerContext::PreparedQueue::iterator pendingBegin, StreamerContext::PreparedQueue::iterator pendingEnd) - { - // Create predictions for all pending requests. Some will be further processed after this. - AZStd::reverse_copy(m_pendingFileExistChecks.begin(), m_pendingFileExistChecks.end(), AZStd::back_inserter(internalPending)); - AZStd::reverse_copy(m_pendingReads.begin(), m_pendingReads.end(), AZStd::back_inserter(internalPending)); - - StreamStackEntry::UpdateCompletionEstimates(now, internalPending, pendingBegin, pendingEnd); - - double totalBytesDecompressed = aznumeric_caster(m_bytesDecompressed.GetTotal()); - double totalDecompressionDuration = aznumeric_caster(m_decompressionDurationMicroSec.GetTotal()); - AZStd::chrono::microseconds cumulativeDelay = AZStd::chrono::microseconds::max(); - - // Check the number of jobs that are processing. - for (u32 i = 0; i < m_maxNumJobs; ++i) + if (m_processingJobs[i].IsProcessing()) { - if (m_processingJobs[i].IsProcessing()) - { - FileRequest* compressedRequest = m_processingJobs[i].m_waitRequest->GetParent(); - AZ_Assert(compressedRequest, "A wait request attached to FullFileDecompressor was completed but didn't have a parent compressed request."); - auto data = AZStd::get_if(&compressedRequest->GetCommand()); - AZ_Assert(data, "Compressed request in the decompression queue in FullFileDecompressor didn't contain compression read data."); - - size_t bytesToDecompress = data->m_compressionInfo.m_compressedSize; - auto decompressionDuration = AZStd::chrono::microseconds( - aznumeric_cast((bytesToDecompress * totalDecompressionDuration) / totalBytesDecompressed)); - auto timeInProcessing = now - m_processingJobs[i].m_jobStartTime; - auto timeLeft = decompressionDuration > timeInProcessing ? decompressionDuration - timeInProcessing : AZStd::chrono::microseconds(0); - // Get the shortest time as this indicates the next decompression to become available. - cumulativeDelay = AZStd::min(timeLeft, cumulativeDelay); - m_processingJobs[i].m_waitRequest->SetEstimatedCompletion(now + timeLeft); - } - } - if (cumulativeDelay == AZStd::chrono::microseconds::max()) - { - cumulativeDelay = AZStd::chrono::microseconds(0); - } - - // Next update all reads that are in flight. These will have an estimation for the read to complete, but will then be queued - // for decompression, so add the time needed decompression. Assume that decompression happens in parallel. - AZStd::chrono::microseconds decompressionDelay = - AZStd::chrono::microseconds(aznumeric_cast(m_decompressionJobDelayMicroSec.CalculateAverage())); - AZStd::chrono::microseconds smallestDecompressionDuration = AZStd::chrono::microseconds::max(); - for (u32 i = 0; i < m_maxNumReads; ++i) - { - AZStd::chrono::system_clock::time_point baseTime; - switch (m_readBufferStatus[i]) - { - case ReadBufferStatus::Unused: - continue; - case ReadBufferStatus::ReadInFlight: - // Internal read requests can start and complete but pending finalization before they're ever scheduled in which case - // the estimated time is not set. - baseTime = m_readRequests[i]->GetEstimatedCompletion(); - if (baseTime == AZStd::chrono::system_clock::time_point()) - { - baseTime = now; - } - break; - case ReadBufferStatus::PendingDecompression: - baseTime = now; - break; - default: - AZ_Assert(false, "Unsupported buffer type: %i.", m_readBufferStatus[i]); - continue; - } - - baseTime += cumulativeDelay; // Delay until the first decompression slot becomes available. - baseTime += decompressionDelay; // The average time it takes for the job system to pick up the decompression job. - - // Calculate the amount of time it will take to decompress the data. - FileRequest* compressedRequest = m_readRequests[i]->GetParent(); + FileRequest* compressedRequest = m_processingJobs[i].m_waitRequest->GetParent(); + AZ_Assert(compressedRequest, "A wait request attached to FullFileDecompressor was completed but didn't have a parent compressed request."); auto data = AZStd::get_if(&compressedRequest->GetCommand()); - + AZ_Assert(data, "Compressed request in the decompression queue in FullFileDecompressor didn't contain compression read data."); + size_t bytesToDecompress = data->m_compressionInfo.m_compressedSize; auto decompressionDuration = AZStd::chrono::microseconds( aznumeric_cast((bytesToDecompress * totalDecompressionDuration) / totalBytesDecompressed)); - smallestDecompressionDuration = AZStd::min(smallestDecompressionDuration, decompressionDuration); - baseTime += decompressionDuration; - - m_readRequests[i]->SetEstimatedCompletion(baseTime); - } - if (smallestDecompressionDuration != AZStd::chrono::microseconds::max()) - { - cumulativeDelay += smallestDecompressionDuration; // Time after which the decompression jobs and pending reads have completed. - } - - // For all internally pending compressed reads add the decompression time. The read time will have already been added downstream. - // Because this call will go from the top of the stack to the bottom, but estimation is calculated from the bottom to the top, this - // list should be processed in reverse order. - for (auto pendingIt = internalPending.rbegin(); pendingIt != internalPending.rend(); ++pendingIt) - { - EstimateCompressedReadRequest(*pendingIt, cumulativeDelay, decompressionDelay, - totalDecompressionDuration, totalBytesDecompressed); - } - - // Finally add a prediction for all the requests that are waiting to be queued. - for (auto requestIt = pendingBegin; requestIt != pendingEnd; ++requestIt) - { - EstimateCompressedReadRequest(*requestIt, cumulativeDelay, decompressionDelay, - totalDecompressionDuration, totalBytesDecompressed); + auto timeInProcessing = now - m_processingJobs[i].m_jobStartTime; + auto timeLeft = decompressionDuration > timeInProcessing ? decompressionDuration - timeInProcessing : AZStd::chrono::microseconds(0); + // Get the shortest time as this indicates the next decompression to become available. + cumulativeDelay = AZStd::min(timeLeft, cumulativeDelay); + m_processingJobs[i].m_waitRequest->SetEstimatedCompletion(now + timeLeft); } } - - void FullFileDecompressor::EstimateCompressedReadRequest(FileRequest* request, AZStd::chrono::microseconds& cumulativeDelay, - AZStd::chrono::microseconds decompressionDelay, double totalDecompressionDurationUs, double totalBytesDecompressed) const + if (cumulativeDelay == AZStd::chrono::microseconds::max()) { - auto data = AZStd::get_if(&request->GetCommand()); - if (data) - { - AZStd::chrono::microseconds processingTime = decompressionDelay; - size_t bytesToDecompress = data->m_compressionInfo.m_compressedSize; - processingTime += AZStd::chrono::microseconds( - aznumeric_cast((bytesToDecompress * totalDecompressionDurationUs) / totalBytesDecompressed)); - - cumulativeDelay += processingTime; - request->SetEstimatedCompletion(request->GetEstimatedCompletion() + processingTime); - } + cumulativeDelay = AZStd::chrono::microseconds(0); } - void FullFileDecompressor::CollectStatistics(AZStd::vector& statistics) const + // Next update all reads that are in flight. These will have an estimation for the read to complete, but will then be queued + // for decompression, so add the time needed decompression. Assume that decompression happens in parallel. + AZStd::chrono::microseconds decompressionDelay = + AZStd::chrono::microseconds(aznumeric_cast(m_decompressionJobDelayMicroSec.CalculateAverage())); + AZStd::chrono::microseconds smallestDecompressionDuration = AZStd::chrono::microseconds::max(); + for (u32 i = 0; i < m_maxNumReads; ++i) { - constexpr double bytesToMB = 1.0 / (1024.0 * 1024.0); - constexpr double usToSec = 1.0 / (1000.0 * 1000.0); - constexpr double usToMs = 1.0 / 1000.0; - - if (m_bytesDecompressed.GetNumRecorded() > 1) // There's always a default added. + AZStd::chrono::system_clock::time_point baseTime; + switch (m_readBufferStatus[i]) { - //It only makes sense to add decompression statistics when reading from PAK files. - statistics.push_back(Statistic::CreateInteger(m_name, "Available decompression slots", m_maxNumJobs - m_numRunningJobs)); - statistics.push_back(Statistic::CreateInteger(m_name, "Available read slots", m_maxNumReads - m_numInFlightReads)); - statistics.push_back(Statistic::CreateInteger(m_name, "Pending decompression", m_numPendingDecompression)); - statistics.push_back(Statistic::CreateFloat(m_name, "Buffer memory (MB)", m_memoryUsage * bytesToMB)); + case ReadBufferStatus::Unused: + continue; + case ReadBufferStatus::ReadInFlight: + // Internal read requests can start and complete but pending finalization before they're ever scheduled in which case + // the estimated time is not set. + baseTime = m_readRequests[i]->GetEstimatedCompletion(); + if (baseTime == AZStd::chrono::system_clock::time_point()) + { + baseTime = now; + } + break; + case ReadBufferStatus::PendingDecompression: + baseTime = now; + break; + default: + AZ_Assert(false, "Unsupported buffer type: %i.", m_readBufferStatus[i]); + continue; + } - double averageJobStartDelay = m_decompressionJobDelayMicroSec.CalculateAverage() * usToMs; - statistics.push_back(Statistic::CreateFloat(m_name, "Decompression job delay (avg. ms)", averageJobStartDelay)); + baseTime += cumulativeDelay; // Delay until the first decompression slot becomes available. + baseTime += decompressionDelay; // The average time it takes for the job system to pick up the decompression job. - double totalBytesDecompressedMB = m_bytesDecompressed.GetTotal() * bytesToMB; - double totalDecompressionTimeSec = m_decompressionDurationMicroSec.GetTotal() * usToSec; - statistics.push_back(Statistic::CreateFloat(m_name, "Decompression Speed per job (avg. mbps)", totalBytesDecompressedMB / totalDecompressionTimeSec)); + // Calculate the amount of time it will take to decompress the data. + FileRequest* compressedRequest = m_readRequests[i]->GetParent(); + auto data = AZStd::get_if(&compressedRequest->GetCommand()); + + size_t bytesToDecompress = data->m_compressionInfo.m_compressedSize; + auto decompressionDuration = AZStd::chrono::microseconds( + aznumeric_cast((bytesToDecompress * totalDecompressionDuration) / totalBytesDecompressed)); + smallestDecompressionDuration = AZStd::min(smallestDecompressionDuration, decompressionDuration); + baseTime += decompressionDuration; + + m_readRequests[i]->SetEstimatedCompletion(baseTime); + } + if (smallestDecompressionDuration != AZStd::chrono::microseconds::max()) + { + cumulativeDelay += smallestDecompressionDuration; // Time after which the decompression jobs and pending reads have completed. + } + + // For all internally pending compressed reads add the decompression time. The read time will have already been added downstream. + // Because this call will go from the top of the stack to the bottom, but estimation is calculated from the bottom to the top, this + // list should be processed in reverse order. + for (auto pendingIt = internalPending.rbegin(); pendingIt != internalPending.rend(); ++pendingIt) + { + EstimateCompressedReadRequest(*pendingIt, cumulativeDelay, decompressionDelay, + totalDecompressionDuration, totalBytesDecompressed); + } + + // Finally add a prediction for all the requests that are waiting to be queued. + for (auto requestIt = pendingBegin; requestIt != pendingEnd; ++requestIt) + { + EstimateCompressedReadRequest(*requestIt, cumulativeDelay, decompressionDelay, + totalDecompressionDuration, totalBytesDecompressed); + } + } + + void FullFileDecompressor::EstimateCompressedReadRequest(FileRequest* request, AZStd::chrono::microseconds& cumulativeDelay, + AZStd::chrono::microseconds decompressionDelay, double totalDecompressionDurationUs, double totalBytesDecompressed) const + { + auto data = AZStd::get_if(&request->GetCommand()); + if (data) + { + AZStd::chrono::microseconds processingTime = decompressionDelay; + size_t bytesToDecompress = data->m_compressionInfo.m_compressedSize; + processingTime += AZStd::chrono::microseconds( + aznumeric_cast((bytesToDecompress * totalDecompressionDurationUs) / totalBytesDecompressed)); + + cumulativeDelay += processingTime; + request->SetEstimatedCompletion(request->GetEstimatedCompletion() + processingTime); + } + } + + void FullFileDecompressor::CollectStatistics(AZStd::vector& statistics) const + { + constexpr double bytesToMB = 1.0 / (1024.0 * 1024.0); + constexpr double usToSec = 1.0 / (1000.0 * 1000.0); + constexpr double usToMs = 1.0 / 1000.0; + + if (m_bytesDecompressed.GetNumRecorded() > 1) // There's always a default added. + { + //It only makes sense to add decompression statistics when reading from PAK files. + statistics.push_back(Statistic::CreateInteger(m_name, "Available decompression slots", m_maxNumJobs - m_numRunningJobs)); + statistics.push_back(Statistic::CreateInteger(m_name, "Available read slots", m_maxNumReads - m_numInFlightReads)); + statistics.push_back(Statistic::CreateInteger(m_name, "Pending decompression", m_numPendingDecompression)); + statistics.push_back(Statistic::CreateFloat(m_name, "Buffer memory (MB)", m_memoryUsage * bytesToMB)); + + double averageJobStartDelay = m_decompressionJobDelayMicroSec.CalculateAverage() * usToMs; + statistics.push_back(Statistic::CreateFloat(m_name, "Decompression job delay (avg. ms)", averageJobStartDelay)); + + double totalBytesDecompressedMB = m_bytesDecompressed.GetTotal() * bytesToMB; + double totalDecompressionTimeSec = m_decompressionDurationMicroSec.GetTotal() * usToSec; + statistics.push_back(Statistic::CreateFloat(m_name, "Decompression Speed per job (avg. mbps)", totalBytesDecompressedMB / totalDecompressionTimeSec)); #if AZ_STREAMER_ADD_EXTRA_PROFILING_INFO - statistics.push_back(Statistic::CreatePercentage(m_name, DecompBoundName, m_decompressionBoundStat.GetAverage())); - statistics.push_back(Statistic::CreatePercentage(m_name, ReadBoundName, m_readBoundStat.GetAverage())); + statistics.push_back(Statistic::CreatePercentage(m_name, DecompBoundName, m_decompressionBoundStat.GetAverage())); + statistics.push_back(Statistic::CreatePercentage(m_name, ReadBoundName, m_readBoundStat.GetAverage())); #endif - } - - StreamStackEntry::CollectStatistics(statistics); } - bool FullFileDecompressor::IsIdle() const - { - return - m_pendingReads.empty() && - m_pendingFileExistChecks.empty() && - m_numInFlightReads == 0 && - m_numPendingDecompression == 0 && - m_numRunningJobs == 0; - } + StreamStackEntry::CollectStatistics(statistics); + } - void FullFileDecompressor::PrepareReadRequest(FileRequest* request, FileRequest::ReadRequestData& data) + bool FullFileDecompressor::IsIdle() const + { + return + m_pendingReads.empty() && + m_pendingFileExistChecks.empty() && + m_numInFlightReads == 0 && + m_numPendingDecompression == 0 && + m_numRunningJobs == 0; + } + + void FullFileDecompressor::PrepareReadRequest(FileRequest* request, FileRequest::ReadRequestData& data) + { + CompressionInfo info; + if (CompressionUtils::FindCompressionInfo(info, data.m_path.GetRelativePath())) { - CompressionInfo info; - if (CompressionUtils::FindCompressionInfo(info, data.m_path.GetRelativePath())) + FileRequest* nextRequest = m_context->GetNewInternalRequest(); + if (info.m_isCompressed) { - FileRequest* nextRequest = m_context->GetNewInternalRequest(); - if (info.m_isCompressed) - { - AZ_Assert(info.m_decompressor, - "FullFileDecompressor::PrepareRequest found a compressed file, but no decompressor to decompress with."); - nextRequest->CreateCompressedRead(request, AZStd::move(info), data.m_output, data.m_offset, data.m_size); - } - else - { - FileRequest* pathStorageRequest = m_context->GetNewInternalRequest(); - pathStorageRequest->CreateRequestPathStore(request, AZStd::move(info.m_archiveFilename)); - auto& pathStorage = AZStd::get(pathStorageRequest->GetCommand()); - - nextRequest->CreateRead(pathStorageRequest, data.m_output, data.m_outputSize, pathStorage.m_path, - info.m_offset + data.m_offset, data.m_size, info.m_isSharedPak); - } - - if (info.m_conflictResolution == ConflictResolution::PreferFile) - { - auto callback = [this, nextRequest](const FileRequest& checkRequest) - { - AZ_PROFILE_FUNCTION(AzCore); - auto check = AZStd::get_if(&checkRequest.GetCommand()); - AZ_Assert(check, - "Callback in FullFileDecompressor::PrepareReadRequest expected FileExistsCheck but got another command."); - if (check->m_found) - { - FileRequest* originalRequest = m_context->RejectRequest(nextRequest); - if (AZStd::holds_alternative(originalRequest->GetCommand())) - { - originalRequest = m_context->RejectRequest(originalRequest); - } - StreamStackEntry::PrepareRequest(originalRequest); - } - else - { - m_context->PushPreparedRequest(nextRequest); - } - }; - FileRequest* fileCheckRequest = m_context->GetNewInternalRequest(); - fileCheckRequest->CreateFileExistsCheck(data.m_path); - fileCheckRequest->SetCompletionCallback(AZStd::move(callback)); - StreamStackEntry::QueueRequest(fileCheckRequest); - } - else - { - m_context->PushPreparedRequest(nextRequest); - } + AZ_Assert(info.m_decompressor, + "FullFileDecompressor::PrepareRequest found a compressed file, but no decompressor to decompress with."); + nextRequest->CreateCompressedRead(request, AZStd::move(info), data.m_output, data.m_offset, data.m_size); } else { - StreamStackEntry::PrepareRequest(request); - } - } + FileRequest* pathStorageRequest = m_context->GetNewInternalRequest(); + pathStorageRequest->CreateRequestPathStore(request, AZStd::move(info.m_archiveFilename)); + auto& pathStorage = AZStd::get(pathStorageRequest->GetCommand()); - void FullFileDecompressor::PrepareDedicatedCache(FileRequest* request, const RequestPath& path) - { - CompressionInfo info; - if (CompressionUtils::FindCompressionInfo(info, path.GetRelativePath())) + nextRequest->CreateRead(pathStorageRequest, data.m_output, data.m_outputSize, pathStorage.m_path, + info.m_offset + data.m_offset, data.m_size, info.m_isSharedPak); + } + + if (info.m_conflictResolution == ConflictResolution::PreferFile) { - FileRequest* nextRequest = m_context->GetNewInternalRequest(); - AZStd::visit([request, &info, nextRequest](auto&& args) + auto callback = [this, nextRequest](const FileRequest& checkRequest) { - using Command = AZStd::decay_t; - if constexpr (AZStd::is_same_v) + AZ_PROFILE_FUNCTION(AzCore); + auto check = AZStd::get_if(&checkRequest.GetCommand()); + AZ_Assert(check, + "Callback in FullFileDecompressor::PrepareReadRequest expected FileExistsCheck but got another command."); + if (check->m_found) { - nextRequest->CreateDedicatedCacheCreation(AZStd::move(info.m_archiveFilename), - FileRange::CreateRange(info.m_offset, info.m_compressedSize), request); - } - else if constexpr (AZStd::is_same_v) - { - nextRequest->CreateDedicatedCacheDestruction(AZStd::move(info.m_archiveFilename), - FileRange::CreateRange(info.m_offset, info.m_compressedSize), request); - } - }, request->GetCommand()); - - if (info.m_conflictResolution == ConflictResolution::PreferFile) - { - auto callback = [this, nextRequest](const FileRequest& checkRequest) - { - AZ_PROFILE_FUNCTION(AzCore); - auto check = AZStd::get_if(&checkRequest.GetCommand()); - AZ_Assert(check, - "Callback in FullFileDecompressor::PrepareDedicatedCache expected FileExistsCheck but got another command."); - if (check->m_found) + FileRequest* originalRequest = m_context->RejectRequest(nextRequest); + if (AZStd::holds_alternative(originalRequest->GetCommand())) { - FileRequest* originalRequest = nextRequest->GetParent(); - m_context->RejectRequest(nextRequest); - StreamStackEntry::PrepareRequest(originalRequest); + originalRequest = m_context->RejectRequest(originalRequest); } - else - { - m_context->PushPreparedRequest(nextRequest); - } - }; - FileRequest* fileCheckRequest = m_context->GetNewInternalRequest(); - fileCheckRequest->CreateFileExistsCheck(path); - fileCheckRequest->SetCompletionCallback(AZStd::move(callback)); - StreamStackEntry::QueueRequest(fileCheckRequest); - } - else - { - m_context->PushPreparedRequest(nextRequest); - } - } - else - { - StreamStackEntry::PrepareRequest(request); - } - } - - void FullFileDecompressor::FileExistsCheck(FileRequest* checkRequest) - { - auto& fileCheckRequest = AZStd::get(checkRequest->GetCommand()); - CompressionInfo info; - if (CompressionUtils::FindCompressionInfo(info, fileCheckRequest.m_path.GetRelativePath())) - { - fileCheckRequest.m_found = true; - } - else - { - // The file isn't in the archive but might still exist as a loose file, so let the next node have a shot. - StreamStackEntry::QueueRequest(checkRequest); - } - } - - void FullFileDecompressor::StartArchiveRead(FileRequest* compressedReadRequest) - { - if (!m_next) - { - compressedReadRequest->SetStatus(IStreamerTypes::RequestStatus::Failed); - m_context->MarkRequestAsCompleted(compressedReadRequest); - return; - } - - for (u32 i = 0; i < m_maxNumReads; ++i) - { - if (m_readBufferStatus[i] == ReadBufferStatus::Unused) - { - auto data = AZStd::get_if(&compressedReadRequest->GetCommand()); - AZ_Assert(data, "Compressed request that's starting a read in FullFileDecompressor didn't contain compression read data."); - AZ_Assert(data->m_compressionInfo.m_decompressor, - "FileRequest for FullFileDecompressor is missing a decompression callback."); - - CompressionInfo& info = data->m_compressionInfo; - AZ_Assert(info.m_decompressor, "FullFileDecompressor is planning to a queue a request for reading but couldn't find a decompressor."); - - // The buffer is aligned down but the offset is not corrected. If the offset was adjusted it would mean the same data is read - // multiple times and negates the block cache's ability to detect these cases. By still adjusting it means that the reads between - // the BlockCache's prolog and epilog are read into aligned buffers. - size_t offsetAdjustment = info.m_offset - AZ_SIZE_ALIGN_DOWN(info.m_offset, aznumeric_cast(m_alignment)); - size_t bufferSize = AZ_SIZE_ALIGN_UP((info.m_compressedSize + offsetAdjustment), aznumeric_cast(m_alignment)); - m_readBuffers[i] = reinterpret_cast(AZ::AllocatorInstance::Get().Allocate( - bufferSize, m_alignment, 0, "AZ::IO::Streamer FullFileDecompressor", __FILE__, __LINE__)); - m_memoryUsage += bufferSize; - - FileRequest* archiveReadRequest = m_context->GetNewInternalRequest(); - archiveReadRequest->CreateRead(compressedReadRequest, m_readBuffers[i] + offsetAdjustment, bufferSize, info.m_archiveFilename, - info.m_offset, info.m_compressedSize, info.m_isSharedPak); - archiveReadRequest->SetCompletionCallback( - [this, readSlot = i](FileRequest& request) - { - AZ_PROFILE_FUNCTION(AzCore); - FinishArchiveRead(&request, readSlot); - }); - m_next->QueueRequest(archiveReadRequest); - - m_readRequests[i] = archiveReadRequest; - m_readBufferStatus[i] = ReadBufferStatus::ReadInFlight; - - AZ_Assert(m_numInFlightReads < m_maxNumReads, - "A FileRequest was queued for reading in FullFileDecompressor, but there's no slots available."); - m_numInFlightReads++; - - return; - } - } - AZ_Assert(false, "%u of %u read slots are use in the FullFileDecompressor, but no empty slot was found.", m_numInFlightReads, m_maxNumReads); - } - - void FullFileDecompressor::FinishArchiveRead(FileRequest* readRequest, u32 readSlot) - { - AZ_Assert(m_readRequests[readSlot] == readRequest, - "Request in the archive read slot isn't the same as request that's being completed."); - - FileRequest* compressedRequest = readRequest->GetParent(); - AZ_Assert(compressedRequest, "Read requests started by FullFileDecompressor is missing a parent request."); - - if (readRequest->GetStatus() == IStreamerTypes::RequestStatus::Completed) - { - m_readBufferStatus[readSlot] = ReadBufferStatus::PendingDecompression; - ++m_numPendingDecompression; - - // Add this wait so the compressed request isn't fully completed yet as only the read part is done. The - // job thread will finish this wait, which in turn will trigger this function again on the main streaming thread. - FileRequest* waitRequest = m_context->GetNewInternalRequest(); - waitRequest->CreateWait(compressedRequest); - m_readRequests[readSlot] = waitRequest; - } - else - { - auto data = AZStd::get_if(&compressedRequest->GetCommand()); - AZ_Assert(data, "Compressed request in FullFileDecompressor that finished unsuccessfully didn't contain compression read data."); - CompressionInfo& info = data->m_compressionInfo; - size_t offsetAdjustment = info.m_offset - AZ_SIZE_ALIGN_DOWN(info.m_offset, aznumeric_cast(m_alignment)); - size_t bufferSize = AZ_SIZE_ALIGN_UP((info.m_compressedSize + offsetAdjustment), aznumeric_cast(m_alignment)); - m_memoryUsage -= bufferSize; - - if (m_readBuffers[readSlot] != nullptr) - { - AZ::AllocatorInstance::Get().DeAllocate(m_readBuffers[readSlot], bufferSize, m_alignment); - m_readBuffers[readSlot] = nullptr; - } - m_readRequests[readSlot] = nullptr; - m_readBufferStatus[readSlot] = ReadBufferStatus::Unused; - AZ_Assert(m_numInFlightReads > 0, - "Trying to decrement a read request after it was canceled or failed in FullFileDecompressor, " - "but no read requests are supposed to be queued."); - m_numInFlightReads--; - } - } - - bool FullFileDecompressor::StartDecompressions() - { - bool queuedJobs = false; - u32 jobSlot = 0; - for (u32 readSlot = 0; readSlot < m_maxNumReads; ++readSlot) - { - // Find completed read. - if (m_readBufferStatus[readSlot] != ReadBufferStatus::PendingDecompression) - { - continue; - } - - // Find decompression slot - for (; jobSlot < m_maxNumJobs; ++jobSlot) - { - if (m_processingJobs[jobSlot].IsProcessing()) - { - continue; - } - - FileRequest* waitRequest = m_readRequests[readSlot]; - AZ_Assert(AZStd::holds_alternative(waitRequest->GetCommand()), - "File request waiting for decompression wasn't marked as being a wait operation."); - FileRequest* compressedRequest = waitRequest->GetParent(); - AZ_Assert(compressedRequest, "Read requests started by FullFileDecompressor is missing a parent request."); - - waitRequest->SetCompletionCallback([this, jobSlot](FileRequest& request) - { - AZ_PROFILE_FUNCTION(AzCore); - FinishDecompression(&request, jobSlot); - }); - - DecompressionInformation& info = m_processingJobs[jobSlot]; - info.m_waitRequest = waitRequest; - info.m_queueStartTime = AZStd::chrono::high_resolution_clock::now(); - info.m_jobStartTime = info.m_queueStartTime; // Set these to the same in case the scheduler requests an update before the job has started. - info.m_compressedData = m_readBuffers[readSlot]; // Transfer ownership of the pointer. - m_readBuffers[readSlot] = nullptr; - - AZ::Job* decompressionJob; - auto data = AZStd::get_if(&compressedRequest->GetCommand()); - AZ_Assert(data, "Compressed request in FullFileDecompressor that's starting decompression didn't contain compression read data."); - AZ_Assert(data->m_compressionInfo.m_decompressor, "FullFileDecompressor is queuing a decompression job but couldn't find a decompressor."); - - info.m_alignmentOffset = aznumeric_caster(data->m_compressionInfo.m_offset - - AZ_SIZE_ALIGN_DOWN(data->m_compressionInfo.m_offset, aznumeric_cast(m_alignment))); - - if (data->m_readOffset == 0 && data->m_readSize == data->m_compressionInfo.m_uncompressedSize) - { - auto job = [this, &info]() - { - FullDecompression(m_context, info); - }; - decompressionJob = AZ::CreateJobFunction(job, true, m_decompressionjobContext.get()); + StreamStackEntry::PrepareRequest(originalRequest); } else { - m_memoryUsage += data->m_compressionInfo.m_uncompressedSize; - auto job = [this, &info]() - { - PartialDecompression(m_context, info); - }; - decompressionJob = AZ::CreateJobFunction(job, true, m_decompressionjobContext.get()); + m_context->PushPreparedRequest(nextRequest); } - --m_numPendingDecompression; - ++m_numRunningJobs; - decompressionJob->Start(); - - m_readRequests[readSlot] = nullptr; - m_readBufferStatus[readSlot] = ReadBufferStatus::Unused; - AZ_Assert(m_numInFlightReads > 0, "Trying to decrement a read request after it's queued for decompression in FullFileDecompressor, but no read requests are supposed to be queued."); - m_numInFlightReads--; - - queuedJobs = true; - break; - } - - if (m_numInFlightReads == 0 || m_numRunningJobs == m_maxNumJobs) - { - return queuedJobs; - } + }; + FileRequest* fileCheckRequest = m_context->GetNewInternalRequest(); + fileCheckRequest->CreateFileExistsCheck(data.m_path); + fileCheckRequest->SetCompletionCallback(AZStd::move(callback)); + StreamStackEntry::QueueRequest(fileCheckRequest); } - return queuedJobs; + else + { + m_context->PushPreparedRequest(nextRequest); + } + } + else + { + StreamStackEntry::PrepareRequest(request); + } + } + + void FullFileDecompressor::PrepareDedicatedCache(FileRequest* request, const RequestPath& path) + { + CompressionInfo info; + if (CompressionUtils::FindCompressionInfo(info, path.GetRelativePath())) + { + FileRequest* nextRequest = m_context->GetNewInternalRequest(); + AZStd::visit([request, &info, nextRequest](auto&& args) + { + using Command = AZStd::decay_t; + if constexpr (AZStd::is_same_v) + { + nextRequest->CreateDedicatedCacheCreation(AZStd::move(info.m_archiveFilename), + FileRange::CreateRange(info.m_offset, info.m_compressedSize), request); + } + else if constexpr (AZStd::is_same_v) + { + nextRequest->CreateDedicatedCacheDestruction(AZStd::move(info.m_archiveFilename), + FileRange::CreateRange(info.m_offset, info.m_compressedSize), request); + } + }, request->GetCommand()); + + if (info.m_conflictResolution == ConflictResolution::PreferFile) + { + auto callback = [this, nextRequest](const FileRequest& checkRequest) + { + AZ_PROFILE_FUNCTION(AzCore); + auto check = AZStd::get_if(&checkRequest.GetCommand()); + AZ_Assert(check, + "Callback in FullFileDecompressor::PrepareDedicatedCache expected FileExistsCheck but got another command."); + if (check->m_found) + { + FileRequest* originalRequest = nextRequest->GetParent(); + m_context->RejectRequest(nextRequest); + StreamStackEntry::PrepareRequest(originalRequest); + } + else + { + m_context->PushPreparedRequest(nextRequest); + } + }; + FileRequest* fileCheckRequest = m_context->GetNewInternalRequest(); + fileCheckRequest->CreateFileExistsCheck(path); + fileCheckRequest->SetCompletionCallback(AZStd::move(callback)); + StreamStackEntry::QueueRequest(fileCheckRequest); + } + else + { + m_context->PushPreparedRequest(nextRequest); + } + } + else + { + StreamStackEntry::PrepareRequest(request); + } + } + + void FullFileDecompressor::FileExistsCheck(FileRequest* checkRequest) + { + auto& fileCheckRequest = AZStd::get(checkRequest->GetCommand()); + CompressionInfo info; + if (CompressionUtils::FindCompressionInfo(info, fileCheckRequest.m_path.GetRelativePath())) + { + fileCheckRequest.m_found = true; + } + else + { + // The file isn't in the archive but might still exist as a loose file, so let the next node have a shot. + StreamStackEntry::QueueRequest(checkRequest); + } + } + + void FullFileDecompressor::StartArchiveRead(FileRequest* compressedReadRequest) + { + if (!m_next) + { + compressedReadRequest->SetStatus(IStreamerTypes::RequestStatus::Failed); + m_context->MarkRequestAsCompleted(compressedReadRequest); + return; } - void FullFileDecompressor::FinishDecompression([[maybe_unused]] FileRequest* waitRequest, u32 jobSlot) + for (u32 i = 0; i < m_maxNumReads; ++i) { - DecompressionInformation& jobInfo = m_processingJobs[jobSlot]; - AZ_Assert(jobInfo.m_waitRequest == waitRequest, "Job slot didn't contain the expected wait request."); + if (m_readBufferStatus[i] == ReadBufferStatus::Unused) + { + auto data = AZStd::get_if(&compressedReadRequest->GetCommand()); + AZ_Assert(data, "Compressed request that's starting a read in FullFileDecompressor didn't contain compression read data."); + AZ_Assert(data->m_compressionInfo.m_decompressor, + "FileRequest for FullFileDecompressor is missing a decompression callback."); - auto endTime = AZStd::chrono::high_resolution_clock::now(); + CompressionInfo& info = data->m_compressionInfo; + AZ_Assert(info.m_decompressor, "FullFileDecompressor is planning to a queue a request for reading but couldn't find a decompressor."); - FileRequest* compressedRequest = jobInfo.m_waitRequest->GetParent(); - AZ_Assert(compressedRequest, "A wait request attached to FullFileDecompressor was completed but didn't have a parent compressed request."); + // The buffer is aligned down but the offset is not corrected. If the offset was adjusted it would mean the same data is read + // multiple times and negates the block cache's ability to detect these cases. By still adjusting it means that the reads between + // the BlockCache's prolog and epilog are read into aligned buffers. + size_t offsetAdjustment = info.m_offset - AZ_SIZE_ALIGN_DOWN(info.m_offset, aznumeric_cast(m_alignment)); + size_t bufferSize = AZ_SIZE_ALIGN_UP((info.m_compressedSize + offsetAdjustment), aznumeric_cast(m_alignment)); + m_readBuffers[i] = reinterpret_cast(AZ::AllocatorInstance::Get().Allocate( + bufferSize, m_alignment, 0, "AZ::IO::Streamer FullFileDecompressor", __FILE__, __LINE__)); + m_memoryUsage += bufferSize; + + FileRequest* archiveReadRequest = m_context->GetNewInternalRequest(); + archiveReadRequest->CreateRead(compressedReadRequest, m_readBuffers[i] + offsetAdjustment, bufferSize, info.m_archiveFilename, + info.m_offset, info.m_compressedSize, info.m_isSharedPak); + archiveReadRequest->SetCompletionCallback( + [this, readSlot = i](FileRequest& request) + { + AZ_PROFILE_FUNCTION(AzCore); + FinishArchiveRead(&request, readSlot); + }); + m_next->QueueRequest(archiveReadRequest); + + m_readRequests[i] = archiveReadRequest; + m_readBufferStatus[i] = ReadBufferStatus::ReadInFlight; + + AZ_Assert(m_numInFlightReads < m_maxNumReads, + "A FileRequest was queued for reading in FullFileDecompressor, but there's no slots available."); + m_numInFlightReads++; + + return; + } + } + AZ_Assert(false, "%u of %u read slots are use in the FullFileDecompressor, but no empty slot was found.", m_numInFlightReads, m_maxNumReads); + } + + void FullFileDecompressor::FinishArchiveRead(FileRequest* readRequest, u32 readSlot) + { + AZ_Assert(m_readRequests[readSlot] == readRequest, + "Request in the archive read slot isn't the same as request that's being completed."); + + FileRequest* compressedRequest = readRequest->GetParent(); + AZ_Assert(compressedRequest, "Read requests started by FullFileDecompressor is missing a parent request."); + + if (readRequest->GetStatus() == IStreamerTypes::RequestStatus::Completed) + { + m_readBufferStatus[readSlot] = ReadBufferStatus::PendingDecompression; + ++m_numPendingDecompression; + + // Add this wait so the compressed request isn't fully completed yet as only the read part is done. The + // job thread will finish this wait, which in turn will trigger this function again on the main streaming thread. + FileRequest* waitRequest = m_context->GetNewInternalRequest(); + waitRequest->CreateWait(compressedRequest); + m_readRequests[readSlot] = waitRequest; + } + else + { auto data = AZStd::get_if(&compressedRequest->GetCommand()); - AZ_Assert(data, "Compressed request in FullFileDecompressor that completed decompression didn't contain compression read data."); + AZ_Assert(data, "Compressed request in FullFileDecompressor that finished unsuccessfully didn't contain compression read data."); CompressionInfo& info = data->m_compressionInfo; size_t offsetAdjustment = info.m_offset - AZ_SIZE_ALIGN_DOWN(info.m_offset, aznumeric_cast(m_alignment)); size_t bufferSize = AZ_SIZE_ALIGN_UP((info.m_compressedSize + offsetAdjustment), aznumeric_cast(m_alignment)); m_memoryUsage -= bufferSize; - if (data->m_readOffset != 0 || data->m_readSize != data->m_compressionInfo.m_uncompressedSize) + + if (m_readBuffers[readSlot] != nullptr) { - m_memoryUsage -= data->m_compressionInfo.m_uncompressedSize; + AZ::AllocatorInstance::Get().DeAllocate(m_readBuffers[readSlot], bufferSize, m_alignment); + m_readBuffers[readSlot] = nullptr; + } + m_readRequests[readSlot] = nullptr; + m_readBufferStatus[readSlot] = ReadBufferStatus::Unused; + AZ_Assert(m_numInFlightReads > 0, + "Trying to decrement a read request after it was canceled or failed in FullFileDecompressor, " + "but no read requests are supposed to be queued."); + m_numInFlightReads--; + } + } + + bool FullFileDecompressor::StartDecompressions() + { + bool queuedJobs = false; + u32 jobSlot = 0; + for (u32 readSlot = 0; readSlot < m_maxNumReads; ++readSlot) + { + // Find completed read. + if (m_readBufferStatus[readSlot] != ReadBufferStatus::PendingDecompression) + { + continue; } - m_decompressionJobDelayMicroSec.PushEntry(AZStd::chrono::duration_cast( - jobInfo.m_jobStartTime - jobInfo.m_queueStartTime).count()); - m_decompressionDurationMicroSec.PushEntry(AZStd::chrono::duration_cast( - endTime - jobInfo.m_jobStartTime).count()); - m_bytesDecompressed.PushEntry(data->m_compressionInfo.m_compressedSize); + // Find decompression slot + for (; jobSlot < m_maxNumJobs; ++jobSlot) + { + if (m_processingJobs[jobSlot].IsProcessing()) + { + continue; + } - AZ::AllocatorInstance::Get().DeAllocate(jobInfo.m_compressedData, bufferSize, m_alignment); - jobInfo.m_compressedData = nullptr; - AZ_Assert(m_numRunningJobs > 0, "About to complete a decompression job, but the internal count doesn't see a running job."); - --m_numRunningJobs; - return; + FileRequest* waitRequest = m_readRequests[readSlot]; + AZ_Assert(AZStd::holds_alternative(waitRequest->GetCommand()), + "File request waiting for decompression wasn't marked as being a wait operation."); + FileRequest* compressedRequest = waitRequest->GetParent(); + AZ_Assert(compressedRequest, "Read requests started by FullFileDecompressor is missing a parent request."); + + waitRequest->SetCompletionCallback([this, jobSlot](FileRequest& request) + { + AZ_PROFILE_FUNCTION(AzCore); + FinishDecompression(&request, jobSlot); + }); + + DecompressionInformation& info = m_processingJobs[jobSlot]; + info.m_waitRequest = waitRequest; + info.m_queueStartTime = AZStd::chrono::high_resolution_clock::now(); + info.m_jobStartTime = info.m_queueStartTime; // Set these to the same in case the scheduler requests an update before the job has started. + info.m_compressedData = m_readBuffers[readSlot]; // Transfer ownership of the pointer. + m_readBuffers[readSlot] = nullptr; + + AZ::Job* decompressionJob; + auto data = AZStd::get_if(&compressedRequest->GetCommand()); + AZ_Assert(data, "Compressed request in FullFileDecompressor that's starting decompression didn't contain compression read data."); + AZ_Assert(data->m_compressionInfo.m_decompressor, "FullFileDecompressor is queuing a decompression job but couldn't find a decompressor."); + + info.m_alignmentOffset = aznumeric_caster(data->m_compressionInfo.m_offset - + AZ_SIZE_ALIGN_DOWN(data->m_compressionInfo.m_offset, aznumeric_cast(m_alignment))); + + if (data->m_readOffset == 0 && data->m_readSize == data->m_compressionInfo.m_uncompressedSize) + { + auto job = [this, &info]() + { + FullDecompression(m_context, info); + }; + decompressionJob = AZ::CreateJobFunction(job, true, m_decompressionjobContext.get()); + } + else + { + m_memoryUsage += data->m_compressionInfo.m_uncompressedSize; + auto job = [this, &info]() + { + PartialDecompression(m_context, info); + }; + decompressionJob = AZ::CreateJobFunction(job, true, m_decompressionjobContext.get()); + } + --m_numPendingDecompression; + ++m_numRunningJobs; + decompressionJob->Start(); + + m_readRequests[readSlot] = nullptr; + m_readBufferStatus[readSlot] = ReadBufferStatus::Unused; + AZ_Assert(m_numInFlightReads > 0, "Trying to decrement a read request after it's queued for decompression in FullFileDecompressor, but no read requests are supposed to be queued."); + m_numInFlightReads--; + + queuedJobs = true; + break; + } + + if (m_numInFlightReads == 0 || m_numRunningJobs == m_maxNumJobs) + { + return queuedJobs; + } } + return queuedJobs; + } - void FullFileDecompressor::FullDecompression(StreamerContext* context, DecompressionInformation& info) + void FullFileDecompressor::FinishDecompression([[maybe_unused]] FileRequest* waitRequest, u32 jobSlot) + { + DecompressionInformation& jobInfo = m_processingJobs[jobSlot]; + AZ_Assert(jobInfo.m_waitRequest == waitRequest, "Job slot didn't contain the expected wait request."); + + auto endTime = AZStd::chrono::high_resolution_clock::now(); + + FileRequest* compressedRequest = jobInfo.m_waitRequest->GetParent(); + AZ_Assert(compressedRequest, "A wait request attached to FullFileDecompressor was completed but didn't have a parent compressed request."); + auto data = AZStd::get_if(&compressedRequest->GetCommand()); + AZ_Assert(data, "Compressed request in FullFileDecompressor that completed decompression didn't contain compression read data."); + CompressionInfo& info = data->m_compressionInfo; + size_t offsetAdjustment = info.m_offset - AZ_SIZE_ALIGN_DOWN(info.m_offset, aznumeric_cast(m_alignment)); + size_t bufferSize = AZ_SIZE_ALIGN_UP((info.m_compressedSize + offsetAdjustment), aznumeric_cast(m_alignment)); + m_memoryUsage -= bufferSize; + if (data->m_readOffset != 0 || data->m_readSize != data->m_compressionInfo.m_uncompressedSize) { - info.m_jobStartTime = AZStd::chrono::high_resolution_clock::now(); - - FileRequest* compressedRequest = info.m_waitRequest->GetParent(); - AZ_Assert(compressedRequest, "A wait request attached to FullFileDecompressor was completed but didn't have a parent compressed request."); - auto request = AZStd::get_if(&compressedRequest->GetCommand()); - AZ_Assert(request, "Compressed request in FullFileDecompressor that's running full decompression didn't contain compression read data."); - CompressionInfo& compressionInfo = request->m_compressionInfo; - AZ_Assert(compressionInfo.m_decompressor, "Full decompressor job started, but there's no decompressor callback assigned."); - - AZ_Assert(request->m_readOffset == 0, "FullFileDecompressor is doing a full decompression on a file request with an offset (%zu).", - request->m_readOffset); - AZ_Assert(compressionInfo.m_uncompressedSize == request->m_readSize, - "FullFileDecompressor is doing a full decompression, but the target buffer size (%llu) doesn't match the decompressed size (%zu).", - request->m_readSize, compressionInfo.m_uncompressedSize); - - bool success = compressionInfo.m_decompressor(compressionInfo, info.m_compressedData + info.m_alignmentOffset, - compressionInfo.m_compressedSize, request->m_output, compressionInfo.m_uncompressedSize); - info.m_waitRequest->SetStatus(success ? IStreamerTypes::RequestStatus::Completed : IStreamerTypes::RequestStatus::Failed); - - context->MarkRequestAsCompleted(info.m_waitRequest); - context->WakeUpSchedulingThread(); + m_memoryUsage -= data->m_compressionInfo.m_uncompressedSize; } - void FullFileDecompressor::PartialDecompression(StreamerContext* context, DecompressionInformation& info) - { - info.m_jobStartTime = AZStd::chrono::high_resolution_clock::now(); + m_decompressionJobDelayMicroSec.PushEntry(AZStd::chrono::duration_cast( + jobInfo.m_jobStartTime - jobInfo.m_queueStartTime).count()); + m_decompressionDurationMicroSec.PushEntry(AZStd::chrono::duration_cast( + endTime - jobInfo.m_jobStartTime).count()); + m_bytesDecompressed.PushEntry(data->m_compressionInfo.m_compressedSize); - FileRequest* compressedRequest = info.m_waitRequest->GetParent(); - AZ_Assert(compressedRequest, "A wait request attached to FullFileDecompressor was completed but didn't have a parent compressed request."); - auto request = AZStd::get_if(&compressedRequest->GetCommand()); - AZ_Assert(request, "Compressed request in FullFileDecompressor that's running partial decompression didn't contain compression read data."); - CompressionInfo& compressionInfo = request->m_compressionInfo; - AZ_Assert(compressionInfo.m_decompressor, "Partial decompressor job started, but there's no decompressor callback assigned."); + AZ::AllocatorInstance::Get().DeAllocate(jobInfo.m_compressedData, bufferSize, m_alignment); + jobInfo.m_compressedData = nullptr; + AZ_Assert(m_numRunningJobs > 0, "About to complete a decompression job, but the internal count doesn't see a running job."); + --m_numRunningJobs; + return; + } - AZStd::unique_ptr decompressionBuffer = AZStd::unique_ptr(new u8[compressionInfo.m_uncompressedSize]); - bool success = compressionInfo.m_decompressor(compressionInfo, info.m_compressedData + info.m_alignmentOffset, - compressionInfo.m_compressedSize, decompressionBuffer.get(), compressionInfo.m_uncompressedSize); - info.m_waitRequest->SetStatus(success ? IStreamerTypes::RequestStatus::Completed : IStreamerTypes::RequestStatus::Failed); - - memcpy(request->m_output, decompressionBuffer.get() + request->m_readOffset, request->m_readSize); + void FullFileDecompressor::FullDecompression(StreamerContext* context, DecompressionInformation& info) + { + info.m_jobStartTime = AZStd::chrono::high_resolution_clock::now(); - context->MarkRequestAsCompleted(info.m_waitRequest); - context->WakeUpSchedulingThread(); - } - } // namespace IO -} // namespace AZ + FileRequest* compressedRequest = info.m_waitRequest->GetParent(); + AZ_Assert(compressedRequest, "A wait request attached to FullFileDecompressor was completed but didn't have a parent compressed request."); + auto request = AZStd::get_if(&compressedRequest->GetCommand()); + AZ_Assert(request, "Compressed request in FullFileDecompressor that's running full decompression didn't contain compression read data."); + CompressionInfo& compressionInfo = request->m_compressionInfo; + AZ_Assert(compressionInfo.m_decompressor, "Full decompressor job started, but there's no decompressor callback assigned."); + + AZ_Assert(request->m_readOffset == 0, "FullFileDecompressor is doing a full decompression on a file request with an offset (%zu).", + request->m_readOffset); + AZ_Assert(compressionInfo.m_uncompressedSize == request->m_readSize, + "FullFileDecompressor is doing a full decompression, but the target buffer size (%llu) doesn't match the decompressed size (%zu).", + request->m_readSize, compressionInfo.m_uncompressedSize); + + bool success = compressionInfo.m_decompressor(compressionInfo, info.m_compressedData + info.m_alignmentOffset, + compressionInfo.m_compressedSize, request->m_output, compressionInfo.m_uncompressedSize); + info.m_waitRequest->SetStatus(success ? IStreamerTypes::RequestStatus::Completed : IStreamerTypes::RequestStatus::Failed); + + context->MarkRequestAsCompleted(info.m_waitRequest); + context->WakeUpSchedulingThread(); + } + + void FullFileDecompressor::PartialDecompression(StreamerContext* context, DecompressionInformation& info) + { + info.m_jobStartTime = AZStd::chrono::high_resolution_clock::now(); + + FileRequest* compressedRequest = info.m_waitRequest->GetParent(); + AZ_Assert(compressedRequest, "A wait request attached to FullFileDecompressor was completed but didn't have a parent compressed request."); + auto request = AZStd::get_if(&compressedRequest->GetCommand()); + AZ_Assert(request, "Compressed request in FullFileDecompressor that's running partial decompression didn't contain compression read data."); + CompressionInfo& compressionInfo = request->m_compressionInfo; + AZ_Assert(compressionInfo.m_decompressor, "Partial decompressor job started, but there's no decompressor callback assigned."); + + AZStd::unique_ptr decompressionBuffer = AZStd::unique_ptr(new u8[compressionInfo.m_uncompressedSize]); + bool success = compressionInfo.m_decompressor(compressionInfo, info.m_compressedData + info.m_alignmentOffset, + compressionInfo.m_compressedSize, decompressionBuffer.get(), compressionInfo.m_uncompressedSize); + info.m_waitRequest->SetStatus(success ? IStreamerTypes::RequestStatus::Completed : IStreamerTypes::RequestStatus::Failed); + + memcpy(request->m_output, decompressionBuffer.get() + request->m_readOffset, request->m_readSize); + + context->MarkRequestAsCompleted(info.m_waitRequest); + context->WakeUpSchedulingThread(); + } +} // namespace AZ::IO diff --git a/Code/Framework/AzCore/AzCore/IO/Streamer/ReadSplitter.cpp b/Code/Framework/AzCore/AzCore/IO/Streamer/ReadSplitter.cpp index 00c1c63933..a952e31a93 100644 --- a/Code/Framework/AzCore/AzCore/IO/Streamer/ReadSplitter.cpp +++ b/Code/Framework/AzCore/AzCore/IO/Streamer/ReadSplitter.cpp @@ -14,376 +14,373 @@ #include #include -namespace AZ +namespace AZ::IO { - namespace IO + AZStd::shared_ptr ReadSplitterConfig::AddStreamStackEntry( + const HardwareInformation& hardware, AZStd::shared_ptr parent) { - AZStd::shared_ptr ReadSplitterConfig::AddStreamStackEntry( - const HardwareInformation& hardware, AZStd::shared_ptr parent) + size_t splitSize; + switch (m_splitSize) { - size_t splitSize; - switch (m_splitSize) - { - case SplitSize::MaxTransfer: - splitSize = hardware.m_maxTransfer; - break; - case SplitSize::MemoryAlignment: - splitSize = hardware.m_maxPhysicalSectorSize; - break; - default: - splitSize = m_splitSize; - break; - } - - size_t bufferSize = m_bufferSizeMib * 1_mib; - if (bufferSize < splitSize) - { - AZ_Warning("Streamer", false, "The buffer size for the Read Splitter is smaller than the individual split size. " - "It will be increased to fit at least one split."); - bufferSize = splitSize; - } - - auto stackEntry = AZStd::make_shared( - splitSize, - aznumeric_caster(hardware.m_maxPhysicalSectorSize), - aznumeric_caster(hardware.m_maxLogicalSectorSize), - bufferSize, m_adjustOffset, m_splitAlignedRequests); - stackEntry->SetNext(AZStd::move(parent)); - return stackEntry; + case SplitSize::MaxTransfer: + splitSize = hardware.m_maxTransfer; + break; + case SplitSize::MemoryAlignment: + splitSize = hardware.m_maxPhysicalSectorSize; + break; + default: + splitSize = m_splitSize; + break; } - void ReadSplitterConfig::Reflect(AZ::ReflectContext* context) + size_t bufferSize = m_bufferSizeMib * 1_mib; + if (bufferSize < splitSize) { - if (auto serializeContext = azrtti_cast(context); serializeContext != nullptr) - { - serializeContext->Enum() - ->Version(1) - ->Value("MaxTransfer", SplitSize::MaxTransfer) - ->Value("MemoryAlignment", SplitSize::MemoryAlignment); - - serializeContext->Class() - ->Version(1) - ->Field("BufferSizeMib", &ReadSplitterConfig::m_bufferSizeMib) - ->Field("SplitSize", &ReadSplitterConfig::m_splitSize) - ->Field("AdjustOffset", &ReadSplitterConfig::m_adjustOffset) - ->Field("SplitAlignedRequests", &ReadSplitterConfig::m_splitAlignedRequests); - } + AZ_Warning("Streamer", false, "The buffer size for the Read Splitter is smaller than the individual split size. " + "It will be increased to fit at least one split."); + bufferSize = splitSize; } - static constexpr char AvgNumSubReadsName[] = "Avg. num sub reads"; - static constexpr char AlignedReadsName[] = "Aligned reads"; - static constexpr char NumAvailableBufferSlotsName[] = "Num available buffer slots"; - static constexpr char NumPendingReadsName[] = "Num pending reads"; + auto stackEntry = AZStd::make_shared( + splitSize, + aznumeric_caster(hardware.m_maxPhysicalSectorSize), + aznumeric_caster(hardware.m_maxLogicalSectorSize), + bufferSize, m_adjustOffset, m_splitAlignedRequests); + stackEntry->SetNext(AZStd::move(parent)); + return stackEntry; + } - ReadSplitter::ReadSplitter(u64 maxReadSize, u32 memoryAlignment, u32 sizeAlignment, size_t bufferSize, - bool adjustOffset, bool splitAlignedRequests) - : StreamStackEntry("Read splitter") - , m_buffer(nullptr) - , m_bufferSize(bufferSize) - , m_maxReadSize(maxReadSize) - , m_memoryAlignment(memoryAlignment) - , m_sizeAlignment(sizeAlignment) - , m_adjustOffset(adjustOffset) - , m_splitAlignedRequests(splitAlignedRequests) + void ReadSplitterConfig::Reflect(AZ::ReflectContext* context) + { + if (auto serializeContext = azrtti_cast(context); serializeContext != nullptr) { - AZ_Assert(IStreamerTypes::IsPowerOf2(memoryAlignment), "Memory alignment needs to be a power of 2"); - AZ_Assert(IStreamerTypes::IsPowerOf2(sizeAlignment), "Size alignment needs to be a power of 2"); - AZ_Assert(IStreamerTypes::IsAlignedTo(maxReadSize, sizeAlignment), - "Maximum read size isn't aligned to a multiple of the size alignment."); + serializeContext->Enum() + ->Version(1) + ->Value("MaxTransfer", SplitSize::MaxTransfer) + ->Value("MemoryAlignment", SplitSize::MemoryAlignment); - size_t numBufferSlots = bufferSize / maxReadSize; - // Don't divide the reads up in more sub-reads than there are dependencies available. - numBufferSlots = AZStd::min(numBufferSlots, FileRequest::GetMaxNumDependencies()); - m_bufferCopyInformation = AZStd::unique_ptr(new BufferCopyInformation[numBufferSlots]); - m_availableBufferSlots.reserve(numBufferSlots); - for (u32 i = aznumeric_caster(numBufferSlots); i > 0; --i) - { - m_availableBufferSlots.push_back(i - 1); - } + serializeContext->Class() + ->Version(1) + ->Field("BufferSizeMib", &ReadSplitterConfig::m_bufferSizeMib) + ->Field("SplitSize", &ReadSplitterConfig::m_splitSize) + ->Field("AdjustOffset", &ReadSplitterConfig::m_adjustOffset) + ->Field("SplitAlignedRequests", &ReadSplitterConfig::m_splitAlignedRequests); + } + } + + static constexpr char AvgNumSubReadsName[] = "Avg. num sub reads"; + static constexpr char AlignedReadsName[] = "Aligned reads"; + static constexpr char NumAvailableBufferSlotsName[] = "Num available buffer slots"; + static constexpr char NumPendingReadsName[] = "Num pending reads"; + + ReadSplitter::ReadSplitter(u64 maxReadSize, u32 memoryAlignment, u32 sizeAlignment, size_t bufferSize, + bool adjustOffset, bool splitAlignedRequests) + : StreamStackEntry("Read splitter") + , m_buffer(nullptr) + , m_bufferSize(bufferSize) + , m_maxReadSize(maxReadSize) + , m_memoryAlignment(memoryAlignment) + , m_sizeAlignment(sizeAlignment) + , m_adjustOffset(adjustOffset) + , m_splitAlignedRequests(splitAlignedRequests) + { + AZ_Assert(IStreamerTypes::IsPowerOf2(memoryAlignment), "Memory alignment needs to be a power of 2"); + AZ_Assert(IStreamerTypes::IsPowerOf2(sizeAlignment), "Size alignment needs to be a power of 2"); + AZ_Assert(IStreamerTypes::IsAlignedTo(maxReadSize, sizeAlignment), + "Maximum read size isn't aligned to a multiple of the size alignment."); + + size_t numBufferSlots = bufferSize / maxReadSize; + // Don't divide the reads up in more sub-reads than there are dependencies available. + numBufferSlots = AZStd::min(numBufferSlots, FileRequest::GetMaxNumDependencies()); + m_bufferCopyInformation = AZStd::unique_ptr(new BufferCopyInformation[numBufferSlots]); + m_availableBufferSlots.reserve(numBufferSlots); + for (u32 i = aznumeric_caster(numBufferSlots); i > 0; --i) + { + m_availableBufferSlots.push_back(i - 1); + } + } + + ReadSplitter::~ReadSplitter() + { + if (m_buffer) + { + AZ::AllocatorInstance::Get().DeAllocate(m_buffer, m_bufferSize, m_memoryAlignment); + } + } + + void ReadSplitter::QueueRequest(FileRequest* request) + { + AZ_Assert(request, "QueueRequest was provided a null request."); + if (!m_next) + { + request->SetStatus(IStreamerTypes::RequestStatus::Failed); + m_context->MarkRequestAsCompleted(request); + return; } - ReadSplitter::~ReadSplitter() + auto data = AZStd::get_if(&request->GetCommand()); + if (data == nullptr) { - if (m_buffer) - { - AZ::AllocatorInstance::Get().DeAllocate(m_buffer, m_bufferSize, m_memoryAlignment); - } + StreamStackEntry::QueueRequest(request); + return; } - void ReadSplitter::QueueRequest(FileRequest* request) - { - AZ_Assert(request, "QueueRequest was provided a null request."); - if (!m_next) - { - request->SetStatus(IStreamerTypes::RequestStatus::Failed); - m_context->MarkRequestAsCompleted(request); - return; - } + m_averageNumSubReadsStat.PushSample(aznumeric_cast((data->m_size / m_maxReadSize) + 1)); + Statistic::PlotImmediate(m_name, AvgNumSubReadsName, m_averageNumSubReadsStat.GetMostRecentSample()); - auto data = AZStd::get_if(&request->GetCommand()); - if (data == nullptr) + bool isAligned = IStreamerTypes::IsAlignedTo(data->m_output, m_memoryAlignment); + if (m_adjustOffset) + { + isAligned = isAligned && IStreamerTypes::IsAlignedTo(data->m_offset, m_sizeAlignment); + } + + if (isAligned || m_bufferSize == 0) + { + m_alignedReadsStat.PushSample(isAligned ? 1.0 : 0.0); + if (!m_splitAlignedRequests) { StreamStackEntry::QueueRequest(request); - return; - } - - m_averageNumSubReadsStat.PushSample(aznumeric_cast((data->m_size / m_maxReadSize) + 1)); - Statistic::PlotImmediate(m_name, AvgNumSubReadsName, m_averageNumSubReadsStat.GetMostRecentSample()); - - bool isAligned = IStreamerTypes::IsAlignedTo(data->m_output, m_memoryAlignment); - if (m_adjustOffset) - { - isAligned = isAligned && IStreamerTypes::IsAlignedTo(data->m_offset, m_sizeAlignment); - } - - if (isAligned || m_bufferSize == 0) - { - m_alignedReadsStat.PushSample(isAligned ? 1.0 : 0.0); - if (!m_splitAlignedRequests) - { - StreamStackEntry::QueueRequest(request); - } - else - { - QueueAlignedRead(request); - } } else { - m_alignedReadsStat.PushSample(0.0); - InitializeBuffer(); - QueueBufferedRead(request); + QueueAlignedRead(request); } } - - void ReadSplitter::QueueAlignedRead(FileRequest* request) + else { - auto data = AZStd::get_if(&request->GetCommand()); - AZ_Assert(data != nullptr, "Provided request to queue by the Read Splitter did not contain a read command."); + m_alignedReadsStat.PushSample(0.0); + InitializeBuffer(); + QueueBufferedRead(request); + } + } - if (data->m_size <= m_maxReadSize) - { - StreamStackEntry::QueueRequest(request); - return; - } + void ReadSplitter::QueueAlignedRead(FileRequest* request) + { + auto data = AZStd::get_if(&request->GetCommand()); + AZ_Assert(data != nullptr, "Provided request to queue by the Read Splitter did not contain a read command."); - PendingRead pendingRead; - pendingRead.m_request = request; - pendingRead.m_output = reinterpret_cast(data->m_output); - pendingRead.m_outputSize = data->m_outputSize; - pendingRead.m_readSize = data->m_size; - pendingRead.m_offset = data->m_offset; - pendingRead.m_isBuffered = false; - - if (!m_pendingReads.empty()) - { - m_pendingReads.push_back(pendingRead); - return; - } - - if (!QueueAlignedRead(pendingRead)) - { - m_pendingReads.push_back(pendingRead); - } + if (data->m_size <= m_maxReadSize) + { + StreamStackEntry::QueueRequest(request); + return; } - bool ReadSplitter::QueueAlignedRead(PendingRead& pending) + PendingRead pendingRead; + pendingRead.m_request = request; + pendingRead.m_output = reinterpret_cast(data->m_output); + pendingRead.m_outputSize = data->m_outputSize; + pendingRead.m_readSize = data->m_size; + pendingRead.m_offset = data->m_offset; + pendingRead.m_isBuffered = false; + + if (!m_pendingReads.empty()) { - auto data = AZStd::get_if(&pending.m_request->GetCommand()); - AZ_Assert(data != nullptr, "Provided request to queue by the Read Splitter did not contain a read command."); + m_pendingReads.push_back(pendingRead); + return; + } - while (pending.m_readSize > 0) + if (!QueueAlignedRead(pendingRead)) + { + m_pendingReads.push_back(pendingRead); + } + } + + bool ReadSplitter::QueueAlignedRead(PendingRead& pending) + { + auto data = AZStd::get_if(&pending.m_request->GetCommand()); + AZ_Assert(data != nullptr, "Provided request to queue by the Read Splitter did not contain a read command."); + + while (pending.m_readSize > 0) + { + if (pending.m_request->GetNumDependencies() >= FileRequest::GetMaxNumDependencies()) { - if (pending.m_request->GetNumDependencies() >= FileRequest::GetMaxNumDependencies()) + // Add a wait to make sure the read request isn't completed if all sub-reads completed before + // the ReadSplitter has had a chance to add new sub-reads to complete the read. + if (pending.m_wait == nullptr) { - // Add a wait to make sure the read request isn't completed if all sub-reads completed before - // the ReadSplitter has had a chance to add new sub-reads to complete the read. - if (pending.m_wait == nullptr) - { - pending.m_wait = m_context->GetNewInternalRequest(); - pending.m_wait->CreateWait(pending.m_request); - } - return false; + pending.m_wait = m_context->GetNewInternalRequest(); + pending.m_wait->CreateWait(pending.m_request); } + return false; + } - u64 readSize = m_maxReadSize; - size_t bufferSize = m_maxReadSize; - if (pending.m_readSize < m_maxReadSize) + u64 readSize = m_maxReadSize; + size_t bufferSize = m_maxReadSize; + if (pending.m_readSize < m_maxReadSize) + { + readSize = pending.m_readSize; + // This will be the last read so give the remainder of the output buffer to the final request. + bufferSize = pending.m_outputSize; + } + + FileRequest* subRequest = m_context->GetNewInternalRequest(); + subRequest->CreateRead(pending.m_request, pending.m_output, bufferSize, data->m_path, pending.m_offset, readSize, data->m_sharedRead); + subRequest->SetCompletionCallback([this](FileRequest&) { - readSize = pending.m_readSize; - // This will be the last read so give the remainder of the output buffer to the final request. - bufferSize = pending.m_outputSize; + AZ_PROFILE_FUNCTION(AzCore); + QueuePendingRequest(); + }); + m_next->QueueRequest(subRequest); + + pending.m_offset += readSize; + pending.m_readSize -= readSize; + pending.m_outputSize -= bufferSize; + pending.m_output += readSize; + } + if (pending.m_wait != nullptr) + { + m_context->MarkRequestAsCompleted(pending.m_wait); + pending.m_wait = nullptr; + } + return true; + } + + void ReadSplitter::QueueBufferedRead(FileRequest* request) + { + auto data = AZStd::get_if(&request->GetCommand()); + AZ_Assert(data != nullptr, "Provided request to queue by the Read Splitter did not contain a read command."); + + PendingRead pendingRead; + pendingRead.m_request = request; + pendingRead.m_output = reinterpret_cast(data->m_output); + pendingRead.m_outputSize = data->m_outputSize; + pendingRead.m_readSize = data->m_size; + pendingRead.m_offset = data->m_offset; + pendingRead.m_isBuffered = true; + + if (!m_pendingReads.empty()) + { + m_pendingReads.push_back(pendingRead); + return; + } + + if (!QueueBufferedRead(pendingRead)) + { + m_pendingReads.push_back(pendingRead); + } + } + + bool ReadSplitter::QueueBufferedRead(PendingRead& pending) + { + auto data = AZStd::get_if(&pending.m_request->GetCommand()); + AZ_Assert(data != nullptr, "Provided request to queue by the Read Splitter did not contain a read command."); + + while (pending.m_readSize > 0) + { + if (!m_availableBufferSlots.empty()) + { + u32 bufferSlot = m_availableBufferSlots.back(); + m_availableBufferSlots.pop_back(); + + u64 readSize; + u64 copySize; + u64 offset; + BufferCopyInformation& copyInfo = m_bufferCopyInformation[bufferSlot]; + copyInfo.m_target = pending.m_output; + + if (m_adjustOffset) + { + offset = AZ_SIZE_ALIGN_DOWN(pending.m_offset, aznumeric_cast(m_sizeAlignment)); + size_t bufferOffset = pending.m_offset - offset; + copyInfo.m_bufferOffset = bufferOffset; + readSize = AZStd::min(pending.m_readSize + bufferOffset, m_maxReadSize); + copySize = readSize - bufferOffset; } - + else + { + offset = pending.m_offset; + readSize = AZStd::min(pending.m_readSize, m_maxReadSize); + copySize = readSize; + } + AZ_Assert(readSize <= m_maxReadSize, "Read size %llu in read splitter exceeds the maximum split size of %llu.", + readSize, m_maxReadSize); + copyInfo.m_size = copySize; + FileRequest* subRequest = m_context->GetNewInternalRequest(); - subRequest->CreateRead(pending.m_request, pending.m_output, bufferSize, data->m_path, pending.m_offset, readSize, data->m_sharedRead); - subRequest->SetCompletionCallback([this](FileRequest&) + subRequest->CreateRead(pending.m_request, GetBufferSlot(bufferSlot), m_maxReadSize, data->m_path, + offset, readSize, data->m_sharedRead); + subRequest->SetCompletionCallback([this, bufferSlot]([[maybe_unused]] FileRequest& request) { AZ_PROFILE_FUNCTION(AzCore); + + BufferCopyInformation& copyInfo = m_bufferCopyInformation[bufferSlot]; + memcpy(copyInfo.m_target, GetBufferSlot(bufferSlot) + copyInfo.m_bufferOffset, copyInfo.m_size); + m_availableBufferSlots.push_back(bufferSlot); + QueuePendingRequest(); }); m_next->QueueRequest(subRequest); - pending.m_offset += readSize; - pending.m_readSize -= readSize; - pending.m_outputSize -= bufferSize; - pending.m_output += readSize; + pending.m_offset += copySize; + pending.m_readSize -= copySize; + pending.m_outputSize -= copySize; + pending.m_output += copySize; } - if (pending.m_wait != nullptr) + else { - m_context->MarkRequestAsCompleted(pending.m_wait); - pending.m_wait = nullptr; - } - return true; - } - - void ReadSplitter::QueueBufferedRead(FileRequest* request) - { - auto data = AZStd::get_if(&request->GetCommand()); - AZ_Assert(data != nullptr, "Provided request to queue by the Read Splitter did not contain a read command."); - - PendingRead pendingRead; - pendingRead.m_request = request; - pendingRead.m_output = reinterpret_cast(data->m_output); - pendingRead.m_outputSize = data->m_outputSize; - pendingRead.m_readSize = data->m_size; - pendingRead.m_offset = data->m_offset; - pendingRead.m_isBuffered = true; - - if (!m_pendingReads.empty()) - { - m_pendingReads.push_back(pendingRead); - return; - } - - if (!QueueBufferedRead(pendingRead)) - { - m_pendingReads.push_back(pendingRead); - } - } - - bool ReadSplitter::QueueBufferedRead(PendingRead& pending) - { - auto data = AZStd::get_if(&pending.m_request->GetCommand()); - AZ_Assert(data != nullptr, "Provided request to queue by the Read Splitter did not contain a read command."); - - while (pending.m_readSize > 0) - { - if (!m_availableBufferSlots.empty()) + // Add a wait to make sure the read request isn't completed if all sub-reads completed before + // the ReadSplitter has had a chance to add new sub-reads to complete the read. + if (pending.m_wait == nullptr) { - u32 bufferSlot = m_availableBufferSlots.back(); - m_availableBufferSlots.pop_back(); - - u64 readSize; - u64 copySize; - u64 offset; - BufferCopyInformation& copyInfo = m_bufferCopyInformation[bufferSlot]; - copyInfo.m_target = pending.m_output; - - if (m_adjustOffset) - { - offset = AZ_SIZE_ALIGN_DOWN(pending.m_offset, aznumeric_cast(m_sizeAlignment)); - size_t bufferOffset = pending.m_offset - offset; - copyInfo.m_bufferOffset = bufferOffset; - readSize = AZStd::min(pending.m_readSize + bufferOffset, m_maxReadSize); - copySize = readSize - bufferOffset; - } - else - { - offset = pending.m_offset; - readSize = AZStd::min(pending.m_readSize, m_maxReadSize); - copySize = readSize; - } - AZ_Assert(readSize <= m_maxReadSize, "Read size %llu in read splitter exceeds the maximum split size of %llu.", - readSize, m_maxReadSize); - copyInfo.m_size = copySize; - - FileRequest* subRequest = m_context->GetNewInternalRequest(); - subRequest->CreateRead(pending.m_request, GetBufferSlot(bufferSlot), m_maxReadSize, data->m_path, - offset, readSize, data->m_sharedRead); - subRequest->SetCompletionCallback([this, bufferSlot]([[maybe_unused]] FileRequest& request) - { - AZ_PROFILE_FUNCTION(AzCore); - - BufferCopyInformation& copyInfo = m_bufferCopyInformation[bufferSlot]; - memcpy(copyInfo.m_target, GetBufferSlot(bufferSlot) + copyInfo.m_bufferOffset, copyInfo.m_size); - m_availableBufferSlots.push_back(bufferSlot); - - QueuePendingRequest(); - }); - m_next->QueueRequest(subRequest); - - pending.m_offset += copySize; - pending.m_readSize -= copySize; - pending.m_outputSize -= copySize; - pending.m_output += copySize; - } - else - { - // Add a wait to make sure the read request isn't completed if all sub-reads completed before - // the ReadSplitter has had a chance to add new sub-reads to complete the read. - if (pending.m_wait == nullptr) - { - pending.m_wait = m_context->GetNewInternalRequest(); - pending.m_wait->CreateWait(pending.m_request); - } - return false; + pending.m_wait = m_context->GetNewInternalRequest(); + pending.m_wait->CreateWait(pending.m_request); } + return false; } - if (pending.m_wait != nullptr) + } + if (pending.m_wait != nullptr) + { + m_context->MarkRequestAsCompleted(pending.m_wait); + pending.m_wait = nullptr; + } + return true; + } + + void ReadSplitter::QueuePendingRequest() + { + if (!m_pendingReads.empty()) + { + PendingRead& pendingRead = m_pendingReads.front(); + if (pendingRead.m_isBuffered ? QueueBufferedRead(pendingRead) : QueueAlignedRead(pendingRead)) { - m_context->MarkRequestAsCompleted(pending.m_wait); - pending.m_wait = nullptr; - } - return true; - } - - void ReadSplitter::QueuePendingRequest() - { - if (!m_pendingReads.empty()) - { - PendingRead& pendingRead = m_pendingReads.front(); - if (pendingRead.m_isBuffered ? QueueBufferedRead(pendingRead) : QueueAlignedRead(pendingRead)) - { - m_pendingReads.pop_front(); - } + m_pendingReads.pop_front(); } } + } - void ReadSplitter::UpdateStatus(Status& status) const + void ReadSplitter::UpdateStatus(Status& status) const + { + StreamStackEntry::UpdateStatus(status); + if (m_bufferSize > 0) { - StreamStackEntry::UpdateStatus(status); - if (m_bufferSize > 0) - { - s32 numAvailableSlots = aznumeric_cast(m_availableBufferSlots.size()); - status.m_numAvailableSlots = AZStd::min(status.m_numAvailableSlots, numAvailableSlots); - status.m_isIdle = status.m_isIdle && m_pendingReads.empty(); - } + s32 numAvailableSlots = aznumeric_cast(m_availableBufferSlots.size()); + status.m_numAvailableSlots = AZStd::min(status.m_numAvailableSlots, numAvailableSlots); + status.m_isIdle = status.m_isIdle && m_pendingReads.empty(); } + } - void ReadSplitter::CollectStatistics(AZStd::vector& statistics) const - { - statistics.push_back(Statistic::CreateFloat(m_name, AvgNumSubReadsName, m_averageNumSubReadsStat.GetAverage())); - statistics.push_back(Statistic::CreatePercentage(m_name, AlignedReadsName, m_alignedReadsStat.GetAverage())); - statistics.push_back(Statistic::CreateInteger(m_name, NumAvailableBufferSlotsName, aznumeric_caster(m_availableBufferSlots.size()))); - statistics.push_back(Statistic::CreateInteger(m_name, NumPendingReadsName, aznumeric_caster(m_pendingReads.size()))); - StreamStackEntry::CollectStatistics(statistics); - } + void ReadSplitter::CollectStatistics(AZStd::vector& statistics) const + { + statistics.push_back(Statistic::CreateFloat(m_name, AvgNumSubReadsName, m_averageNumSubReadsStat.GetAverage())); + statistics.push_back(Statistic::CreatePercentage(m_name, AlignedReadsName, m_alignedReadsStat.GetAverage())); + statistics.push_back(Statistic::CreateInteger(m_name, NumAvailableBufferSlotsName, aznumeric_caster(m_availableBufferSlots.size()))); + statistics.push_back(Statistic::CreateInteger(m_name, NumPendingReadsName, aznumeric_caster(m_pendingReads.size()))); + StreamStackEntry::CollectStatistics(statistics); + } - void ReadSplitter::InitializeBuffer() + void ReadSplitter::InitializeBuffer() + { + // Lazy initialization to avoid allocating memory if it's not needed. + if (m_bufferSize != 0 && m_buffer == nullptr) { - // Lazy initialization to avoid allocating memory if it's not needed. - if (m_bufferSize != 0 && m_buffer == nullptr) - { - m_buffer = reinterpret_cast(AZ::AllocatorInstance::Get().Allocate( - m_bufferSize, m_memoryAlignment, 0, "AZ::IO::Streamer ReadSplitter", __FILE__, __LINE__)); - } + m_buffer = reinterpret_cast(AZ::AllocatorInstance::Get().Allocate( + m_bufferSize, m_memoryAlignment, 0, "AZ::IO::Streamer ReadSplitter", __FILE__, __LINE__)); } + } - u8* ReadSplitter::GetBufferSlot(size_t index) - { - AZ_Assert(m_buffer != nullptr, "A buffer slot was requested by the Read Splitter before the buffer was initialized."); - return m_buffer + (index * m_maxReadSize); - } - } // namespace IO -} // namesapce AZ + u8* ReadSplitter::GetBufferSlot(size_t index) + { + AZ_Assert(m_buffer != nullptr, "A buffer slot was requested by the Read Splitter before the buffer was initialized."); + return m_buffer + (index * m_maxReadSize); + } +} // namespace AZ::IO diff --git a/Code/Framework/AzCore/AzCore/IPC/SharedMemory.cpp b/Code/Framework/AzCore/AzCore/IPC/SharedMemory.cpp index af73ab4936..c3fa69a205 100644 --- a/Code/Framework/AzCore/AzCore/IPC/SharedMemory.cpp +++ b/Code/Framework/AzCore/AzCore/IPC/SharedMemory.cpp @@ -13,23 +13,18 @@ #include -namespace AZ +namespace AZ::Internal { - namespace Internal + struct RingData { - struct RingData - { - AZ::u32 m_readOffset; - AZ::u32 m_writeOffset; - AZ::u32 m_startOffset; - AZ::u32 m_endOffset; - AZ::u32 m_dataToRead; - AZ::u8 m_pad[32 - sizeof(AZStd::spin_mutex)]; - }; - } // namespace Internal -} // namespace AZ - - + AZ::u32 m_readOffset; + AZ::u32 m_writeOffset; + AZ::u32 m_startOffset; + AZ::u32 m_endOffset; + AZ::u32 m_dataToRead; + AZ::u8 m_pad[32 - sizeof(AZStd::spin_mutex)]; + }; +} // namespace AZ::Internal using namespace AZ; diff --git a/Code/Framework/AzCore/AzCore/Math/Geometry2DUtils.cpp b/Code/Framework/AzCore/AzCore/Math/Geometry2DUtils.cpp index 47a7e0a2db..912794a518 100644 --- a/Code/Framework/AzCore/AzCore/Math/Geometry2DUtils.cpp +++ b/Code/Framework/AzCore/AzCore/Math/Geometry2DUtils.cpp @@ -8,138 +8,135 @@ #include -namespace AZ +namespace AZ::Geometry2DUtils { - namespace Geometry2DUtils + float ShortestDistanceSqPointSegment(const Vector2& point, const Vector2& segmentStart, const Vector2& segmentEnd, + float epsilon) { - float ShortestDistanceSqPointSegment(const Vector2& point, const Vector2& segmentStart, const Vector2& segmentEnd, - float epsilon) + const AZ::Vector2 segmentVector = segmentEnd - segmentStart; + + // check if the line degenerates to a point + const float segmentLengthSq = segmentVector.GetLengthSq(); + if (segmentLengthSq < epsilon * epsilon) { - const AZ::Vector2 segmentVector = segmentEnd - segmentStart; - - // check if the line degenerates to a point - const float segmentLengthSq = segmentVector.GetLengthSq(); - if (segmentLengthSq < epsilon * epsilon) - { - return (point - segmentStart).GetLengthSq(); - } - - // if the point projects on to the line segment then the shortest distance is the perpendicular - const float projection = (point - segmentStart).Dot(segmentVector); - if (projection >= 0.0f && projection <= segmentLengthSq) - { - const Vector2 perpendicular = (point - segmentStart - projection / segmentLengthSq * segmentVector); - return perpendicular.GetLengthSq(); - } - - // otherwise the point must be closest to one of the end points of the segment - return GetMin( - (point - segmentStart).GetLengthSq(), - (point - segmentEnd).GetLengthSq()); + return (point - segmentStart).GetLengthSq(); } - float Signed2DTriangleArea(const Vector2& a, const Vector2& b, const Vector2& c) + // if the point projects on to the line segment then the shortest distance is the perpendicular + const float projection = (point - segmentStart).Dot(segmentVector); + if (projection >= 0.0f && projection <= segmentLengthSq) { - return 0.5f * ((a.GetX() - c.GetX()) * (b.GetY() - c.GetY()) - (a.GetY() - c.GetY()) * (b.GetX() - c.GetX())); + const Vector2 perpendicular = (point - segmentStart - projection / segmentLengthSq * segmentVector); + return perpendicular.GetLengthSq(); } - float ShortestDistanceSqSegmentSegment( - const Vector2& segment1Start, const Vector2& segment1End, - const Vector2& segment2Start, const Vector2& segment2End) + // otherwise the point must be closest to one of the end points of the segment + return GetMin( + (point - segmentStart).GetLengthSq(), + (point - segmentEnd).GetLengthSq()); + } + + float Signed2DTriangleArea(const Vector2& a, const Vector2& b, const Vector2& c) + { + return 0.5f * ((a.GetX() - c.GetX()) * (b.GetY() - c.GetY()) - (a.GetY() - c.GetY()) * (b.GetX() - c.GetX())); + } + + float ShortestDistanceSqSegmentSegment( + const Vector2& segment1Start, const Vector2& segment1End, + const Vector2& segment2Start, const Vector2& segment2End) + { + // if the segments cross, then the distance is zero + + // if the two ends of segment 2 are on different sides of segment 1, then these two triangles will have + // different winding orders (see Real-Time Collision Detection, Christer Ericson, ISBN 978-1558607323, + // Chapter 5.1.9.1) + const float area1 = Signed2DTriangleArea(segment1Start, segment1End, segment2End); + const float area2 = Signed2DTriangleArea(segment1Start, segment1End, segment2Start); + if (area1 * area2 < 0.0f) { - // if the segments cross, then the distance is zero - - // if the two ends of segment 2 are on different sides of segment 1, then these two triangles will have - // different winding orders (see Real-Time Collision Detection, Christer Ericson, ISBN 978-1558607323, - // Chapter 5.1.9.1) - const float area1 = Signed2DTriangleArea(segment1Start, segment1End, segment2End); - const float area2 = Signed2DTriangleArea(segment1Start, segment1End, segment2Start); - if (area1 * area2 < 0.0f) + // similarly we can check if the two ends of segment 1 are on different sides of segment 2 + const float area3 = Signed2DTriangleArea(segment2Start, segment2End, segment1Start); + const float area4 = area3 + area2 - area1; + if (area3 * area4 < 0.0f) { - // similarly we can check if the two ends of segment 1 are on different sides of segment 2 - const float area3 = Signed2DTriangleArea(segment2Start, segment2End, segment1Start); - const float area4 = area3 + area2 - area1; - if (area3 * area4 < 0.0f) - { - return 0.0f; - } + return 0.0f; } - - // otherwise the shortest distance must be between one of the segment end points and the other segment - return GetMin( - GetMin( - ShortestDistanceSqPointSegment(segment1Start, segment2Start, segment2End), - ShortestDistanceSqPointSegment(segment1End, segment2Start, segment2End)), - GetMin( - ShortestDistanceSqPointSegment(segment2Start, segment1Start, segment1End), - ShortestDistanceSqPointSegment(segment2End, segment1Start, segment1End)) - ); } - bool IsSimplePolygon(const AZStd::vector& vertices, float epsilon) + // otherwise the shortest distance must be between one of the segment end points and the other segment + return GetMin( + GetMin( + ShortestDistanceSqPointSegment(segment1Start, segment2Start, segment2End), + ShortestDistanceSqPointSegment(segment1End, segment2Start, segment2End)), + GetMin( + ShortestDistanceSqPointSegment(segment2Start, segment1Start, segment1End), + ShortestDistanceSqPointSegment(segment2End, segment1Start, segment1End)) + ); + } + + bool IsSimplePolygon(const AZStd::vector& vertices, float epsilon) + { + // note that this implementation is quadratic in the number of vertices + // if it becomes a bottleneck, there are approaches which are O(n log n), e.g. the Bentley-Ottmann algorithm + + const size_t vertexCount = vertices.size(); + + if (vertexCount < 3) { - // note that this implementation is quadratic in the number of vertices - // if it becomes a bottleneck, there are approaches which are O(n log n), e.g. the Bentley-Ottmann algorithm - - const size_t vertexCount = vertices.size(); - - if (vertexCount < 3) - { - return false; - } - - if (vertexCount == 3) - { - return true; - } - - const float epsilonSq = epsilon * epsilon; - - for (size_t i = 0; i < vertexCount; ++i) - { - // make it easy to nicely wrap indices - const size_t safeIndex = i + vertexCount; - - const size_t endIndex = (safeIndex - 1) % vertexCount; - const size_t beginIndex = (safeIndex + 2) % vertexCount; - - for (size_t j = beginIndex; j != endIndex; j = (j + 1) % vertexCount) - { - const float distSq = ShortestDistanceSqSegmentSegment( - vertices[i], - vertices[(i + 1) % vertexCount], - vertices[j], - vertices[(j + 1) % vertexCount] - ); - - if (distSq < epsilonSq) - { - return false; - } - } - } + return false; + } + if (vertexCount == 3) + { return true; } - bool IsConvex(const AZStd::vector& vertices) + const float epsilonSq = epsilon * epsilon; + + for (size_t i = 0; i < vertexCount; ++i) { - const size_t vertexCount = vertices.size(); + // make it easy to nicely wrap indices + const size_t safeIndex = i + vertexCount; - if (vertexCount < 3) - { - return false; - } + const size_t endIndex = (safeIndex - 1) % vertexCount; + const size_t beginIndex = (safeIndex + 2) % vertexCount; - for (size_t i = 0; i < vertexCount; ++i) + for (size_t j = beginIndex; j != endIndex; j = (j + 1) % vertexCount) { - if (Signed2DTriangleArea(vertices[i], vertices[(i + 1) % vertexCount], vertices[(i + 2) % vertexCount]) < 0.0f) + const float distSq = ShortestDistanceSqSegmentSegment( + vertices[i], + vertices[(i + 1) % vertexCount], + vertices[j], + vertices[(j + 1) % vertexCount] + ); + + if (distSq < epsilonSq) { return false; } } - - return true; } - } // namespace Geometry2DUtils -} // namespace AZ + + return true; + } + + bool IsConvex(const AZStd::vector& vertices) + { + const size_t vertexCount = vertices.size(); + + if (vertexCount < 3) + { + return false; + } + + for (size_t i = 0; i < vertexCount; ++i) + { + if (Signed2DTriangleArea(vertices[i], vertices[(i + 1) % vertexCount], vertices[(i + 2) % vertexCount]) < 0.0f) + { + return false; + } + } + + return true; + } +} // namespace AZ::Geometry2DUtils diff --git a/Code/Framework/AzCore/AzCore/Math/Sfmt.cpp b/Code/Framework/AzCore/AzCore/Math/Sfmt.cpp index 66404a9b3f..5f4dc9e5df 100644 --- a/Code/Framework/AzCore/AzCore/Math/Sfmt.cpp +++ b/Code/Framework/AzCore/AzCore/Math/Sfmt.cpp @@ -14,28 +14,26 @@ #include // for memset -namespace AZ +namespace AZ::SfmtInternal { - namespace SfmtInternal - { - static const int N32 = N * 4; - static const int N64 = N * 2; - static const int POS1 = 122; - static const int SL1 = 18; - static const int SR1 = 11; - static const int SL2 = 1; - static const int SR2 = 1; - static const unsigned int MSK1 = 0xdfffffefU; - static const unsigned int MSK2 = 0xddfecb7fU; - static const unsigned int MSK3 = 0xbffaffffU; - static const unsigned int MSK4 = 0xbffffff6U; - static const unsigned int PARITY1 = 0x00000001U; - static const unsigned int PARITY2 = 0x00000000U; - static const unsigned int PARITY3 = 0x00000000U; - static const unsigned int PARITY4 = 0x13c9e684U; + static const int N32 = N * 4; + static const int N64 = N * 2; + static const int POS1 = 122; + static const int SL1 = 18; + static const int SR1 = 11; + static const int SL2 = 1; + static const int SR2 = 1; + static const unsigned int MSK1 = 0xdfffffefU; + static const unsigned int MSK2 = 0xddfecb7fU; + static const unsigned int MSK3 = 0xbffaffffU; + static const unsigned int MSK4 = 0xbffffff6U; + static const unsigned int PARITY1 = 0x00000001U; + static const unsigned int PARITY2 = 0x00000000U; + static const unsigned int PARITY3 = 0x00000000U; + static const unsigned int PARITY4 = 0x13c9e684U; - /** a parity check vector which certificate the period of 2^{MEXP} */ - static unsigned int parity[4] = {PARITY1, PARITY2, PARITY3, PARITY4}; + /** a parity check vector which certificate the period of 2^{MEXP} */ + static unsigned int parity[4] = {PARITY1, PARITY2, PARITY3, PARITY4}; #ifdef ONLY64 # define idxof(_i) (_i ^ 1) @@ -45,259 +43,257 @@ namespace AZ #if AZ_TRAIT_USE_PLATFORM_SIMD_SSE - /** - * This function represents the recursion formula. - * @param a a 128-bit part of the internal state array - * @param b a 128-bit part of the internal state array - * @param c a 128-bit part of the internal state array - * @param d a 128-bit part of the internal state array - * @param mask 128-bit mask - * @return output - */ - AZ_FORCE_INLINE static Simd::Vec4::Int32Type simd_recursion(Simd::Vec4::Int32Type* a, Simd::Vec4::Int32Type* b, Simd::Vec4::Int32Type c, Simd::Vec4::Int32Type d, Simd::Vec4::Int32Type mask) + /** + * This function represents the recursion formula. + * @param a a 128-bit part of the internal state array + * @param b a 128-bit part of the internal state array + * @param c a 128-bit part of the internal state array + * @param d a 128-bit part of the internal state array + * @param mask 128-bit mask + * @return output + */ + AZ_FORCE_INLINE static Simd::Vec4::Int32Type simd_recursion(Simd::Vec4::Int32Type* a, Simd::Vec4::Int32Type* b, Simd::Vec4::Int32Type c, Simd::Vec4::Int32Type d, Simd::Vec4::Int32Type mask) + { + Simd::Vec4::Int32Type v, x, y, z; + x = *a; + y = _mm_srli_epi32(*b, SR1); + z = _mm_srli_si128(c, SR2); + v = _mm_slli_epi32(d, SL1); + z = Simd::Vec4::Xor(z, x); + z = Simd::Vec4::Xor(z, v); + x = _mm_slli_si128(x, SL2); + y = Simd::Vec4::And(y, mask); + z = Simd::Vec4::Xor(z, x); + z = Simd::Vec4::Xor(z, y); + return z; + } + + /** + * This function fills the internal state array with pseudorandom + * integers. + */ + inline void gen_rand_all(Sfmt& g) + { + int i; + Simd::Vec4::Int32Type r, r1, r2, mask; + mask = Simd::Vec4::LoadImmediate((int32_t)MSK4, (int32_t)MSK3, (int32_t)MSK2, (int32_t)MSK1); + + r1 = Simd::Vec4::LoadAligned((const int32_t*)&g.m_sfmt[N - 2].si); + r2 = Simd::Vec4::LoadAligned((const int32_t*)&g.m_sfmt[N - 1].si); + for (i = 0; i < N - POS1; i++) { - Simd::Vec4::Int32Type v, x, y, z; - x = *a; - y = _mm_srli_epi32(*b, SR1); - z = _mm_srli_si128(c, SR2); - v = _mm_slli_epi32(d, SL1); - z = Simd::Vec4::Xor(z, x); - z = Simd::Vec4::Xor(z, v); - x = _mm_slli_si128(x, SL2); - y = Simd::Vec4::And(y, mask); - z = Simd::Vec4::Xor(z, x); - z = Simd::Vec4::Xor(z, y); - return z; + r = simd_recursion(&g.m_sfmt[i].si, &g.m_sfmt[i + POS1].si, r1, r2, mask); + Simd::Vec4::StoreAligned((int32_t*)&g.m_sfmt[i].si, r); + r1 = r2; + r2 = r; } - - /** - * This function fills the internal state array with pseudorandom - * integers. - */ - inline void gen_rand_all(Sfmt& g) + for (; i < N; i++) { - int i; - Simd::Vec4::Int32Type r, r1, r2, mask; - mask = Simd::Vec4::LoadImmediate((int32_t)MSK4, (int32_t)MSK3, (int32_t)MSK2, (int32_t)MSK1); - - r1 = Simd::Vec4::LoadAligned((const int32_t*)&g.m_sfmt[N - 2].si); - r2 = Simd::Vec4::LoadAligned((const int32_t*)&g.m_sfmt[N - 1].si); - for (i = 0; i < N - POS1; i++) - { - r = simd_recursion(&g.m_sfmt[i].si, &g.m_sfmt[i + POS1].si, r1, r2, mask); - Simd::Vec4::StoreAligned((int32_t*)&g.m_sfmt[i].si, r); - r1 = r2; - r2 = r; - } - for (; i < N; i++) - { - r = simd_recursion(&g.m_sfmt[i].si, &g.m_sfmt[i + POS1 - N].si, r1, r2, mask); - Simd::Vec4::StoreAligned((int32_t*)&g.m_sfmt[i].si, r); - r1 = r2; - r2 = r; - } + r = simd_recursion(&g.m_sfmt[i].si, &g.m_sfmt[i + POS1 - N].si, r1, r2, mask); + Simd::Vec4::StoreAligned((int32_t*)&g.m_sfmt[i].si, r); + r1 = r2; + r2 = r; } + } - /** - * This function fills the user-specified array with pseudorandom - * integers. - * - * @param array an 128-bit array to be filled by pseudorandom numbers. - * @param size number of 128-bit pesudorandom numbers to be generated. - */ - inline void gen_rand_array(Sfmt& g, w128_t* array, int size) + /** + * This function fills the user-specified array with pseudorandom + * integers. + * + * @param array an 128-bit array to be filled by pseudorandom numbers. + * @param size number of 128-bit pesudorandom numbers to be generated. + */ + inline void gen_rand_array(Sfmt& g, w128_t* array, int size) + { + int i, j; + Simd::Vec4::Int32Type r, r1, r2, mask; + mask = Simd::Vec4::LoadImmediate((int32_t)MSK4, (int32_t)MSK3, (int32_t)MSK2, (int32_t)MSK1); + + r1 = Simd::Vec4::LoadAligned((const int32_t*)&g.m_sfmt[N - 2].si); + r2 = Simd::Vec4::LoadAligned((const int32_t*)&g.m_sfmt[N - 1].si); + for (i = 0; i < N - POS1; i++) { - int i, j; - Simd::Vec4::Int32Type r, r1, r2, mask; - mask = Simd::Vec4::LoadImmediate((int32_t)MSK4, (int32_t)MSK3, (int32_t)MSK2, (int32_t)MSK1); - - r1 = Simd::Vec4::LoadAligned((const int32_t*)&g.m_sfmt[N - 2].si); - r2 = Simd::Vec4::LoadAligned((const int32_t*)&g.m_sfmt[N - 1].si); - for (i = 0; i < N - POS1; i++) - { - r = simd_recursion(&g.m_sfmt[i].si, &g.m_sfmt[i + POS1].si, r1, r2, mask); - Simd::Vec4::StoreAligned((int32_t*)&array[i].si, r); - r1 = r2; - r2 = r; - } - for (; i < N; i++) - { - r = simd_recursion(&g.m_sfmt[i].si, &array[i + POS1 - N].si, r1, r2, mask); - Simd::Vec4::StoreAligned((int32_t*)&array[i].si, r); - r1 = r2; - r2 = r; - } - /* main loop */ - for (; i < size - N; i++) - { - r = simd_recursion(&array[i - N].si, &array[i + POS1 - N].si, r1, r2, mask); - Simd::Vec4::StoreAligned((int32_t*)&array[i].si, r); - r1 = r2; - r2 = r; - } - for (j = 0; j < 2 * N - size; j++) - { - r = Simd::Vec4::LoadAligned((const int32_t*)&array[j + size - N].si); - Simd::Vec4::StoreAligned((int32_t*)&g.m_sfmt[j].si, r); - } - for (; i < size; i++) - { - r = simd_recursion(&array[i - N].si, &array[i + POS1 - N].si, r1, r2, mask); - Simd::Vec4::StoreAligned((int32_t*)&array[i].si, r); - Simd::Vec4::StoreAligned((int32_t*)&g.m_sfmt[j++].si, r); - r1 = r2; - r2 = r; - } + r = simd_recursion(&g.m_sfmt[i].si, &g.m_sfmt[i + POS1].si, r1, r2, mask); + Simd::Vec4::StoreAligned((int32_t*)&array[i].si, r); + r1 = r2; + r2 = r; } + for (; i < N; i++) + { + r = simd_recursion(&g.m_sfmt[i].si, &array[i + POS1 - N].si, r1, r2, mask); + Simd::Vec4::StoreAligned((int32_t*)&array[i].si, r); + r1 = r2; + r2 = r; + } + /* main loop */ + for (; i < size - N; i++) + { + r = simd_recursion(&array[i - N].si, &array[i + POS1 - N].si, r1, r2, mask); + Simd::Vec4::StoreAligned((int32_t*)&array[i].si, r); + r1 = r2; + r2 = r; + } + for (j = 0; j < 2 * N - size; j++) + { + r = Simd::Vec4::LoadAligned((const int32_t*)&array[j + size - N].si); + Simd::Vec4::StoreAligned((int32_t*)&g.m_sfmt[j].si, r); + } + for (; i < size; i++) + { + r = simd_recursion(&array[i - N].si, &array[i + POS1 - N].si, r1, r2, mask); + Simd::Vec4::StoreAligned((int32_t*)&array[i].si, r); + Simd::Vec4::StoreAligned((int32_t*)&g.m_sfmt[j++].si, r); + r1 = r2; + r2 = r; + } + } #else - inline void rshift128(w128_t* out, w128_t const* in, int shift) + inline void rshift128(w128_t* out, w128_t const* in, int shift) + { + AZ::u64 th, tl, oh, ol; + #ifdef ONLY64 + th = ((AZ::u64)in->u[2] << 32) | ((AZ::u64)in->u[3]); + tl = ((AZ::u64)in->u[0] << 32) | ((AZ::u64)in->u[1]); + + oh = th >> (shift * 8); + ol = tl >> (shift * 8); + ol |= th << (64 - shift * 8); + out->u[0] = (AZ::u32)(ol >> 32); + out->u[1] = (AZ::u32)ol; + out->u[2] = (AZ::u32)(oh >> 32); + out->u[3] = (AZ::u32)oh; +#else + th = ((AZ::u64)in->u[3] << 32) | ((AZ::u64)in->u[2]); + tl = ((AZ::u64)in->u[1] << 32) | ((AZ::u64)in->u[0]); + + oh = th >> (shift * 8); + ol = tl >> (shift * 8); + ol |= th << (64 - shift * 8); + out->u[1] = (AZ::u32)(ol >> 32); + out->u[0] = (AZ::u32)ol; + out->u[3] = (AZ::u32)(oh >> 32); + out->u[2] = (AZ::u32)oh; +#endif + } + + inline void lshift128(w128_t* out, w128_t const* in, int shift) + { + AZ::u64 th, tl, oh, ol; +#ifdef ONLY64 + th = ((AZ::u64)in->u[2] << 32) | ((AZ::u64)in->u[3]); + tl = ((AZ::u64)in->u[0] << 32) | ((AZ::u64)in->u[1]); + + oh = th << (shift * 8); + ol = tl << (shift * 8); + oh |= tl >> (64 - shift * 8); + out->u[0] = (AZ::u32)(ol >> 32); + out->u[1] = (AZ::u32)ol; + out->u[2] = (AZ::u32)(oh >> 32); + out->u[3] = (AZ::u32)oh; +#else + th = ((AZ::u64)in->u[3] << 32) | ((AZ::u64)in->u[2]); + tl = ((AZ::u64)in->u[1] << 32) | ((AZ::u64)in->u[0]); + + oh = th << (shift * 8); + ol = tl << (shift * 8); + oh |= tl >> (64 - shift * 8); + out->u[1] = (AZ::u32)(ol >> 32); + out->u[0] = (AZ::u32)ol; + out->u[3] = (AZ::u32)(oh >> 32); + out->u[2] = (AZ::u32)oh; +#endif + } + + inline void do_recursion(w128_t* r, w128_t* a, w128_t* b, w128_t* c, w128_t* d) + { + w128_t x; + w128_t y; + lshift128(&x, a, SL2); + rshift128(&y, c, SR2); +#ifdef ONLY64 + r->u[0] = a->u[0] ^ x.u[0] ^ ((b->u[0] >> SR1) & MSK2) ^ y.u[0] ^ (d->u[0] << SL1); + r->u[1] = a->u[1] ^ x.u[1] ^ ((b->u[1] >> SR1) & MSK1) ^ y.u[1] ^ (d->u[1] << SL1); + r->u[2] = a->u[2] ^ x.u[2] ^ ((b->u[2] >> SR1) & MSK4) ^ y.u[2] ^ (d->u[2] << SL1); + r->u[3] = a->u[3] ^ x.u[3] ^ ((b->u[3] >> SR1) & MSK3) ^ y.u[3] ^ (d->u[3] << SL1); +#else + r->u[0] = a->u[0] ^ x.u[0] ^ ((b->u[0] >> SR1) & MSK1) ^ y.u[0] ^ (d->u[0] << SL1); + r->u[1] = a->u[1] ^ x.u[1] ^ ((b->u[1] >> SR1) & MSK2) ^ y.u[1] ^ (d->u[1] << SL1); + r->u[2] = a->u[2] ^ x.u[2] ^ ((b->u[2] >> SR1) & MSK3) ^ y.u[2] ^ (d->u[2] << SL1); + r->u[3] = a->u[3] ^ x.u[3] ^ ((b->u[3] >> SR1) & MSK4) ^ y.u[3] ^ (d->u[3] << SL1); +#endif + } + /** + * This function fills the internal state array with pseudorandom + * integers. + */ + inline void gen_rand_all(Sfmt& g) + { + int i; + w128_t* r1, * r2; + + r1 = &g.m_sfmt[N - 2]; + r2 = &g.m_sfmt[N - 1]; + for (i = 0; i < N - POS1; i++) { - AZ::u64 th, tl, oh, ol; - #ifdef ONLY64 - th = ((AZ::u64)in->u[2] << 32) | ((AZ::u64)in->u[3]); - tl = ((AZ::u64)in->u[0] << 32) | ((AZ::u64)in->u[1]); - - oh = th >> (shift * 8); - ol = tl >> (shift * 8); - ol |= th << (64 - shift * 8); - out->u[0] = (AZ::u32)(ol >> 32); - out->u[1] = (AZ::u32)ol; - out->u[2] = (AZ::u32)(oh >> 32); - out->u[3] = (AZ::u32)oh; - #else - th = ((AZ::u64)in->u[3] << 32) | ((AZ::u64)in->u[2]); - tl = ((AZ::u64)in->u[1] << 32) | ((AZ::u64)in->u[0]); - - oh = th >> (shift * 8); - ol = tl >> (shift * 8); - ol |= th << (64 - shift * 8); - out->u[1] = (AZ::u32)(ol >> 32); - out->u[0] = (AZ::u32)ol; - out->u[3] = (AZ::u32)(oh >> 32); - out->u[2] = (AZ::u32)oh; - #endif + do_recursion(&g.m_sfmt[i], &g.m_sfmt[i], &g.m_sfmt[i + POS1], r1, r2); + r1 = r2; + r2 = &g.m_sfmt[i]; } - - inline void lshift128(w128_t* out, w128_t const* in, int shift) + for (; i < N; i++) { - AZ::u64 th, tl, oh, ol; - #ifdef ONLY64 - th = ((AZ::u64)in->u[2] << 32) | ((AZ::u64)in->u[3]); - tl = ((AZ::u64)in->u[0] << 32) | ((AZ::u64)in->u[1]); - - oh = th << (shift * 8); - ol = tl << (shift * 8); - oh |= tl >> (64 - shift * 8); - out->u[0] = (AZ::u32)(ol >> 32); - out->u[1] = (AZ::u32)ol; - out->u[2] = (AZ::u32)(oh >> 32); - out->u[3] = (AZ::u32)oh; - #else - th = ((AZ::u64)in->u[3] << 32) | ((AZ::u64)in->u[2]); - tl = ((AZ::u64)in->u[1] << 32) | ((AZ::u64)in->u[0]); - - oh = th << (shift * 8); - ol = tl << (shift * 8); - oh |= tl >> (64 - shift * 8); - out->u[1] = (AZ::u32)(ol >> 32); - out->u[0] = (AZ::u32)ol; - out->u[3] = (AZ::u32)(oh >> 32); - out->u[2] = (AZ::u32)oh; - #endif + do_recursion(&g.m_sfmt[i], &g.m_sfmt[i], &g.m_sfmt[i + POS1 - N], r1, r2); + r1 = r2; + r2 = &g.m_sfmt[i]; } + } - inline void do_recursion(w128_t* r, w128_t* a, w128_t* b, w128_t* c, w128_t* d) + /** + * This function fills the user-specified array with pseudorandom + * integers. + * + * @param array an 128-bit array to be filled by pseudorandom numbers. + * @param size number of 128-bit pseudorandom numbers to be generated. + */ + inline void gen_rand_array(Sfmt& g, w128_t* array, int size) + { + int i, j; + w128_t* r1, * r2; + + r1 = &g.m_sfmt[N - 2]; + r2 = &g.m_sfmt[N - 1]; + for (i = 0; i < N - POS1; i++) { - w128_t x; - w128_t y; - lshift128(&x, a, SL2); - rshift128(&y, c, SR2); - #ifdef ONLY64 - r->u[0] = a->u[0] ^ x.u[0] ^ ((b->u[0] >> SR1) & MSK2) ^ y.u[0] ^ (d->u[0] << SL1); - r->u[1] = a->u[1] ^ x.u[1] ^ ((b->u[1] >> SR1) & MSK1) ^ y.u[1] ^ (d->u[1] << SL1); - r->u[2] = a->u[2] ^ x.u[2] ^ ((b->u[2] >> SR1) & MSK4) ^ y.u[2] ^ (d->u[2] << SL1); - r->u[3] = a->u[3] ^ x.u[3] ^ ((b->u[3] >> SR1) & MSK3) ^ y.u[3] ^ (d->u[3] << SL1); - #else - r->u[0] = a->u[0] ^ x.u[0] ^ ((b->u[0] >> SR1) & MSK1) ^ y.u[0] ^ (d->u[0] << SL1); - r->u[1] = a->u[1] ^ x.u[1] ^ ((b->u[1] >> SR1) & MSK2) ^ y.u[1] ^ (d->u[1] << SL1); - r->u[2] = a->u[2] ^ x.u[2] ^ ((b->u[2] >> SR1) & MSK3) ^ y.u[2] ^ (d->u[2] << SL1); - r->u[3] = a->u[3] ^ x.u[3] ^ ((b->u[3] >> SR1) & MSK4) ^ y.u[3] ^ (d->u[3] << SL1); - #endif + do_recursion(&array[i], &g.m_sfmt[i], &g.m_sfmt[i + POS1], r1, r2); + r1 = r2; + r2 = &array[i]; } - /** - * This function fills the internal state array with pseudorandom - * integers. - */ - inline void gen_rand_all(Sfmt& g) + for (; i < N; i++) { - int i; - w128_t* r1, * r2; - - r1 = &g.m_sfmt[N - 2]; - r2 = &g.m_sfmt[N - 1]; - for (i = 0; i < N - POS1; i++) - { - do_recursion(&g.m_sfmt[i], &g.m_sfmt[i], &g.m_sfmt[i + POS1], r1, r2); - r1 = r2; - r2 = &g.m_sfmt[i]; - } - for (; i < N; i++) - { - do_recursion(&g.m_sfmt[i], &g.m_sfmt[i], &g.m_sfmt[i + POS1 - N], r1, r2); - r1 = r2; - r2 = &g.m_sfmt[i]; - } + do_recursion(&array[i], &g.m_sfmt[i], &array[i + POS1 - N], r1, r2); + r1 = r2; + r2 = &array[i]; } - - /** - * This function fills the user-specified array with pseudorandom - * integers. - * - * @param array an 128-bit array to be filled by pseudorandom numbers. - * @param size number of 128-bit pseudorandom numbers to be generated. - */ - inline void gen_rand_array(Sfmt& g, w128_t* array, int size) + for (; i < size - N; i++) { - int i, j; - w128_t* r1, * r2; - - r1 = &g.m_sfmt[N - 2]; - r2 = &g.m_sfmt[N - 1]; - for (i = 0; i < N - POS1; i++) - { - do_recursion(&array[i], &g.m_sfmt[i], &g.m_sfmt[i + POS1], r1, r2); - r1 = r2; - r2 = &array[i]; - } - for (; i < N; i++) - { - do_recursion(&array[i], &g.m_sfmt[i], &array[i + POS1 - N], r1, r2); - r1 = r2; - r2 = &array[i]; - } - for (; i < size - N; i++) - { - do_recursion(&array[i], &array[i - N], &array[i + POS1 - N], r1, r2); - r1 = r2; - r2 = &array[i]; - } - for (j = 0; j < 2 * N - size; j++) - { - g.m_sfmt[j] = array[j + size - N]; - } - for (; i < size; i++, j++) - { - do_recursion(&array[i], &array[i - N], &array[i + POS1 - N], r1, r2); - r1 = r2; - r2 = &array[i]; - g.m_sfmt[j] = array[i]; - } + do_recursion(&array[i], &array[i - N], &array[i + POS1 - N], r1, r2); + r1 = r2; + r2 = &array[i]; } + for (j = 0; j < 2 * N - size; j++) + { + g.m_sfmt[j] = array[j + size - N]; + } + for (; i < size; i++, j++) + { + do_recursion(&array[i], &array[i - N], &array[i + POS1 - N], r1, r2); + r1 = r2; + r2 = &array[i]; + g.m_sfmt[j] = array[i]; + } + } #endif - } // SmftInternal -} // AZ - +} // namespace AZ::SfmtInternal using namespace AZ; diff --git a/Code/Framework/AzCore/AzCore/Math/Uuid.h b/Code/Framework/AzCore/AzCore/Math/Uuid.h index 6b77e7ec3e..eeacb04877 100644 --- a/Code/Framework/AzCore/AzCore/Math/Uuid.h +++ b/Code/Framework/AzCore/AzCore/Math/Uuid.h @@ -45,7 +45,7 @@ namespace AZ static constexpr int ValidUuidStringLength = 32; /// Number of characters (data only, no extra formatting) in a valid UUID string static const size_t MaxStringBuffer = 39; /// 32 Uuid + 4 dashes + 2 brackets + 1 terminate - Uuid() {} + Uuid() = default; Uuid(const char* string, size_t stringLength = 0) { *this = CreateString(string, stringLength); } static Uuid CreateNull(); diff --git a/Code/Framework/AzCore/AzCore/Math/Vector2.h b/Code/Framework/AzCore/AzCore/Math/Vector2.h index 7c37d74135..91eb61d6c8 100644 --- a/Code/Framework/AzCore/AzCore/Math/Vector2.h +++ b/Code/Framework/AzCore/AzCore/Math/Vector2.h @@ -30,7 +30,7 @@ namespace AZ Vector2() = default; - Vector2(const Vector2& v); + Vector2(const Vector2& v) = default; //! Constructs vector with all components set to the same specified value. explicit Vector2(float x); diff --git a/Code/Framework/AzCore/AzCore/Math/Vector2.inl b/Code/Framework/AzCore/AzCore/Math/Vector2.inl index 086be2bbc3..9691dd3a5c 100644 --- a/Code/Framework/AzCore/AzCore/Math/Vector2.inl +++ b/Code/Framework/AzCore/AzCore/Math/Vector2.inl @@ -8,13 +8,6 @@ namespace AZ { - AZ_MATH_INLINE Vector2::Vector2(const Vector2& v) - : m_value(v.m_value) - { - ; - } - - AZ_MATH_INLINE Vector2::Vector2(float x) : m_value(Simd::Vec2::Splat(x)) { diff --git a/Code/Framework/AzCore/AzCore/Memory/AllocatorManager.cpp b/Code/Framework/AzCore/AzCore/Memory/AllocatorManager.cpp index 3c09b4cae6..44ce08ebd9 100644 --- a/Code/Framework/AzCore/AzCore/Memory/AllocatorManager.cpp +++ b/Code/Framework/AzCore/AzCore/Memory/AllocatorManager.cpp @@ -20,44 +20,42 @@ #include #include -using namespace AZ; - #if !defined(RELEASE) && !defined(AZCORE_MEMORY_ENABLE_OVERRIDES) # define AZCORE_MEMORY_ENABLE_OVERRIDES #endif -namespace AZ +namespace AZ::Internal { - namespace Internal + struct AMStringHasher { - struct AMStringHasher + using is_transparent = void; + template + size_t operator()(const ConvertibleToStringView& key) { - using is_transparent = void; - template - size_t operator()(const ConvertibleToStringView& key) - { - return AZStd::hash{}(key); - } - }; - using AMString = AZStd::basic_string, AZStdIAllocator>; - using AllocatorNameMap = AZStd::unordered_map, AZStdIAllocator>; - using AllocatorRemappings = AZStd::unordered_map, AZStdIAllocator>; + return AZStd::hash{}(key); + } + }; + using AMString = AZStd::basic_string, AZStdIAllocator>; + using AllocatorNameMap = AZStd::unordered_map, AZStdIAllocator>; + using AllocatorRemappings = AZStd::unordered_map, AZStdIAllocator>; - // For allocators that are created before we have an environment, we keep some module-local data for them so that we can register them - // properly once the environment is attached. - struct PreEnvironmentAttachData - { - static const int MAX_UNREGISTERED_ALLOCATORS = 8; - AZStd::mutex m_mutex; - MallocSchema m_mallocSchema; - IAllocator* m_unregisteredAllocators[MAX_UNREGISTERED_ALLOCATORS]; - int m_unregisteredAllocatorCount = 0; - }; + // For allocators that are created before we have an environment, we keep some module-local data for them so that we can register them + // properly once the environment is attached. + struct PreEnvironmentAttachData + { + static const int MAX_UNREGISTERED_ALLOCATORS = 8; + AZStd::mutex m_mutex; + MallocSchema m_mallocSchema; + IAllocator* m_unregisteredAllocators[MAX_UNREGISTERED_ALLOCATORS]; + int m_unregisteredAllocatorCount = 0; + }; - } } -struct AZ::AllocatorManager::InternalData +namespace AZ +{ + +struct AllocatorManager::InternalData { explicit InternalData(const AZStdIAllocator& alloc) : m_allocatorMap(alloc) @@ -69,13 +67,13 @@ struct AZ::AllocatorManager::InternalData Internal::AllocatorRemappings m_remappingsReverse; }; -static AZ::EnvironmentVariable s_allocManager = nullptr; +static EnvironmentVariable s_allocManager = nullptr; static AllocatorManager* s_allocManagerDebug = nullptr; // For easier viewing in crash dumps /// Returns a module-local instance of data to use for allocators that are created before the environment is attached. -static AZ::Internal::PreEnvironmentAttachData& GetPreEnvironmentAttachData() +static Internal::PreEnvironmentAttachData& GetPreEnvironmentAttachData() { - static AZ::Internal::PreEnvironmentAttachData s_data; + static Internal::PreEnvironmentAttachData s_data; return s_data; } @@ -131,7 +129,7 @@ AllocatorManager& AllocatorManager::Instance() if (!s_allocManager) { AZ_Assert(Environment::IsReady(), "Environment must be ready before calling Instance()"); - s_allocManager = AZ::Environment::CreateVariable(AZ_CRC("AZ::AllocatorManager::s_allocManager", 0x6bdd908c)); + s_allocManager = Environment::CreateVariable(AZ_CRC_CE("AZ::AllocatorManager::s_allocManager")); // Register any allocators that were created in this module before we attached to the environment auto& data = GetPreEnvironmentAttachData(); @@ -156,9 +154,9 @@ AllocatorManager& AllocatorManager::Instance() ////////////////////////////////////////////////////////////////////////// // Create malloc schema using custom AZ_OS_MALLOC allocator. -AZ::MallocSchema* AllocatorManager::CreateMallocSchema() +MallocSchema* AllocatorManager::CreateMallocSchema() { - return static_cast(new(AZ_OS_MALLOC(sizeof(AZ::MallocSchema), alignof(AZ::MallocSchema))) AZ::MallocSchema()); + return static_cast(new(AZ_OS_MALLOC(sizeof(MallocSchema), alignof(MallocSchema))) MallocSchema()); } @@ -168,7 +166,7 @@ AZ::MallocSchema* AllocatorManager::CreateMallocSchema() //========================================================================= AllocatorManager::AllocatorManager() : m_profilingRefcount(0) - , m_mallocSchema(CreateMallocSchema(), [](AZ::MallocSchema* schema) + , m_mallocSchema(CreateMallocSchema(), [](MallocSchema* schema) { if (schema) { @@ -182,7 +180,7 @@ AllocatorManager::AllocatorManager() m_numAllocators = 0; m_isAllocatorLeaking = false; m_configurationFinalized = false; - m_defaultTrackingRecordMode = AZ::Debug::AllocationRecords::RECORD_NO_RECORDS; + m_defaultTrackingRecordMode = Debug::AllocationRecords::RECORD_NO_RECORDS; m_data = new (m_mallocSchema->Allocate(sizeof(InternalData), AZStd::alignment_of::value, 0)) InternalData(AZStdIAllocator(m_mallocSchema.get())); } @@ -411,12 +409,12 @@ AllocatorManager::RemoveOutOfMemoryListener() // [9/16/2011] //========================================================================= void -AllocatorManager::SetTrackingMode(AZ::Debug::AllocationRecords::Mode mode) +AllocatorManager::SetTrackingMode(Debug::AllocationRecords::Mode mode) { AZStd::lock_guard lock(m_allocatorListMutex); for (int i = 0; i < m_numAllocators; ++i) { - AZ::Debug::AllocationRecords* records = m_allocators[i]->GetRecords(); + Debug::AllocationRecords* records = m_allocators[i]->GetRecords(); if (records) { records->SetMode(mode); @@ -595,31 +593,31 @@ void AllocatorManager::GetAllocatorStats(size_t& allocatedBytes, size_t& capacit AZStd::lock_guard lock(m_allocatorListMutex); const int allocatorCount = GetNumAllocators(); - AZStd::unordered_map existingAllocators; - AZStd::unordered_map sourcesToAllocators; + AZStd::unordered_map existingAllocators; + AZStd::unordered_map sourcesToAllocators; // Build a mapping of original allocator sources to their allocators for (int i = 0; i < allocatorCount; ++i) { - AZ::IAllocator* allocator = GetAllocator(i); + IAllocator* allocator = GetAllocator(i); sourcesToAllocators.emplace(allocator->GetOriginalAllocationSource(), allocator); } for (int i = 0; i < allocatorCount; ++i) { - AZ::IAllocator* allocator = GetAllocator(i); - AZ::IAllocatorAllocate* source = allocator->GetAllocationSource(); - AZ::IAllocatorAllocate* originalSource = allocator->GetOriginalAllocationSource(); - AZ::IAllocatorAllocate* schema = allocator->GetSchema(); - AZ::IAllocator* alias = (source != originalSource) ? sourcesToAllocators[source] : nullptr; + IAllocator* allocator = GetAllocator(i); + IAllocatorAllocate* source = allocator->GetAllocationSource(); + IAllocatorAllocate* originalSource = allocator->GetOriginalAllocationSource(); + IAllocatorAllocate* schema = allocator->GetSchema(); + IAllocator* alias = (source != originalSource) ? sourcesToAllocators[source] : nullptr; if (schema && !alias) { // Check to see if this allocator's source maps to another allocator // Need to check both the schema and the allocator itself, as either one might be used as the alias depending on how it's implemented - AZStd::array checkAllocators = { { schema, allocator->GetAllocationSource() } }; + AZStd::array checkAllocators = { { schema, allocator->GetAllocationSource() } }; - for (AZ::IAllocatorAllocate* check : checkAllocators) + for (IAllocatorAllocate* check : checkAllocators) { auto existing = existingAllocators.emplace(check, allocator); @@ -631,7 +629,7 @@ void AllocatorManager::GetAllocatorStats(size_t& allocatedBytes, size_t& capacit } } - static const AZ::IAllocator* OS_ALLOCATOR = &AZ::AllocatorInstance::GetAllocator(); + static const IAllocator* OS_ALLOCATOR = &AllocatorInstance::GetAllocator(); size_t sourceAllocatedBytes = source->NumAllocatedBytes(); size_t sourceCapacityBytes = source->Capacity(); @@ -742,3 +740,5 @@ AllocatorManager::DebugBreak(void* address, const Debug::AllocationInfo& info) } } } + +} // namespace AZ diff --git a/Code/Framework/AzCore/AzCore/Memory/AllocatorOverrideShim.cpp b/Code/Framework/AzCore/AzCore/Memory/AllocatorOverrideShim.cpp index 154d59edd3..e4928e83c5 100644 --- a/Code/Framework/AzCore/AzCore/Memory/AllocatorOverrideShim.cpp +++ b/Code/Framework/AzCore/AzCore/Memory/AllocatorOverrideShim.cpp @@ -8,223 +8,220 @@ #include -namespace AZ +namespace AZ::Internal { - namespace Internal + AllocatorOverrideShim* AllocatorOverrideShim::Create(IAllocator* owningAllocator, IAllocatorAllocate* shimAllocationSource) { - AllocatorOverrideShim* AllocatorOverrideShim::Create(IAllocator* owningAllocator, IAllocatorAllocate* shimAllocationSource) + void* memory = shimAllocationSource->Allocate(sizeof(AllocatorOverrideShim), AZStd::alignment_of::value, 0); + auto result = new (memory) AllocatorOverrideShim(owningAllocator, shimAllocationSource); + return result; + } + + void AllocatorOverrideShim::Destroy(AllocatorOverrideShim* source) + { + auto shimAllocationSource = source->m_shimAllocationSource; + source->~AllocatorOverrideShim(); + shimAllocationSource->DeAllocate(source); + } + + AllocatorOverrideShim::AllocatorOverrideShim(IAllocator* owningAllocator, IAllocatorAllocate* shimAllocationSource) + : m_owningAllocator(owningAllocator) + , m_source(owningAllocator->GetOriginalAllocationSource()) + , m_overridingSource(owningAllocator->GetOriginalAllocationSource()) + , m_shimAllocationSource(shimAllocationSource) + , m_records(typename AllocationSet::hasher(), typename AllocationSet::key_eq(), StdAllocationSrc(shimAllocationSource)) + { + } + + void AllocatorOverrideShim::SetOverride(IAllocatorAllocate* source) + { + m_overridingSource = source; + } + + IAllocatorAllocate* AllocatorOverrideShim::GetOverride() const + { + return m_overridingSource; + } + + bool AllocatorOverrideShim::IsOverridden() const + { + return m_source != m_overridingSource; + } + + bool AllocatorOverrideShim::HasOrphanedAllocations() const + { + return !m_records.empty(); + } + + void AllocatorOverrideShim::SetFinalizedConfiguration() + { + m_finalizedConfiguration = true; + } + + typename AllocatorOverrideShim::pointer_type AllocatorOverrideShim::Allocate(size_type byteSize, size_type alignment, int flags, const char* name, const char* fileName, int lineNum, unsigned int suppressStackRecord) + { + pointer_type ptr = m_overridingSource->Allocate(byteSize, alignment, flags, name, fileName, lineNum, suppressStackRecord); + + if (!IsOverridden()) { - void* memory = shimAllocationSource->Allocate(sizeof(AllocatorOverrideShim), AZStd::alignment_of::value, 0); - auto result = new (memory) AllocatorOverrideShim(owningAllocator, shimAllocationSource); - return result; + lock_type lock(m_mutex); + m_records.insert(ptr); // Record in case we need to orphan this allocation later } - void AllocatorOverrideShim::Destroy(AllocatorOverrideShim* source) + return ptr; + } + + void AllocatorOverrideShim::DeAllocate(pointer_type ptr, size_type byteSize, size_type alignment) + { + IAllocatorAllocate* source = m_overridingSource; + bool destroy = false; + { - auto shimAllocationSource = source->m_shimAllocationSource; - source->~AllocatorOverrideShim(); - shimAllocationSource->DeAllocate(source); + lock_type lock(m_mutex); + + // Check to see if this came from a prior allocation source + if (m_records.erase(ptr) && IsOverridden()) + { + source = m_source; + + if (m_records.empty() && m_finalizedConfiguration) + { + // All orphaned records are gone; we are no longer needed + m_owningAllocator->SetAllocationSource(m_overridingSource); + destroy = true; // Must destroy outside the lock + } + } } - AllocatorOverrideShim::AllocatorOverrideShim(IAllocator* owningAllocator, IAllocatorAllocate* shimAllocationSource) - : m_owningAllocator(owningAllocator) - , m_source(owningAllocator->GetOriginalAllocationSource()) - , m_overridingSource(owningAllocator->GetOriginalAllocationSource()) - , m_shimAllocationSource(shimAllocationSource) - , m_records(typename AllocationSet::hasher(), typename AllocationSet::key_eq(), StdAllocationSrc(shimAllocationSource)) + source->DeAllocate(ptr, byteSize, alignment); + + if (destroy) { + Destroy(this); + } + } + + typename AllocatorOverrideShim::size_type AllocatorOverrideShim::Resize(pointer_type ptr, size_type newSize) + { + IAllocatorAllocate* source = m_overridingSource; + + if (IsOverridden()) + { + // Determine who owns the allocation + lock_type lock(m_mutex); + + if (m_records.count(ptr)) + { + source = m_source; + } } - void AllocatorOverrideShim::SetOverride(IAllocatorAllocate* source) + size_t result = source->Resize(ptr, newSize); + + return result; + } + + typename AllocatorOverrideShim::pointer_type AllocatorOverrideShim::ReAllocate(pointer_type ptr, size_type newSize, size_type newAlignment) + { + pointer_type newPtr = nullptr; + bool useOverride = true; + bool destroy = false; + + if (IsOverridden()) { - m_overridingSource = source; + lock_type lock(m_mutex); + + if (m_records.erase(ptr)) + { + // An old allocation needs to be transferred to the new, overriding allocator. + useOverride = false; // We'll do the reallocation here + size_t oldSize = m_source->AllocationSize(ptr); + + if (newSize) + { + newPtr = m_overridingSource->Allocate(newSize, newAlignment, 0); + memcpy(newPtr, ptr, AZStd::min(newSize, oldSize)); + } + + m_source->DeAllocate(ptr, oldSize); + + if (m_records.empty() && m_finalizedConfiguration) + { + // All orphaned records are gone; we are no longer needed + m_owningAllocator->SetAllocationSource(m_overridingSource); + destroy = true; // Must destroy outside the lock + } + } } - IAllocatorAllocate* AllocatorOverrideShim::GetOverride() const + if (useOverride) { - return m_overridingSource; - } - - bool AllocatorOverrideShim::IsOverridden() const - { - return m_source != m_overridingSource; - } - - bool AllocatorOverrideShim::HasOrphanedAllocations() const - { - return !m_records.empty(); - } - - void AllocatorOverrideShim::SetFinalizedConfiguration() - { - m_finalizedConfiguration = true; - } - - typename AllocatorOverrideShim::pointer_type AllocatorOverrideShim::Allocate(size_type byteSize, size_type alignment, int flags, const char* name, const char* fileName, int lineNum, unsigned int suppressStackRecord) - { - pointer_type ptr = m_overridingSource->Allocate(byteSize, alignment, flags, name, fileName, lineNum, suppressStackRecord); + // Default behavior, we weren't deleting an old allocation + newPtr = m_overridingSource->ReAllocate(ptr, newSize, newAlignment); if (!IsOverridden()) { + // Still need to do bookkeeping if we haven't been overridden yet lock_type lock(m_mutex); - m_records.insert(ptr); // Record in case we need to orphan this allocation later - } - - return ptr; - } - - void AllocatorOverrideShim::DeAllocate(pointer_type ptr, size_type byteSize, size_type alignment) - { - IAllocatorAllocate* source = m_overridingSource; - bool destroy = false; - - { - lock_type lock(m_mutex); - - // Check to see if this came from a prior allocation source - if (m_records.erase(ptr) && IsOverridden()) - { - source = m_source; - - if (m_records.empty() && m_finalizedConfiguration) - { - // All orphaned records are gone; we are no longer needed - m_owningAllocator->SetAllocationSource(m_overridingSource); - destroy = true; // Must destroy outside the lock - } - } - } - - source->DeAllocate(ptr, byteSize, alignment); - - if (destroy) - { - Destroy(this); + m_records.erase(ptr); + m_records.insert(newPtr); } } - typename AllocatorOverrideShim::size_type AllocatorOverrideShim::Resize(pointer_type ptr, size_type newSize) + if (destroy) { - IAllocatorAllocate* source = m_overridingSource; - - if (IsOverridden()) - { - // Determine who owns the allocation - lock_type lock(m_mutex); - - if (m_records.count(ptr)) - { - source = m_source; - } - } - - size_t result = source->Resize(ptr, newSize); - - return result; - } - - typename AllocatorOverrideShim::pointer_type AllocatorOverrideShim::ReAllocate(pointer_type ptr, size_type newSize, size_type newAlignment) - { - pointer_type newPtr = nullptr; - bool useOverride = true; - bool destroy = false; - - if (IsOverridden()) - { - lock_type lock(m_mutex); - - if (m_records.erase(ptr)) - { - // An old allocation needs to be transferred to the new, overriding allocator. - useOverride = false; // We'll do the reallocation here - size_t oldSize = m_source->AllocationSize(ptr); - - if (newSize) - { - newPtr = m_overridingSource->Allocate(newSize, newAlignment, 0); - memcpy(newPtr, ptr, AZStd::min(newSize, oldSize)); - } - - m_source->DeAllocate(ptr, oldSize); - - if (m_records.empty() && m_finalizedConfiguration) - { - // All orphaned records are gone; we are no longer needed - m_owningAllocator->SetAllocationSource(m_overridingSource); - destroy = true; // Must destroy outside the lock - } - } - } - - if (useOverride) - { - // Default behavior, we weren't deleting an old allocation - newPtr = m_overridingSource->ReAllocate(ptr, newSize, newAlignment); - - if (!IsOverridden()) - { - // Still need to do bookkeeping if we haven't been overridden yet - lock_type lock(m_mutex); - m_records.erase(ptr); - m_records.insert(newPtr); - } - } - - if (destroy) - { - Destroy(this); - } - - return newPtr; - } - - typename AllocatorOverrideShim::size_type AllocatorOverrideShim::AllocationSize(pointer_type ptr) - { - IAllocatorAllocate* source = m_overridingSource; - - if (IsOverridden()) - { - // Determine who owns the allocation - lock_type lock(m_mutex); - - if (m_records.count(ptr)) - { - source = m_source; - } - } - - return source->AllocationSize(ptr); - } - - void AllocatorOverrideShim::GarbageCollect() - { - m_source->GarbageCollect(); - } - - typename AllocatorOverrideShim::size_type AllocatorOverrideShim::NumAllocatedBytes() const - { - return m_source->NumAllocatedBytes(); - } - - typename AllocatorOverrideShim::size_type AllocatorOverrideShim::Capacity() const - { - return m_source->Capacity(); - } - - typename AllocatorOverrideShim::size_type AllocatorOverrideShim::GetMaxAllocationSize() const - { - return m_source->GetMaxAllocationSize(); - } - - auto AllocatorOverrideShim::GetMaxContiguousAllocationSize() const -> size_type - { - return m_source->GetMaxContiguousAllocationSize(); - } - - IAllocatorAllocate* AllocatorOverrideShim::GetSubAllocator() - { - return m_source->GetSubAllocator(); + Destroy(this); } + return newPtr; } -} + + typename AllocatorOverrideShim::size_type AllocatorOverrideShim::AllocationSize(pointer_type ptr) + { + IAllocatorAllocate* source = m_overridingSource; + + if (IsOverridden()) + { + // Determine who owns the allocation + lock_type lock(m_mutex); + + if (m_records.count(ptr)) + { + source = m_source; + } + } + + return source->AllocationSize(ptr); + } + + void AllocatorOverrideShim::GarbageCollect() + { + m_source->GarbageCollect(); + } + + typename AllocatorOverrideShim::size_type AllocatorOverrideShim::NumAllocatedBytes() const + { + return m_source->NumAllocatedBytes(); + } + + typename AllocatorOverrideShim::size_type AllocatorOverrideShim::Capacity() const + { + return m_source->Capacity(); + } + + typename AllocatorOverrideShim::size_type AllocatorOverrideShim::GetMaxAllocationSize() const + { + return m_source->GetMaxAllocationSize(); + } + + auto AllocatorOverrideShim::GetMaxContiguousAllocationSize() const -> size_type + { + return m_source->GetMaxContiguousAllocationSize(); + } + + IAllocatorAllocate* AllocatorOverrideShim::GetSubAllocator() + { + return m_source->GetSubAllocator(); + } + +} // namespace AZ::Internal diff --git a/Code/Framework/AzCore/AzCore/Memory/HphaSchema.cpp b/Code/Framework/AzCore/AzCore/Memory/HphaSchema.cpp index 6af8f201c2..6e40ccd8cd 100644 --- a/Code/Framework/AzCore/AzCore/Memory/HphaSchema.cpp +++ b/Code/Framework/AzCore/AzCore/Memory/HphaSchema.cpp @@ -140,8 +140,8 @@ namespace AZ { class const_iterator; class iterator { - typedef T& reference; - typedef T* pointer; + using reference = T&; + using pointer = T*; friend class const_iterator; T* mPtr; public: @@ -171,8 +171,8 @@ namespace AZ { class const_iterator { - typedef const T& reference; - typedef const T* pointer; + using reference = const T &; + using pointer = const T *; const T* mPtr; public: const_iterator() @@ -327,7 +327,7 @@ namespace AZ { uint64_t mSizeAndFlags; public: - typedef block_header* block_ptr; + using block_ptr = block_header *; size_t size() const { return mSizeAndFlags & ~BL_FLAG_MASK; } block_ptr next() const {return (block_ptr)((char*)mem() + size()); } block_ptr prev() const {return mPrev; } @@ -415,7 +415,7 @@ namespace AZ { void dec_ref() { HPPA_ASSERT(mUseCount > 0); mUseCount--; } bool check_marker(size_t marker) const { return mMarker == (marker ^ ((size_t)this)); } }; - typedef intrusive_list page_list; + using page_list = intrusive_list; class bucket { page_list mPageList; diff --git a/Code/Framework/AzCore/AzCore/Memory/MallocSchema.cpp b/Code/Framework/AzCore/AzCore/Memory/MallocSchema.cpp index 76a71e0f08..9aa31cd8b6 100644 --- a/Code/Framework/AzCore/AzCore/Memory/MallocSchema.cpp +++ b/Code/Framework/AzCore/AzCore/Memory/MallocSchema.cpp @@ -11,149 +11,168 @@ #include #include +namespace AZ::Internal +{ + struct Header + { + uint32_t offset; + uint32_t size; + }; +} // namespace AZ::Internal + namespace AZ { - namespace Internal + //--------------------------------------------------------------------- + // MallocSchema methods + //--------------------------------------------------------------------- + + MallocSchema::MallocSchema(const Descriptor& desc) + : m_bytesAllocated(0) { - struct Header + if (desc.m_useAZMalloc) { - uint32_t offset; - uint32_t size; - }; + static const int DEFAULT_ALIGNMENT = sizeof(void*) * 2; // Default malloc alignment + + m_mallocFn = [](size_t byteSize) + { + return AZ_OS_MALLOC(byteSize, DEFAULT_ALIGNMENT); + }; + m_freeFn = [](void* ptr) + { + AZ_OS_FREE(ptr); + }; + } + else + { + m_mallocFn = &malloc; + m_freeFn = &free; + } } -} -//--------------------------------------------------------------------- -// MallocSchema methods -//--------------------------------------------------------------------- - -AZ::MallocSchema::MallocSchema(const Descriptor& desc) : - m_bytesAllocated(0) -{ - if (desc.m_useAZMalloc) + MallocSchema::~MallocSchema() { - static const int DEFAULT_ALIGNMENT = sizeof(void*) * 2; // Default malloc alignment - - m_mallocFn = [](size_t byteSize) { return AZ_OS_MALLOC(byteSize, DEFAULT_ALIGNMENT); }; - m_freeFn = [](void* ptr) { AZ_OS_FREE(ptr); }; } - else + + MallocSchema::pointer_type MallocSchema::Allocate( + size_type byteSize, + size_type alignment, + int flags, + const char* name, + const char* fileName, + int lineNum, + unsigned int suppressStackRecord) { - m_mallocFn = &malloc; - m_freeFn = &free; + (void)flags; + (void)name; + (void)fileName; + (void)lineNum; + (void)suppressStackRecord; + + if (!byteSize) + { + return nullptr; + } + + if (alignment == 0) + { + alignment = sizeof(void*) * 2; // Default malloc alignment + } + + AZ_Assert(byteSize < 0x100000000ull, "Malloc allocator only allocates up to 4GB"); + + size_type required = byteSize + sizeof(Internal::Header) + + ((alignment > sizeof(double)) + ? alignment + : 0); // Malloc will align to a minimum boundary for native objects, so we only pad if aligning to a large value + void* data = (*m_mallocFn)(required); + void* result = PointerAlignUp(reinterpret_cast(reinterpret_cast(data) + sizeof(Internal::Header)), alignment); + Internal::Header* header = PointerAlignDown( + (Internal::Header*)(reinterpret_cast(result) - sizeof(Internal::Header)), AZStd::alignment_of::value); + + header->offset = static_cast(reinterpret_cast(result) - reinterpret_cast(data)); + header->size = static_cast(byteSize); + m_bytesAllocated += byteSize; + + return result; } -} -AZ::MallocSchema::~MallocSchema() -{ -} + void MallocSchema::DeAllocate(pointer_type ptr, size_type byteSize, size_type alignment) + { + (void)byteSize; + (void)alignment; -AZ::MallocSchema::pointer_type AZ::MallocSchema::Allocate(size_type byteSize, size_type alignment, int flags, const char* name, const char* fileName, int lineNum, unsigned int suppressStackRecord) -{ - (void)flags; - (void)name; - (void)fileName; - (void)lineNum; - (void)suppressStackRecord; + if (!ptr) + { + return; + } - if (!byteSize) + Internal::Header* header = PointerAlignDown( + reinterpret_cast(reinterpret_cast(ptr) - sizeof(Internal::Header)), + AZStd::alignment_of::value); + void* freePtr = reinterpret_cast(reinterpret_cast(ptr) - static_cast(header->offset)); + + m_bytesAllocated -= header->size; + (*m_freeFn)(freePtr); + } + + MallocSchema::pointer_type MallocSchema::ReAllocate(pointer_type ptr, size_type newSize, size_type newAlignment) + { + void* newPtr = Allocate(newSize, newAlignment, 0); + size_t oldSize = AllocationSize(ptr); + + memcpy(newPtr, ptr, AZStd::min(oldSize, newSize)); + DeAllocate(ptr, 0, 0); + + return newPtr; + } + + MallocSchema::size_type MallocSchema::Resize(pointer_type ptr, size_type newSize) + { + (void)ptr; + (void)newSize; + + return 0; + } + + MallocSchema::size_type MallocSchema::AllocationSize(pointer_type ptr) + { + if (!ptr) + { + return 0; + } + Internal::Header* header = PointerAlignDown( + reinterpret_cast(reinterpret_cast(ptr) - sizeof(Internal::Header)), + AZStd::alignment_of::value); + return header->size; + } + + MallocSchema::size_type MallocSchema::NumAllocatedBytes() const + { + return m_bytesAllocated; + } + + MallocSchema::size_type MallocSchema::Capacity() const + { + return 0; + } + + MallocSchema::size_type MallocSchema::GetMaxAllocationSize() const + { + return 0xFFFFFFFFull; + } + + MallocSchema::size_type MallocSchema::GetMaxContiguousAllocationSize() const + { + return AZ_CORE_MAX_ALLOCATOR_SIZE; + } + + IAllocatorAllocate* MallocSchema::GetSubAllocator() { return nullptr; } - if (alignment == 0) + void MallocSchema::GarbageCollect() { - alignment = sizeof(void*) * 2; // Default malloc alignment } - AZ_Assert(byteSize < 0x100000000ull, "Malloc allocator only allocates up to 4GB"); - - size_type required = byteSize + sizeof(Internal::Header) + ((alignment > sizeof(double)) ? alignment : 0); // Malloc will align to a minimum boundary for native objects, so we only pad if aligning to a large value - void* data = (*m_mallocFn)(required); - void* result = PointerAlignUp(reinterpret_cast(reinterpret_cast(data) + sizeof(Internal::Header)), alignment); - Internal::Header* header = PointerAlignDown((Internal::Header*)(reinterpret_cast(result) - sizeof(Internal::Header)), AZStd::alignment_of::value); - - header->offset = static_cast(reinterpret_cast(result) - reinterpret_cast(data)); - header->size = static_cast(byteSize); - m_bytesAllocated += byteSize; - - return result; -} - -void AZ::MallocSchema::DeAllocate(pointer_type ptr, size_type byteSize, size_type alignment) -{ - (void)byteSize; - (void)alignment; - - if (!ptr) - { - return; - } - - Internal::Header* header = PointerAlignDown(reinterpret_cast(reinterpret_cast(ptr) - sizeof(Internal::Header)), AZStd::alignment_of::value); - void* freePtr = reinterpret_cast(reinterpret_cast(ptr) - static_cast(header->offset)); - - m_bytesAllocated -= header->size; - (*m_freeFn)(freePtr); -} - -AZ::MallocSchema::pointer_type AZ::MallocSchema::ReAllocate(pointer_type ptr, size_type newSize, size_type newAlignment) -{ - void* newPtr = Allocate(newSize, newAlignment, 0); - size_t oldSize = AllocationSize(ptr); - - memcpy(newPtr, ptr, AZStd::min(oldSize, newSize)); - DeAllocate(ptr, 0, 0); - - return newPtr; -} - -AZ::MallocSchema::size_type AZ::MallocSchema::Resize(pointer_type ptr, size_type newSize) -{ - (void)ptr; - (void)newSize; - - return 0; -} - -AZ::MallocSchema::size_type AZ::MallocSchema::AllocationSize(pointer_type ptr) -{ - size_type result = 0; - - if (ptr) - { - Internal::Header* header = PointerAlignDown(reinterpret_cast(reinterpret_cast(ptr) - sizeof(Internal::Header)), AZStd::alignment_of::value); - result = header->size; - } - - return result; -} - -AZ::MallocSchema::size_type AZ::MallocSchema::NumAllocatedBytes() const -{ - return m_bytesAllocated; -} - -AZ::MallocSchema::size_type AZ::MallocSchema::Capacity() const -{ - return 0; -} - -AZ::MallocSchema::size_type AZ::MallocSchema::GetMaxAllocationSize() const -{ - return 0xFFFFFFFFull; -} - -AZ::MallocSchema::size_type AZ::MallocSchema::GetMaxContiguousAllocationSize() const -{ - return AZ_CORE_MAX_ALLOCATOR_SIZE; -} - -AZ::IAllocatorAllocate* AZ::MallocSchema::GetSubAllocator() -{ - return nullptr; -} - -void AZ::MallocSchema::GarbageCollect() -{ -} +} // namespace AZ diff --git a/Code/Framework/AzCore/AzCore/Memory/MemoryDriller.cpp b/Code/Framework/AzCore/AzCore/Memory/MemoryDriller.cpp index 2ea25c3397..dc35c6322b 100644 --- a/Code/Framework/AzCore/AzCore/Memory/MemoryDriller.cpp +++ b/Code/Framework/AzCore/AzCore/Memory/MemoryDriller.cpp @@ -15,280 +15,277 @@ #include #include -namespace AZ +namespace AZ::Debug { - namespace Debug + //========================================================================= + // MemoryDriller + // [2/6/2013] + //========================================================================= + MemoryDriller::MemoryDriller(const Descriptor& desc) { - //========================================================================= - // MemoryDriller - // [2/6/2013] - //========================================================================= - MemoryDriller::MemoryDriller(const Descriptor& desc) + (void)desc; + BusConnect(); + + AllocatorManager::Instance().EnterProfilingMode(); + { - (void)desc; - BusConnect(); - - AllocatorManager::Instance().EnterProfilingMode(); - - { - // Register all allocators that were created before the driller existed - auto allocatorLock = AllocatorManager::Instance().LockAllocators(); - - for (int i = 0; i < AllocatorManager::Instance().GetNumAllocators(); ++i) - { - IAllocator* allocator = AllocatorManager::Instance().GetAllocator(i); - RegisterAllocator(allocator); - } - } - } - - //========================================================================= - // ~MemoryDriller - // [2/6/2013] - //========================================================================= - MemoryDriller::~MemoryDriller() - { - BusDisconnect(); - AllocatorManager::Instance().ExitProfilingMode(); - } - - //========================================================================= - // Start - // [2/6/2013] - //========================================================================= - void MemoryDriller::Start(const Param* params, int numParams) - { - (void)params; - (void)numParams; - - // dump current allocations for all allocators with tracking + // Register all allocators that were created before the driller existed auto allocatorLock = AllocatorManager::Instance().LockAllocators(); + for (int i = 0; i < AllocatorManager::Instance().GetNumAllocators(); ++i) { IAllocator* allocator = AllocatorManager::Instance().GetAllocator(i); - if (auto records = allocator->GetRecords()) - { - RegisterAllocatorOutput(allocator); - const AllocationRecordsType& allocMap = records->GetMap(); - for (AllocationRecordsType::const_iterator allocIt = allocMap.begin(); allocIt != allocMap.end(); ++allocIt) - { - RegisterAllocationOutput(allocator, allocIt->first, &allocIt->second); - } - } + RegisterAllocator(allocator); } } + } - //========================================================================= - // Stop - // [2/6/2013] - //========================================================================= - void MemoryDriller::Stop() + //========================================================================= + // ~MemoryDriller + // [2/6/2013] + //========================================================================= + MemoryDriller::~MemoryDriller() + { + BusDisconnect(); + AllocatorManager::Instance().ExitProfilingMode(); + } + + //========================================================================= + // Start + // [2/6/2013] + //========================================================================= + void MemoryDriller::Start(const Param* params, int numParams) + { + (void)params; + (void)numParams; + + // dump current allocations for all allocators with tracking + auto allocatorLock = AllocatorManager::Instance().LockAllocators(); + for (int i = 0; i < AllocatorManager::Instance().GetNumAllocators(); ++i) { - } - - //========================================================================= - // RegisterAllocator - // [2/6/2013] - //========================================================================= - void MemoryDriller::RegisterAllocator(IAllocator* allocator) - { - // Ignore if our allocator is already registered - if (allocator->GetRecords() != nullptr) + IAllocator* allocator = AllocatorManager::Instance().GetAllocator(i); + if (auto records = allocator->GetRecords()) { - return; - } - - auto debugConfig = allocator->GetDebugConfig(); - - if (!debugConfig.m_excludeFromDebugging) - { - allocator->SetRecords(aznew Debug::AllocationRecords((unsigned char)debugConfig.m_stackRecordLevels, debugConfig.m_usesMemoryGuards, debugConfig.m_marksUnallocatedMemory, allocator->GetName())); - - m_allAllocatorRecords.push_back(allocator->GetRecords()); - - if (m_output == nullptr) - { - return; // we have no active output - } RegisterAllocatorOutput(allocator); - } - } - //========================================================================= - // RegisterAllocatorOutput - // [2/6/2013] - //========================================================================= - void MemoryDriller::RegisterAllocatorOutput(IAllocator* allocator) - { - auto records = allocator->GetRecords(); - m_output->BeginTag(AZ_CRC("MemoryDriller", 0x1b31269d)); - m_output->BeginTag(AZ_CRC("RegisterAllocator", 0x19f08114)); - m_output->Write(AZ_CRC("Name", 0x5e237e06), allocator->GetName()); - m_output->Write(AZ_CRC("Id", 0xbf396750), allocator); - m_output->Write(AZ_CRC("Capacity", 0xb5e8b174), allocator->GetAllocationSource()->Capacity()); - m_output->Write(AZ_CRC("RecordsId", 0x7caaca88), records); - if (records) - { - m_output->Write(AZ_CRC("RecordsMode", 0x764c147a), (char)records->GetMode()); - m_output->Write(AZ_CRC("NumStackLevels", 0xad9cff15), records->GetNumStackLevels()); - } - m_output->EndTag(AZ_CRC("RegisterAllocator", 0x19f08114)); - m_output->EndTag(AZ_CRC("MemoryDriller", 0x1b31269d)); - } - - //========================================================================= - // UnregisterAllocator - // [2/6/2013] - //========================================================================= - void MemoryDriller::UnregisterAllocator(IAllocator* allocator) - { - auto allocatorRecords = allocator->GetRecords(); - AZ_Assert(allocatorRecords, "This allocator is not registered with the memory driller!"); - for (auto records : m_allAllocatorRecords) - { - if (records == allocatorRecords) + const AllocationRecordsType& allocMap = records->GetMap(); + for (AllocationRecordsType::const_iterator allocIt = allocMap.begin(); allocIt != allocMap.end(); ++allocIt) { - m_allAllocatorRecords.remove(records); - break; + RegisterAllocationOutput(allocator, allocIt->first, &allocIt->second); } } - delete allocatorRecords; - allocator->SetRecords(nullptr); + } + } + + //========================================================================= + // Stop + // [2/6/2013] + //========================================================================= + void MemoryDriller::Stop() + { + } + + //========================================================================= + // RegisterAllocator + // [2/6/2013] + //========================================================================= + void MemoryDriller::RegisterAllocator(IAllocator* allocator) + { + // Ignore if our allocator is already registered + if (allocator->GetRecords() != nullptr) + { + return; + } + + auto debugConfig = allocator->GetDebugConfig(); + + if (!debugConfig.m_excludeFromDebugging) + { + allocator->SetRecords(aznew Debug::AllocationRecords((unsigned char)debugConfig.m_stackRecordLevels, debugConfig.m_usesMemoryGuards, debugConfig.m_marksUnallocatedMemory, allocator->GetName())); + + m_allAllocatorRecords.push_back(allocator->GetRecords()); + + if (m_output == nullptr) + { + return; // we have no active output + } + RegisterAllocatorOutput(allocator); + } + } + //========================================================================= + // RegisterAllocatorOutput + // [2/6/2013] + //========================================================================= + void MemoryDriller::RegisterAllocatorOutput(IAllocator* allocator) + { + auto records = allocator->GetRecords(); + m_output->BeginTag(AZ_CRC("MemoryDriller", 0x1b31269d)); + m_output->BeginTag(AZ_CRC("RegisterAllocator", 0x19f08114)); + m_output->Write(AZ_CRC("Name", 0x5e237e06), allocator->GetName()); + m_output->Write(AZ_CRC("Id", 0xbf396750), allocator); + m_output->Write(AZ_CRC("Capacity", 0xb5e8b174), allocator->GetAllocationSource()->Capacity()); + m_output->Write(AZ_CRC("RecordsId", 0x7caaca88), records); + if (records) + { + m_output->Write(AZ_CRC("RecordsMode", 0x764c147a), (char)records->GetMode()); + m_output->Write(AZ_CRC("NumStackLevels", 0xad9cff15), records->GetNumStackLevels()); + } + m_output->EndTag(AZ_CRC("RegisterAllocator", 0x19f08114)); + m_output->EndTag(AZ_CRC("MemoryDriller", 0x1b31269d)); + } + + //========================================================================= + // UnregisterAllocator + // [2/6/2013] + //========================================================================= + void MemoryDriller::UnregisterAllocator(IAllocator* allocator) + { + auto allocatorRecords = allocator->GetRecords(); + AZ_Assert(allocatorRecords, "This allocator is not registered with the memory driller!"); + for (auto records : m_allAllocatorRecords) + { + if (records == allocatorRecords) + { + m_allAllocatorRecords.remove(records); + break; + } + } + delete allocatorRecords; + allocator->SetRecords(nullptr); + + if (m_output == nullptr) + { + return; // we have no active output + } + m_output->BeginTag(AZ_CRC("MemoryDriller", 0x1b31269d)); + m_output->Write(AZ_CRC("UnregisterAllocator", 0xb2b54f93), allocator); + m_output->EndTag(AZ_CRC("MemoryDriller", 0x1b31269d)); + } + + //========================================================================= + // RegisterAllocation + // [2/6/2013] + //========================================================================= + void MemoryDriller::RegisterAllocation(IAllocator* allocator, void* address, size_t byteSize, size_t alignment, const char* name, const char* fileName, int lineNum, unsigned int stackSuppressCount) + { + auto records = allocator->GetRecords(); + if (records) + { + const AllocationInfo* info = records->RegisterAllocation(address, byteSize, alignment, name, fileName, lineNum, stackSuppressCount + 1); + if (m_output == nullptr) + { + return; // we have no active output + } + RegisterAllocationOutput(allocator, address, info); + } + } + + //========================================================================= + // RegisterAllocationOutput + // [2/6/2013] + //========================================================================= + void MemoryDriller::RegisterAllocationOutput(IAllocator* allocator, void* address, const AllocationInfo* info) + { + auto records = allocator->GetRecords(); + m_output->BeginTag(AZ_CRC("MemoryDriller", 0x1b31269d)); + m_output->BeginTag(AZ_CRC("RegisterAllocation", 0x992a9780)); + m_output->Write(AZ_CRC("RecordsId", 0x7caaca88), records); + m_output->Write(AZ_CRC("Address", 0x0d4e6f81), address); + if (info) + { + if (info->m_name) + { + m_output->Write(AZ_CRC("Name", 0x5e237e06), info->m_name); + } + m_output->Write(AZ_CRC("Alignment", 0x2cce1e5c), info->m_alignment); + m_output->Write(AZ_CRC("Size", 0xf7c0246a), info->m_byteSize); + if (info->m_fileName) + { + m_output->Write(AZ_CRC("FileName", 0x3c0be965), info->m_fileName); + m_output->Write(AZ_CRC("FileLine", 0xb33c2395), info->m_lineNum); + } + // copy the stack frames directly, resolving the stack should happen later as this is a SLOW procedure. + if (info->m_stackFrames) + { + m_output->Write(AZ_CRC("Stack", 0x41a87b6a), info->m_stackFrames, info->m_stackFrames + records->GetNumStackLevels()); + } + } + m_output->EndTag(AZ_CRC("RegisterAllocation", 0x992a9780)); + m_output->EndTag(AZ_CRC("MemoryDriller", 0x1b31269d)); + } + + //========================================================================= + // UnRegisterAllocation + // [2/6/2013] + //========================================================================= + void MemoryDriller::UnregisterAllocation(IAllocator* allocator, void* address, size_t byteSize, size_t alignment, AllocationInfo* info) + { + auto records = allocator->GetRecords(); + if (records) + { + records->UnregisterAllocation(address, byteSize, alignment, info); if (m_output == nullptr) { return; // we have no active output } m_output->BeginTag(AZ_CRC("MemoryDriller", 0x1b31269d)); - m_output->Write(AZ_CRC("UnregisterAllocator", 0xb2b54f93), allocator); - m_output->EndTag(AZ_CRC("MemoryDriller", 0x1b31269d)); - } - - //========================================================================= - // RegisterAllocation - // [2/6/2013] - //========================================================================= - void MemoryDriller::RegisterAllocation(IAllocator* allocator, void* address, size_t byteSize, size_t alignment, const char* name, const char* fileName, int lineNum, unsigned int stackSuppressCount) - { - auto records = allocator->GetRecords(); - if (records) - { - const AllocationInfo* info = records->RegisterAllocation(address, byteSize, alignment, name, fileName, lineNum, stackSuppressCount + 1); - if (m_output == nullptr) - { - return; // we have no active output - } - RegisterAllocationOutput(allocator, address, info); - } - } - - //========================================================================= - // RegisterAllocationOutput - // [2/6/2013] - //========================================================================= - void MemoryDriller::RegisterAllocationOutput(IAllocator* allocator, void* address, const AllocationInfo* info) - { - auto records = allocator->GetRecords(); - m_output->BeginTag(AZ_CRC("MemoryDriller", 0x1b31269d)); - m_output->BeginTag(AZ_CRC("RegisterAllocation", 0x992a9780)); + m_output->BeginTag(AZ_CRC("UnRegisterAllocation", 0xea5dc4cd)); m_output->Write(AZ_CRC("RecordsId", 0x7caaca88), records); m_output->Write(AZ_CRC("Address", 0x0d4e6f81), address); - if (info) - { - if (info->m_name) - { - m_output->Write(AZ_CRC("Name", 0x5e237e06), info->m_name); - } - m_output->Write(AZ_CRC("Alignment", 0x2cce1e5c), info->m_alignment); - m_output->Write(AZ_CRC("Size", 0xf7c0246a), info->m_byteSize); - if (info->m_fileName) - { - m_output->Write(AZ_CRC("FileName", 0x3c0be965), info->m_fileName); - m_output->Write(AZ_CRC("FileLine", 0xb33c2395), info->m_lineNum); - } - // copy the stack frames directly, resolving the stack should happen later as this is a SLOW procedure. - if (info->m_stackFrames) - { - m_output->Write(AZ_CRC("Stack", 0x41a87b6a), info->m_stackFrames, info->m_stackFrames + records->GetNumStackLevels()); - } - } - m_output->EndTag(AZ_CRC("RegisterAllocation", 0x992a9780)); + m_output->EndTag(AZ_CRC("UnRegisterAllocation", 0xea5dc4cd)); m_output->EndTag(AZ_CRC("MemoryDriller", 0x1b31269d)); } + } - //========================================================================= - // UnRegisterAllocation - // [2/6/2013] - //========================================================================= - void MemoryDriller::UnregisterAllocation(IAllocator* allocator, void* address, size_t byteSize, size_t alignment, AllocationInfo* info) + //========================================================================= + // ReallocateAllocation + // [10/1/2018] + //========================================================================= + void MemoryDriller::ReallocateAllocation(IAllocator* allocator, void* prevAddress, void* newAddress, size_t newByteSize, size_t newAlignment) + { + AllocationInfo info; + UnregisterAllocation(allocator, prevAddress, 0, 0, &info); + RegisterAllocation(allocator, newAddress, newByteSize, newAlignment, info.m_name, info.m_fileName, info.m_lineNum, 0); + } + + //========================================================================= + // ResizeAllocation + // [2/6/2013] + //========================================================================= + void MemoryDriller::ResizeAllocation(IAllocator* allocator, void* address, size_t newSize) + { + auto records = allocator->GetRecords(); + if (records) { - auto records = allocator->GetRecords(); - if (records) - { - records->UnregisterAllocation(address, byteSize, alignment, info); + records->ResizeAllocation(address, newSize); - if (m_output == nullptr) - { - return; // we have no active output - } - m_output->BeginTag(AZ_CRC("MemoryDriller", 0x1b31269d)); - m_output->BeginTag(AZ_CRC("UnRegisterAllocation", 0xea5dc4cd)); - m_output->Write(AZ_CRC("RecordsId", 0x7caaca88), records); - m_output->Write(AZ_CRC("Address", 0x0d4e6f81), address); - m_output->EndTag(AZ_CRC("UnRegisterAllocation", 0xea5dc4cd)); - m_output->EndTag(AZ_CRC("MemoryDriller", 0x1b31269d)); + if (m_output == nullptr) + { + return; // we have no active output + } + m_output->BeginTag(AZ_CRC("MemoryDriller", 0x1b31269d)); + m_output->BeginTag(AZ_CRC("ResizeAllocation", 0x8a9c78dc)); + m_output->Write(AZ_CRC("RecordsId", 0x7caaca88), records); + m_output->Write(AZ_CRC("Address", 0x0d4e6f81), address); + m_output->Write(AZ_CRC("Size", 0xf7c0246a), newSize); + m_output->EndTag(AZ_CRC("ResizeAllocation", 0x8a9c78dc)); + m_output->EndTag(AZ_CRC("MemoryDriller", 0x1b31269d)); + } + } + + void MemoryDriller::DumpAllAllocations() + { + // Create a copy so allocations done during the printing dont end up affecting the container + const AZStd::list allocationRecords = m_allAllocatorRecords; + + for (auto records : allocationRecords) + { + // Skip if we have had no allocations made + if (records->RequestedAllocs()) + { + records->EnumerateAllocations(AZ::Debug::PrintAllocationsCB(true, true)); } } + } - //========================================================================= - // ReallocateAllocation - // [10/1/2018] - //========================================================================= - void MemoryDriller::ReallocateAllocation(IAllocator* allocator, void* prevAddress, void* newAddress, size_t newByteSize, size_t newAlignment) - { - AllocationInfo info; - UnregisterAllocation(allocator, prevAddress, 0, 0, &info); - RegisterAllocation(allocator, newAddress, newByteSize, newAlignment, info.m_name, info.m_fileName, info.m_lineNum, 0); - } - - //========================================================================= - // ResizeAllocation - // [2/6/2013] - //========================================================================= - void MemoryDriller::ResizeAllocation(IAllocator* allocator, void* address, size_t newSize) - { - auto records = allocator->GetRecords(); - if (records) - { - records->ResizeAllocation(address, newSize); - - if (m_output == nullptr) - { - return; // we have no active output - } - m_output->BeginTag(AZ_CRC("MemoryDriller", 0x1b31269d)); - m_output->BeginTag(AZ_CRC("ResizeAllocation", 0x8a9c78dc)); - m_output->Write(AZ_CRC("RecordsId", 0x7caaca88), records); - m_output->Write(AZ_CRC("Address", 0x0d4e6f81), address); - m_output->Write(AZ_CRC("Size", 0xf7c0246a), newSize); - m_output->EndTag(AZ_CRC("ResizeAllocation", 0x8a9c78dc)); - m_output->EndTag(AZ_CRC("MemoryDriller", 0x1b31269d)); - } - } - - void MemoryDriller::DumpAllAllocations() - { - // Create a copy so allocations done during the printing dont end up affecting the container - const AZStd::list allocationRecords = m_allAllocatorRecords; - - for (auto records : allocationRecords) - { - // Skip if we have had no allocations made - if (records->RequestedAllocs()) - { - records->EnumerateAllocations(AZ::Debug::PrintAllocationsCB(true, true)); - } - } - } - - }// namespace Debug -} // namespace AZ +} // namespace AZ::Debug diff --git a/Code/Framework/AzCore/AzCore/Memory/PoolSchema.cpp b/Code/Framework/AzCore/AzCore/Memory/PoolSchema.cpp index 0bef6b7d28..9167864450 100644 --- a/Code/Framework/AzCore/AzCore/Memory/PoolSchema.cpp +++ b/Code/Framework/AzCore/AzCore/Memory/PoolSchema.cpp @@ -35,8 +35,8 @@ namespace AZ public: AZ_CLASS_ALLOCATOR(PoolAllocation, SystemAllocator, 0) - typedef typename Allocator::Page PageType; - typedef typename Allocator::Bucket BucketType; + using PageType = typename Allocator::Page; + using BucketType = typename Allocator::Bucket; PoolAllocation(Allocator* alloc, size_t pageSize, size_t minAllocationSize, size_t maxAllocationSize); virtual ~PoolAllocation(); @@ -89,7 +89,7 @@ namespace AZ void SetupFreeList(size_t elementSize, size_t pageDataBlockSize); /// We just use a free list of nodes which we cast to the pool type. - typedef AZStd::intrusive_slist > FreeListType; + using FreeListType = AZStd::intrusive_slist>; FreeListType m_freeList; u32 m_bin; @@ -103,7 +103,7 @@ namespace AZ */ struct Bucket { - typedef AZStd::intrusive_list > PageListType; + using PageListType = AZStd::intrusive_list>; PageListType m_pages; }; @@ -162,7 +162,7 @@ namespace AZ return page; } - typedef PoolAllocation AllocatorType; + using AllocatorType = PoolAllocation; IAllocatorAllocate* m_pageAllocator; AllocatorType m_allocator; void* m_staticDataBlock; @@ -199,7 +199,7 @@ namespace AZ void SetupFreeList(size_t elementSize, size_t pageDataBlockSize); /// We just use a free list of nodes which we cast to the pool type. - typedef AZStd::intrusive_slist > FreeListType; + using FreeListType = AZStd::intrusive_slist>; FreeListType m_freeList; AZStd::lock_free_intrusive_stack_node m_lfStack; ///< Lock Free stack node @@ -215,7 +215,7 @@ namespace AZ */ struct Bucket { - typedef AZStd::intrusive_list > PageListType; + using PageListType = AZStd::intrusive_list>; PageListType m_pages; }; @@ -291,7 +291,7 @@ namespace AZ ThreadPoolSchema::SetThreadPoolData m_threadPoolSetter; // Fox X64 we push/pop pages using the m_mutex to sync. Pages are - typedef Bucket::PageListType FreePagesType; + using FreePagesType = Bucket::PageListType; FreePagesType m_freePages; AZStd::vector m_threads; ///< Array with all separate thread data. Used to traverse end free elements. @@ -313,12 +313,12 @@ namespace AZ ThreadPoolData(ThreadPoolSchemaImpl* alloc, size_t pageSize, size_t minAllocationSize, size_t maxAllocationSize); ~ThreadPoolData(); - typedef PoolAllocation AllocatorType; + using AllocatorType = PoolAllocation; /** * Stack with freed elements from other threads. We don't need stamped stack since the ABA problem can not * happen here. We push from many threads and pop from only one (we don't push from it). */ - typedef AZStd::lock_free_intrusive_stack > FreedElementsStack; + using FreedElementsStack = AZStd::lock_free_intrusive_stack>; AllocatorType m_allocator; FreedElementsStack m_freedElements; diff --git a/Code/Framework/AzCore/AzCore/Module/Environment.cpp b/Code/Framework/AzCore/AzCore/Module/Environment.cpp index 71da948a35..790af730bf 100644 --- a/Code/Framework/AzCore/AzCore/Module/Environment.cpp +++ b/Code/Framework/AzCore/AzCore/Module/Environment.cpp @@ -24,10 +24,10 @@ namespace AZ class OSStdAllocator { public: - typedef void* pointer_type; - typedef AZStd::size_t size_type; - typedef AZStd::ptrdiff_t difference_type; - typedef AZStd::false_type allow_memory_leaks; ///< Regular allocators should not leak. + using pointer_type = void *; + using size_type = AZStd::size_t; + using difference_type = AZStd::ptrdiff_t; + using allow_memory_leaks = AZStd::false_type; ///< Regular allocators should not leak. OSStdAllocator(Environment::AllocatorInterface* allocator) : m_name("GlobalEnvironmentAllocator") @@ -122,7 +122,7 @@ namespace AZ : public EnvironmentInterface { public: - typedef AZStd::unordered_map, AZStd::equal_to, OSStdAllocator> MapType; + using MapType = AZStd::unordered_map, AZStd::equal_to, OSStdAllocator>; static EnvironmentInterface* Get(); static void Attach(EnvironmentInstance sourceEnvironment, bool useAsGetFallback); diff --git a/Code/Framework/AzCore/AzCore/Module/Internal/ModuleManagerSearchPathTool.cpp b/Code/Framework/AzCore/AzCore/Module/Internal/ModuleManagerSearchPathTool.cpp index 10aa9904a3..0797fef93f 100644 --- a/Code/Framework/AzCore/AzCore/Module/Internal/ModuleManagerSearchPathTool.cpp +++ b/Code/Framework/AzCore/AzCore/Module/Internal/ModuleManagerSearchPathTool.cpp @@ -10,21 +10,18 @@ #include #include -namespace AZ +namespace AZ::Internal { - namespace Internal + AZ::OSString ModuleManagerSearchPathTool::GetModuleDirectory(const AZ::DynamicModuleDescriptor& moduleDesc) { - AZ::OSString ModuleManagerSearchPathTool::GetModuleDirectory(const AZ::DynamicModuleDescriptor& moduleDesc) + // For each module that is loaded, attempt to set the module's folder as a path for dependent module resolution + AZ::OSString modulePath = moduleDesc.m_dynamicLibraryPath; + AZ::ComponentApplicationBus::Broadcast(&AZ::ComponentApplicationBus::Events::ResolveModulePath, modulePath); + auto lastPathSep = modulePath.find_last_of(AZ_TRAIT_OS_PATH_SEPARATOR); + if (lastPathSep != modulePath.npos) { - // For each module that is loaded, attempt to set the module's folder as a path for dependent module resolution - AZ::OSString modulePath = moduleDesc.m_dynamicLibraryPath; - AZ::ComponentApplicationBus::Broadcast(&AZ::ComponentApplicationBus::Events::ResolveModulePath, modulePath); - auto lastPathSep = modulePath.find_last_of(AZ_TRAIT_OS_PATH_SEPARATOR); - if (lastPathSep != modulePath.npos) - { - modulePath = modulePath.substr(0, lastPathSep); - } - return modulePath; + modulePath = modulePath.substr(0, lastPathSep); } - } // namespace Internal -} // namespace AZ + return modulePath; + } +} // namespace AZ::Internal diff --git a/Code/Framework/AzCore/AzCore/Name/Internal/NameData.cpp b/Code/Framework/AzCore/AzCore/Name/Internal/NameData.cpp index 574b0bcc7e..69bead11ac 100644 --- a/Code/Framework/AzCore/AzCore/Name/Internal/NameData.cpp +++ b/Code/Framework/AzCore/AzCore/Name/Internal/NameData.cpp @@ -9,45 +9,41 @@ #include #include -namespace AZ +namespace AZ::Internal { - namespace Internal + NameData::NameData(AZStd::string&& name, Hash hash) + : m_name{AZStd::move(name)} + , m_hash{hash} + {} + + AZStd::string_view NameData::GetName() const { - NameData::NameData(AZStd::string&& name, Hash hash) - : m_name{AZStd::move(name)} - , m_hash{hash} - {} + return m_name; + } - AZStd::string_view NameData::GetName() const - { - return m_name; - } + NameData::Hash NameData::GetHash() const + { + return m_hash; + } - NameData::Hash NameData::GetHash() const - { - return m_hash; - } + void NameData::add_ref() + { + AZ_Assert(m_useCount >= 0, "NameData has been deleted"); + ++m_useCount; + } - void NameData::add_ref() + void NameData::release() + { + // this could be released after we decrement the counter, therefore we will + // base the release on the hash which is stable + Hash hash = m_hash; + AZ_Assert(m_useCount > 0, "m_useCount is already 0!"); + if (m_useCount.fetch_sub(1) == 1) { - AZ_Assert(m_useCount >= 0, "NameData has been deleted"); - ++m_useCount; - } - - void NameData::release() - { - // this could be released after we decrement the counter, therefore we will - // base the release on the hash which is stable - Hash hash = m_hash; - AZ_Assert(m_useCount > 0, "m_useCount is already 0!"); - if (m_useCount.fetch_sub(1) == 1) + if (AZ::NameDictionary::IsReady()) { - if (AZ::NameDictionary::IsReady()) - { - AZ::NameDictionary::Instance().TryReleaseName(hash); - } + AZ::NameDictionary::Instance().TryReleaseName(hash); } } } -} - +} // namespace AZ::Internal diff --git a/Code/Framework/AzCore/AzCore/Platform.cpp b/Code/Framework/AzCore/AzCore/Platform.cpp index ad345f65e8..0defef826e 100644 --- a/Code/Framework/AzCore/AzCore/Platform.cpp +++ b/Code/Framework/AzCore/AzCore/Platform.cpp @@ -8,19 +8,16 @@ #include -namespace AZ +namespace AZ::Platform { - namespace Platform - { - MachineId s_machineId = MachineId(0); + MachineId s_machineId = MachineId(0); - void SetLocalMachineId(AZ::u32 machineId) + void SetLocalMachineId(AZ::u32 machineId) + { + AZ_Assert(machineId != 0, "0 machine ID is reserved!"); + if (s_machineId != 0) { - AZ_Assert(machineId != 0, "0 machine ID is reserved!"); - if (s_machineId != 0) - { - s_machineId = machineId; - } + s_machineId = machineId; } } -} +} // namespace AZ::Platform diff --git a/Code/Framework/AzCore/AzCore/RTTI/BehaviorContextUtilities.cpp b/Code/Framework/AzCore/AzCore/RTTI/BehaviorContextUtilities.cpp index 6fb03073b0..5632c258c3 100644 --- a/Code/Framework/AzCore/AzCore/RTTI/BehaviorContextUtilities.cpp +++ b/Code/Framework/AzCore/AzCore/RTTI/BehaviorContextUtilities.cpp @@ -117,9 +117,9 @@ namespace AZ if (!explicitOverloads.m_overloads.empty()) { - for (auto methodAndClass : explicitOverloads.m_overloads) + for (const auto& methodAndClass : explicitOverloads.m_overloads) { - overloads.push_back({ methodAndClass.first, methodAndClass.second }); + overloads.emplace_back(methodAndClass.first, methodAndClass.second); } } else @@ -128,7 +128,7 @@ namespace AZ do { - overloads.push_back({ overload, behaviorClass }); + overloads.emplace_back(overload, behaviorClass); overload = overload->m_overload; } while (overload); diff --git a/Code/Framework/AzCore/AzCore/Script/ScriptSystemComponent.cpp b/Code/Framework/AzCore/AzCore/Script/ScriptSystemComponent.cpp index 3b2aebdee6..4b71c57682 100644 --- a/Code/Framework/AzCore/AzCore/Script/ScriptSystemComponent.cpp +++ b/Code/Framework/AzCore/AzCore/Script/ScriptSystemComponent.cpp @@ -170,61 +170,77 @@ ScriptContext* ScriptSystemComponent::AddContext(ScriptContext* context, int ga ScriptContext* ScriptSystemComponent::AddContextWithId(ScriptContextId id) { AZ_Assert(m_contexts.empty() || id != ScriptContextIds::DefaultScriptContextId, "Default script context ID is reserved! Please provide a Unique context ID for you ScriptContext!"); - if (GetContext(id) == nullptr) + if (GetContext(id) != nullptr) { - m_contexts.emplace_back(); - ContextContainer& cc = m_contexts.back(); - cc.m_context = aznew ScriptContext(id); - cc.m_isOwner = true; - cc.m_garbageCollectorSteps = m_defaultGarbageCollectorSteps; - - cc.m_context->SetRequireHook(AZStd::bind(&ScriptSystemComponent::DefaultRequireHook, this, AZStd::placeholders::_1, AZStd::placeholders::_2, AZStd::placeholders::_3)); - - if (id != ScriptContextIds::CryScriptContextId) + return nullptr; + } + m_contexts.emplace_back(); + ContextContainer& cc = m_contexts.back(); + cc.m_context = aznew ScriptContext(id); + cc.m_isOwner = true; + cc.m_garbageCollectorSteps = m_defaultGarbageCollectorSteps; + cc.m_context->SetRequireHook( + [this](lua_State* lua, ScriptContext* context, const char* module) -> int { - // Reflect script classes - ComponentApplication* app = nullptr; - EBUS_EVENT_RESULT(app, ComponentApplicationBus, GetApplication); - if (app && app->GetDescriptor().m_enableScriptReflection) + return DefaultRequireHook(lua, context, module); + }); + + if (id != ScriptContextIds::CryScriptContextId) + { + // Reflect script classes + ComponentApplication* app = nullptr; + EBUS_EVENT_RESULT(app, ComponentApplicationBus, GetApplication); + if (app && app->GetDescriptor().m_enableScriptReflection) + { + if (app->GetBehaviorContext()) { - if (app->GetBehaviorContext()) - { - cc.m_context->BindTo(app->GetBehaviorContext()); - } - else - { - AZ_Error("Script", false, "We are asked to enabled scripting, but the Applicaion has no BehaviorContext! Scripting relies on BehaviorContext!"); - } + cc.m_context->BindTo(app->GetBehaviorContext()); + } + else + { + AZ_Error("Script", false, "We are asked to enabled scripting, but the Applicaion has no BehaviorContext! Scripting relies on BehaviorContext!"); } } - - return cc.m_context; } - return nullptr; + return cc.m_context; } void ScriptSystemComponent::RestoreDefaultRequireHook(ScriptContextId id) { - if (auto context = GetContext(id)) + auto context = GetContext(id); + if (!context) { - for (auto& inMemoryModule : m_inMemoryModules) - { - ClearAssetReferences(inMemoryModule.second->GetId()); - } - - m_inMemoryModules.clear(); - context->SetRequireHook(AZStd::bind(&ScriptSystemComponent::DefaultRequireHook, this, AZStd::placeholders::_1, AZStd::placeholders::_2, AZStd::placeholders::_3)); + return; } + + for (auto& inMemoryModule : m_inMemoryModules) + { + ClearAssetReferences(inMemoryModule.second->GetId()); + } + + m_inMemoryModules.clear(); + context->SetRequireHook( + [this](lua_State* lua, ScriptContext* context, const char* module) -> int + { + return DefaultRequireHook(lua, context, module); + }); } void ScriptSystemComponent::UseInMemoryRequireHook(const InMemoryScriptModules& modules, ScriptContextId id) { - if (auto context = GetContext(id)) + auto context = GetContext(id); + if (nullptr == context) { - m_inMemoryModules = modules; - context->SetRequireHook(AZStd::bind(&ScriptSystemComponent::InMemoryRequireHook, this, AZStd::placeholders::_1, AZStd::placeholders::_2, AZStd::placeholders::_3)); + return; } + + m_inMemoryModules = modules; + context->SetRequireHook( + [this](lua_State* lua, ScriptContext* context, const char* module) -> int + { + return InMemoryRequireHook(lua, context, module); + }); } //========================================================================= diff --git a/Code/Framework/AzCore/AzCore/ScriptCanvas/ScriptCanvasOnDemandNames.cpp b/Code/Framework/AzCore/AzCore/ScriptCanvas/ScriptCanvasOnDemandNames.cpp index b0d141bba9..d0bea11432 100644 --- a/Code/Framework/AzCore/AzCore/ScriptCanvas/ScriptCanvasOnDemandNames.cpp +++ b/Code/Framework/AzCore/AzCore/ScriptCanvas/ScriptCanvasOnDemandNames.cpp @@ -28,133 +28,130 @@ #include #include -namespace AZ +namespace AZ::ScriptCanvasOnDemandReflection { - namespace ScriptCanvasOnDemandReflection + // the use of this might have to come at the end of on demand reflection...instead of instantly + // basically, it required that dependent classes are reflected first, I'm not sure they are yet. + AZStd::string GetPrettyNameForAZTypeId(AZ::BehaviorContext& context, AZ::Uuid typeId) { - // the use of this might have to come at the end of on demand reflection...instead of instantly - // basically, it required that dependent classes are reflected first, I'm not sure they are yet. - AZStd::string GetPrettyNameForAZTypeId(AZ::BehaviorContext& context, AZ::Uuid typeId) + // return capitalized versions of what we need, otherwise just the regular name + // then strip all the stuff + if (typeId == azrtti_typeid()) { - // return capitalized versions of what we need, otherwise just the regular name - // then strip all the stuff - if (typeId == azrtti_typeid()) + return "AABB"; + } + else if (typeId == azrtti_typeid()) + { + return "Boolean"; + } + else if (typeId == azrtti_typeid()) + { + return "Color"; + } + else if (typeId == azrtti_typeid()) + { + return "CRC"; + } + else if (typeId == azrtti_typeid()) + { + return "EntityId"; + } + else if (typeId == azrtti_typeid()) + { + return "Matrix3x3"; + } + else if (typeId == azrtti_typeid()) + { + return "Matrix4x4"; + } + else if (typeId == azrtti_typeid()) + { + return "Number:s8"; + } + else if (typeId == azrtti_typeid()) + { + return "Number:s16"; + } + else if (typeId == azrtti_typeid()) + { + return "Number:s32"; + } + else if (typeId == azrtti_typeid()) + { + return "Number:s64"; + } + else if (typeId == azrtti_typeid()) + { + return "Number:u8"; + } + else if (typeId == azrtti_typeid()) + { + return "Number:u16"; + } + else if (typeId == azrtti_typeid()) + { + return "Number:u32"; + } + else if (typeId == azrtti_typeid()) + { + return "Number:u64"; + } + else if (typeId == azrtti_typeid()) + { + return "Number:float"; + } + else if (typeId == azrtti_typeid()) + { + return "Number"; + } + else if (typeId == azrtti_typeid()) + { + return "OBB"; + } + else if (typeId == azrtti_typeid()) + { + return "Plane"; + } + else if (typeId == azrtti_typeid()) + { + return "Quaternion"; + } + else if (typeId == azrtti_typeid() || typeId == azrtti_typeid()) + { + return "String"; + } + else if (typeId == azrtti_typeid()) + { + return "Transform"; + } + else if (typeId == azrtti_typeid()) + { + return "Vector2"; + } + else if (typeId == azrtti_typeid()) + { + return "Vector3"; + } + else if (typeId == azrtti_typeid()) + { + return "Vector4"; + } + else + { + auto bcClassIter = context.m_typeToClassMap.find(typeId); + if (bcClassIter != context.m_typeToClassMap.end()) { - return "AABB"; - } - else if (typeId == azrtti_typeid()) - { - return "Boolean"; - } - else if (typeId == azrtti_typeid()) - { - return "Color"; - } - else if (typeId == azrtti_typeid()) - { - return "CRC"; - } - else if (typeId == azrtti_typeid()) - { - return "EntityId"; - } - else if (typeId == azrtti_typeid()) - { - return "Matrix3x3"; - } - else if (typeId == azrtti_typeid()) - { - return "Matrix4x4"; - } - else if (typeId == azrtti_typeid()) - { - return "Number:s8"; - } - else if (typeId == azrtti_typeid()) - { - return "Number:s16"; - } - else if (typeId == azrtti_typeid()) - { - return "Number:s32"; - } - else if (typeId == azrtti_typeid()) - { - return "Number:s64"; - } - else if (typeId == azrtti_typeid()) - { - return "Number:u8"; - } - else if (typeId == azrtti_typeid()) - { - return "Number:u16"; - } - else if (typeId == azrtti_typeid()) - { - return "Number:u32"; - } - else if (typeId == azrtti_typeid()) - { - return "Number:u64"; - } - else if (typeId == azrtti_typeid()) - { - return "Number:float"; - } - else if (typeId == azrtti_typeid()) - { - return "Number"; - } - else if (typeId == azrtti_typeid()) - { - return "OBB"; - } - else if (typeId == azrtti_typeid()) - { - return "Plane"; - } - else if (typeId == azrtti_typeid()) - { - return "Quaternion"; - } - else if (typeId == azrtti_typeid() || typeId == azrtti_typeid()) - { - return "String"; - } - else if (typeId == azrtti_typeid()) - { - return "Transform"; - } - else if (typeId == azrtti_typeid()) - { - return "Vector2"; - } - else if (typeId == azrtti_typeid()) - { - return "Vector3"; - } - else if (typeId == azrtti_typeid()) - { - return "Vector4"; + const AZ::BehaviorClass& bcClass = *(bcClassIter->second); + AZStd::string uglyName = bcClass.m_name; + AZ::StringFunc::Replace(uglyName, "AZStd::", "", true); + AZ::StringFunc::Replace(uglyName, "AZ::", "", true); + AZ::StringFunc::Replace(uglyName, "::", ".", true); + return uglyName; } else { - auto bcClassIter = context.m_typeToClassMap.find(typeId); - if (bcClassIter != context.m_typeToClassMap.end()) - { - const AZ::BehaviorClass& bcClass = *(bcClassIter->second); - AZStd::string uglyName = bcClass.m_name; - AZ::StringFunc::Replace(uglyName, "AZStd::", "", true); - AZ::StringFunc::Replace(uglyName, "AZ::", "", true); - AZ::StringFunc::Replace(uglyName, "::", ".", true); - return uglyName; - } - else - { - return "Invalid"; - } + return "Invalid"; } } - } // namespace ScriptCanvasOnDemandReflection -} // namespace AZ + } +} // namespace AZ::ScriptCanvasOnDemandReflection diff --git a/Code/Framework/AzCore/AzCore/Serialization/DataOverlayProviderMsgs.cpp b/Code/Framework/AzCore/AzCore/Serialization/DataOverlayProviderMsgs.cpp index 37f4623301..58f86bf831 100644 --- a/Code/Framework/AzCore/AzCore/Serialization/DataOverlayProviderMsgs.cpp +++ b/Code/Framework/AzCore/AzCore/Serialization/DataOverlayProviderMsgs.cpp @@ -19,8 +19,11 @@ namespace AZ nodeStack.push_back(m_dataContainer); SerializeContext::EnumerateInstanceCallContext callContext( - AZStd::bind(&DataOverlayTarget::ElementBegin, this, &nodeStack, AZStd::placeholders::_1, AZStd::placeholders::_2, AZStd::placeholders::_3), - AZStd::bind(&DataOverlayTarget::ElementEnd, this, &nodeStack), + [this, &nodeStack](void* instancePointer, const SerializeContext::ClassData* classData, const SerializeContext::ClassElement* classElement)->bool + { + return ElementBegin(&nodeStack, instancePointer, classData, classElement); + }, + [this, &nodeStack]()->bool { return ElementEnd(&nodeStack); }, m_sc, SerializeContext::ENUM_ACCESS_FOR_READ, m_errorLogger diff --git a/Code/Framework/AzCore/AzCore/Serialization/DataPatch.cpp b/Code/Framework/AzCore/AzCore/Serialization/DataPatch.cpp index 6f1635148c..d27a5005b4 100644 --- a/Code/Framework/AzCore/AzCore/Serialization/DataPatch.cpp +++ b/Code/Framework/AzCore/AzCore/Serialization/DataPatch.cpp @@ -36,7 +36,7 @@ namespace AZ class DataNode { public: - typedef AZStd::list ChildDataNodes; + using ChildDataNodes = AZStd::list; DataNode() { @@ -148,25 +148,28 @@ namespace AZ m_root.Reset(); m_currentNode = nullptr; - if (m_context && rootClassPtr) + if (!m_context || !rootClassPtr) { - SerializeContext::EnumerateInstanceCallContext callContext( - AZStd::bind(&DataNodeTree::BeginNode, this, AZStd::placeholders::_1, AZStd::placeholders::_2, AZStd::placeholders::_3), - AZStd::bind(&DataNodeTree::EndNode, this), - m_context, - SerializeContext::ENUM_ACCESS_FOR_READ, - nullptr - ); - - m_context->EnumerateInstanceConst( - &callContext, - rootClassPtr, - rootClassId, - nullptr, - nullptr - ); + return; } + SerializeContext::EnumerateInstanceCallContext callContext( + [this](void* instancePointer, const SerializeContext::ClassData* classData, const SerializeContext::ClassElement* classElement)->bool + { + return BeginNode(instancePointer, classData, classElement); + }, + [this]()->bool { return EndNode(); }, + m_context, + SerializeContext::ENUM_ACCESS_FOR_READ, + nullptr + ); + m_context->EnumerateInstanceConst( + &callContext, + rootClassPtr, + rootClassId, + nullptr, + nullptr + ); m_currentNode = nullptr; } diff --git a/Code/Framework/AzCore/AzCore/Serialization/EditContext.cpp b/Code/Framework/AzCore/AzCore/Serialization/EditContext.cpp index d0c433df16..8a4688d748 100644 --- a/Code/Framework/AzCore/AzCore/Serialization/EditContext.cpp +++ b/Code/Framework/AzCore/AzCore/Serialization/EditContext.cpp @@ -34,7 +34,7 @@ namespace AZ } classIt->ClearElements(); } - for (auto enumIt : m_enumData) + for (auto& enumIt : m_enumData) { enumIt.second.ClearAttributes(); } @@ -103,7 +103,7 @@ namespace AZ //========================================================================= void ElementData::ClearAttributes() { - for (auto attrib : m_attributes) + for (auto& attrib : m_attributes) { delete attrib.second; } diff --git a/Code/Framework/AzCore/AzCore/Serialization/Json/JsonDeserializer.cpp b/Code/Framework/AzCore/AzCore/Serialization/Json/JsonDeserializer.cpp index a741294544..4e80d9ba69 100644 --- a/Code/Framework/AzCore/AzCore/Serialization/Json/JsonDeserializer.cpp +++ b/Code/Framework/AzCore/AzCore/Serialization/Json/JsonDeserializer.cpp @@ -759,6 +759,7 @@ namespace AZ else { typeIdResult.m_determination = JsonDeserializer::TypeIdDetermination::FailedToDetermine; + typeIdResult.m_typeId = Uuid::CreateNull(); } } else if (input.IsString()) @@ -768,6 +769,7 @@ namespace AZ else { typeIdResult.m_determination = JsonDeserializer::TypeIdDetermination::FailedToDetermine; + typeIdResult.m_typeId = Uuid::CreateNull(); } switch (typeIdResult.m_determination) diff --git a/Code/Framework/AzCore/AzCore/Serialization/Json/JsonSerializationResult.cpp b/Code/Framework/AzCore/AzCore/Serialization/Json/JsonSerializationResult.cpp index 822c1c43d5..16bad5a466 100644 --- a/Code/Framework/AzCore/AzCore/Serialization/Json/JsonSerializationResult.cpp +++ b/Code/Framework/AzCore/AzCore/Serialization/Json/JsonSerializationResult.cpp @@ -10,265 +10,264 @@ #include #include -namespace AZ +namespace AZ::JsonSerializationResult::Internal { - namespace JsonSerializationResult + template + void AppendToString(AZ::JsonSerializationResult::ResultCode code, + StringType& target, AZStd::string_view path) { - namespace Internal + if (code.GetTask() == static_cast(0)) { - template - void AppendToString(AZ::JsonSerializationResult::ResultCode code, - StringType& target, AZStd::string_view path) - { - if (code.GetTask() == static_cast(0)) - { - target.append("The result code wasn't initialized"); - return; - } - - target.append("The operation "); - switch (code.GetProcessing()) - { - case Processing::Halted: - target.append("has halted during "); - break; - case Processing::Altered: - target.append("has taken an alternative approach for "); - break; - case Processing::PartialAlter: - target.append("has taken a partially alternative approach for "); - break; - case Processing::Completed: - target.append("has completed "); - break; - default: - target.append("has unknown processing status for "); - break; - } - - switch (code.GetTask()) - { - case Tasks::RetrieveInfo: - target.append("a retrieve info operation "); - break; - case Tasks::CreateDefault: - target.append("a create default operation "); - break; - case Tasks::Convert: - target.append("a convert operation "); - break; - case Tasks::ReadField: - target.append("a read field operation "); - break; - case Tasks::WriteValue: - target.append("a write value operation "); - break; - case Tasks::Merge: - target.append("a merge operation "); - break; - case Tasks::CreatePatch: - target.append("a create patch operation "); - break; - case Tasks::Import: - target.append("an import operation"); - break; - default: - target.append("an unknown operation "); - break; - } - - if (!path.empty()) - { - target.append("for '"); - target.append(path.begin(), path.end()); - target.append("' "); - } - - switch (code.GetOutcome()) - { - case Outcomes::Success: - target.append("which resulted in success"); - break; - case Outcomes::DefaultsUsed: - target.append("by using only default values"); - break; - case Outcomes::PartialDefaults: - target.append("by using one or more default values"); - break; - case Outcomes::Skipped: - target.append("because a field or value was skipped"); - break; - case Outcomes::PartialSkip: - target.append("because one or more fields or values were skipped"); - break; - case Outcomes::Unavailable: - target.append("because the target was unavailable"); - break; - case Outcomes::Unsupported: - target.append("because the action was unsupported"); - break; - case Outcomes::TypeMismatch: - target.append("because the source and target are unrelated types"); - break; - case Outcomes::TestFailed: - target.append("because a test against a value failed"); - break; - case Outcomes::Missing: - target.append("because a required field or value was missing"); - break; - case Outcomes::Invalid: - target.append("because a field or element has an invalid value"); - break; - case Outcomes::Unknown: - target.append("because information was missing"); - break; - case Outcomes::Catastrophic: - target.append("because a catastrophic issue was encountered"); - break; - default: - break; - } - } - } // namespace JsonSerializationResultInternal - - ResultCode::ResultCode(Tasks task) - : m_code(0) - { - m_options.m_task = task; + target.append("The result code wasn't initialized"); + return; } - ResultCode::ResultCode(uint32_t code) - : m_code(code) - {} - - ResultCode::ResultCode(Tasks task, Outcomes outcome) + target.append("The operation "); + switch (code.GetProcessing()) { - m_options.m_task = task; - switch (outcome) - { - case Outcomes::Success: // fall through - case Outcomes::Skipped: // fall through - case Outcomes::PartialSkip: // fall through - case Outcomes::DefaultsUsed: // fall through - case Outcomes::PartialDefaults: - m_options.m_processing = Processing::Completed; - break; - case Outcomes::Unavailable: // fall through - case Outcomes::Unsupported: - m_options.m_processing = Processing::Altered; - break; - case Outcomes::TypeMismatch: // fall through - case Outcomes::TestFailed: // fall through - case Outcomes::Missing: // fall through - case Outcomes::Invalid: // fall through - case Outcomes::Unknown: // fall through - case Outcomes::Catastrophic: // fall through - default: - m_options.m_processing = Processing::Halted; - break; - } - m_options.m_outcome = outcome; + case Processing::Halted: + target.append("has halted during "); + break; + case Processing::Altered: + target.append("has taken an alternative approach for "); + break; + case Processing::PartialAlter: + target.append("has taken a partially alternative approach for "); + break; + case Processing::Completed: + target.append("has completed "); + break; + default: + target.append("has unknown processing status for "); + break; } - bool ResultCode::HasDoneWork() const + switch (code.GetTask()) { - return m_options.m_outcome != static_cast(0); + case Tasks::RetrieveInfo: + target.append("a retrieve info operation "); + break; + case Tasks::CreateDefault: + target.append("a create default operation "); + break; + case Tasks::Convert: + target.append("a convert operation "); + break; + case Tasks::ReadField: + target.append("a read field operation "); + break; + case Tasks::WriteValue: + target.append("a write value operation "); + break; + case Tasks::Merge: + target.append("a merge operation "); + break; + case Tasks::CreatePatch: + target.append("a create patch operation "); + break; + case Tasks::Import: + target.append("an import operation"); + break; + default: + target.append("an unknown operation "); + break; } - ResultCode& ResultCode::Combine(ResultCode other) + if (!path.empty()) { - *this = Combine(*this, other); - return *this; + target.append("for '"); + target.append(path.begin(), path.end()); + target.append("' "); } - ResultCode& ResultCode::Combine(const Result& other) + switch (code.GetOutcome()) { - *this = Combine(*this, other.GetResultCode()); - return *this; + case Outcomes::Success: + target.append("which resulted in success"); + break; + case Outcomes::DefaultsUsed: + target.append("by using only default values"); + break; + case Outcomes::PartialDefaults: + target.append("by using one or more default values"); + break; + case Outcomes::Skipped: + target.append("because a field or value was skipped"); + break; + case Outcomes::PartialSkip: + target.append("because one or more fields or values were skipped"); + break; + case Outcomes::Unavailable: + target.append("because the target was unavailable"); + break; + case Outcomes::Unsupported: + target.append("because the action was unsupported"); + break; + case Outcomes::TypeMismatch: + target.append("because the source and target are unrelated types"); + break; + case Outcomes::TestFailed: + target.append("because a test against a value failed"); + break; + case Outcomes::Missing: + target.append("because a required field or value was missing"); + break; + case Outcomes::Invalid: + target.append("because a field or element has an invalid value"); + break; + case Outcomes::Unknown: + target.append("because information was missing"); + break; + case Outcomes::Catastrophic: + target.append("because a catastrophic issue was encountered"); + break; + default: + break; } + } +} // namespace AZ::JsonSerializationResult::Internal - ResultCode ResultCode::Combine(ResultCode lhs, ResultCode rhs) +namespace AZ::JsonSerializationResult +{ + + ResultCode::ResultCode(Tasks task) + : m_code(0) + { + m_options.m_task = task; + } + + ResultCode::ResultCode(uint32_t code) + : m_code(code) + {} + + ResultCode::ResultCode(Tasks task, Outcomes outcome) + { + m_options.m_task = task; + switch (outcome) { - ResultCode result = ResultCode(AZStd::max(lhs.m_code, rhs.m_code)); - - if ((lhs.m_options.m_outcome == Outcomes::Success && rhs.m_options.m_outcome == Outcomes::DefaultsUsed) || - (lhs.m_options.m_outcome == Outcomes::DefaultsUsed && rhs.m_options.m_outcome == Outcomes::Success)) - { - result.m_options.m_outcome = Outcomes::PartialDefaults; - } - else if ((lhs.m_options.m_outcome == Outcomes::Success && rhs.m_options.m_outcome == Outcomes::Skipped) || - (lhs.m_options.m_outcome == Outcomes::Skipped && rhs.m_options.m_outcome == Outcomes::Success)) - { - result.m_options.m_outcome = Outcomes::PartialSkip; - } - - if ((lhs.m_options.m_processing == Processing::Completed && rhs.m_options.m_processing == Processing::Altered) || - (lhs.m_options.m_processing == Processing::Altered && rhs.m_options.m_processing == Processing::Completed)) - { - result.m_options.m_processing = Processing::PartialAlter; - } - - return result; + case Outcomes::Success: // fall through + case Outcomes::Skipped: // fall through + case Outcomes::PartialSkip: // fall through + case Outcomes::DefaultsUsed: // fall through + case Outcomes::PartialDefaults: + m_options.m_processing = Processing::Completed; + break; + case Outcomes::Unavailable: // fall through + case Outcomes::Unsupported: + m_options.m_processing = Processing::Altered; + break; + case Outcomes::TypeMismatch: // fall through + case Outcomes::TestFailed: // fall through + case Outcomes::Missing: // fall through + case Outcomes::Invalid: // fall through + case Outcomes::Unknown: // fall through + case Outcomes::Catastrophic: // fall through + default: + m_options.m_processing = Processing::Halted; + break; } + m_options.m_outcome = outcome; + } - Tasks ResultCode::GetTask() const + bool ResultCode::HasDoneWork() const + { + return m_options.m_outcome != static_cast(0); + } + + ResultCode& ResultCode::Combine(ResultCode other) + { + *this = Combine(*this, other); + return *this; + } + + ResultCode& ResultCode::Combine(const Result& other) + { + *this = Combine(*this, other.GetResultCode()); + return *this; + } + + ResultCode ResultCode::Combine(ResultCode lhs, ResultCode rhs) + { + ResultCode result = ResultCode(AZStd::max(lhs.m_code, rhs.m_code)); + + if ((lhs.m_options.m_outcome == Outcomes::Success && rhs.m_options.m_outcome == Outcomes::DefaultsUsed) || + (lhs.m_options.m_outcome == Outcomes::DefaultsUsed && rhs.m_options.m_outcome == Outcomes::Success)) { - return m_options.m_task; + result.m_options.m_outcome = Outcomes::PartialDefaults; } - - Processing ResultCode::GetProcessing() const + else if ((lhs.m_options.m_outcome == Outcomes::Success && rhs.m_options.m_outcome == Outcomes::Skipped) || + (lhs.m_options.m_outcome == Outcomes::Skipped && rhs.m_options.m_outcome == Outcomes::Success)) { - return m_options.m_processing == static_cast(0) ? - Processing::Completed : m_options.m_processing; + result.m_options.m_outcome = Outcomes::PartialSkip; } - Outcomes ResultCode::GetOutcome() const + if ((lhs.m_options.m_processing == Processing::Completed && rhs.m_options.m_processing == Processing::Altered) || + (lhs.m_options.m_processing == Processing::Altered && rhs.m_options.m_processing == Processing::Completed)) { - return m_options.m_outcome == static_cast(0) ? - Outcomes::DefaultsUsed : m_options.m_outcome; + result.m_options.m_processing = Processing::PartialAlter; } - void ResultCode::AppendToString(AZ::OSString& target, AZStd::string_view path) const - { - Internal::AppendToString(*this, target, path); - } + return result; + } - void ResultCode::AppendToString(AZStd::string& target, AZStd::string_view path) const - { - Internal::AppendToString(*this, target, path); - } + Tasks ResultCode::GetTask() const + { + return m_options.m_task; + } - AZStd::string ResultCode::ToString(AZStd::string_view path) const - { - AZStd::string result; - AppendToString(result, path); - return result; - } + Processing ResultCode::GetProcessing() const + { + return m_options.m_processing == static_cast(0) ? + Processing::Completed : m_options.m_processing; + } - AZ::OSString ResultCode::ToOSString(AZStd::string_view path) const - { - AZ::OSString result; - AppendToString(result, path); - return result; - } + Outcomes ResultCode::GetOutcome() const + { + return m_options.m_outcome == static_cast(0) ? + Outcomes::DefaultsUsed : m_options.m_outcome; + } + + void ResultCode::AppendToString(AZ::OSString& target, AZStd::string_view path) const + { + Internal::AppendToString(*this, target, path); + } + + void ResultCode::AppendToString(AZStd::string& target, AZStd::string_view path) const + { + Internal::AppendToString(*this, target, path); + } + + AZStd::string ResultCode::ToString(AZStd::string_view path) const + { + AZStd::string result; + AppendToString(result, path); + return result; + } + + AZ::OSString ResultCode::ToOSString(AZStd::string_view path) const + { + AZ::OSString result; + AppendToString(result, path); + return result; + } - Result::Result(const JsonIssueCallback& callback, AZStd::string_view message, ResultCode result, AZStd::string_view path) - : m_result(callback(message, result, path)) - {} + Result::Result(const JsonIssueCallback& callback, AZStd::string_view message, ResultCode result, AZStd::string_view path) + : m_result(callback(message, result, path)) + {} - Result::Result(const JsonIssueCallback& callback, AZStd::string_view message, Tasks task, Outcomes outcome, AZStd::string_view path) - : m_result(callback(message, ResultCode(task, outcome), path)) - {} + Result::Result(const JsonIssueCallback& callback, AZStd::string_view message, Tasks task, Outcomes outcome, AZStd::string_view path) + : m_result(callback(message, ResultCode(task, outcome), path)) + {} - Result::operator ResultCode() const - { - return m_result; - } + Result::operator ResultCode() const + { + return m_result; + } - ResultCode Result::GetResultCode() const - { - return m_result; - } - } // namespace JsonSerializationResult -} // namespace AZ + ResultCode Result::GetResultCode() const + { + return m_result; + } + +} // namespace AZ::JsonSerializationResult diff --git a/Code/Framework/AzCore/AzCore/Serialization/Json/JsonUtils.cpp b/Code/Framework/AzCore/AzCore/Serialization/Json/JsonUtils.cpp index 1a4e7b3e80..6d7edb0716 100644 --- a/Code/Framework/AzCore/AzCore/Serialization/Json/JsonUtils.cpp +++ b/Code/Framework/AzCore/AzCore/Serialization/Json/JsonUtils.cpp @@ -24,438 +24,435 @@ #include -namespace AZ +namespace AZ::JsonSerializationUtils { - namespace JsonSerializationUtils + static const char* FileTypeTag = "Type"; + static const char* FileType = "JsonSerialization"; + static const char* VersionTag = "Version"; + static const char* ClassNameTag = "ClassName"; + static const char* ClassDataTag = "ClassData"; + + AZ::Outcome WriteJsonString(const rapidjson::Document& document, AZStd::string& jsonText, WriteJsonSettings settings) { - static const char* FileTypeTag = "Type"; - static const char* FileType = "JsonSerialization"; - static const char* VersionTag = "Version"; - static const char* ClassNameTag = "ClassName"; - static const char* ClassDataTag = "ClassData"; + AZ::IO::ByteContainerStream stream{&jsonText}; + return WriteJsonStream(document, stream, settings); + } - AZ::Outcome WriteJsonString(const rapidjson::Document& document, AZStd::string& jsonText, WriteJsonSettings settings) + AZ::Outcome WriteJsonFile(const rapidjson::Document& document, AZStd::string_view filePath, WriteJsonSettings settings) + { + // Write the json into memory first and then write the file, rather than passing a file stream to rapidjson. + // This should avoid creating a large number of micro-writes to the file. + AZStd::string fileContent; + auto outcome = WriteJsonString(document, fileContent, settings); + if (!outcome.IsSuccess()) { - AZ::IO::ByteContainerStream stream{&jsonText}; - return WriteJsonStream(document, stream, settings); + return outcome; } - AZ::Outcome WriteJsonFile(const rapidjson::Document& document, AZStd::string_view filePath, WriteJsonSettings settings) - { - // Write the json into memory first and then write the file, rather than passing a file stream to rapidjson. - // This should avoid creating a large number of micro-writes to the file. - AZStd::string fileContent; - auto outcome = WriteJsonString(document, fileContent, settings); - if (!outcome.IsSuccess()) - { - return outcome; - } + return AZ::Utils::WriteFile(fileContent, filePath); + } - return AZ::Utils::WriteFile(fileContent, filePath); + AZ::Outcome WriteJsonStream(const rapidjson::Document& document, IO::GenericStream& stream, WriteJsonSettings settings) + { + AZ::IO::RapidJSONStreamWriter jsonStreamWriter(&stream); + + rapidjson::PrettyWriter writer(jsonStreamWriter); + + if (settings.m_maxDecimalPlaces >= 0) + { + writer.SetMaxDecimalPlaces(settings.m_maxDecimalPlaces); } - AZ::Outcome WriteJsonStream(const rapidjson::Document& document, IO::GenericStream& stream, WriteJsonSettings settings) + if (document.Accept(writer)) { - AZ::IO::RapidJSONStreamWriter jsonStreamWriter(&stream); + return AZ::Success(); + } + else + { + return AZ::Failure(AZStd::string{"Json Writer failed"}); + } + } - rapidjson::PrettyWriter writer(jsonStreamWriter); - - if (settings.m_maxDecimalPlaces >= 0) - { - writer.SetMaxDecimalPlaces(settings.m_maxDecimalPlaces); - } - - if (document.Accept(writer)) - { - return AZ::Success(); - } - else - { - return AZ::Failure(AZStd::string{"Json Writer failed"}); - } + AZ::Outcome SaveObjectToStreamByType(const void* objectPtr, const Uuid& classId, IO::GenericStream& stream, + const void* defaultObjectPtr, const JsonSerializerSettings* settings) + { + if (!stream.CanWrite()) + { + return AZ::Failure(AZStd::string("The GenericStream can't be written to")); } - AZ::Outcome SaveObjectToStreamByType(const void* objectPtr, const Uuid& classId, IO::GenericStream& stream, - const void* defaultObjectPtr, const JsonSerializerSettings* settings) + JsonSerializerSettings saveSettings; + if (settings) { - if (!stream.CanWrite()) - { - return AZ::Failure(AZStd::string("The GenericStream can't be written to")); - } + saveSettings = *settings; + } - JsonSerializerSettings saveSettings; - if (settings) - { - saveSettings = *settings; - } - - AZ::SerializeContext* serializeContext = saveSettings.m_serializeContext; + AZ::SerializeContext* serializeContext = saveSettings.m_serializeContext; + if (!serializeContext) + { + AZ::ComponentApplicationBus::BroadcastResult(serializeContext, &AZ::ComponentApplicationBus::Events::GetSerializeContext); if (!serializeContext) { - AZ::ComponentApplicationBus::BroadcastResult(serializeContext, &AZ::ComponentApplicationBus::Events::GetSerializeContext); - if (!serializeContext) - { - return AZ::Failure(AZStd::string::format("Need SerializeContext for saving")); - } - saveSettings.m_serializeContext = serializeContext; + return AZ::Failure(AZStd::string::format("Need SerializeContext for saving")); } - - rapidjson::Document jsonDocument; - jsonDocument.SetObject(); - jsonDocument.AddMember(rapidjson::StringRef(FileTypeTag), rapidjson::StringRef(FileType), jsonDocument.GetAllocator()); - - rapidjson::Value serializedObject; - - JsonSerializationResult::ResultCode jsonResult = JsonSerialization::Store(serializedObject, jsonDocument.GetAllocator(), - objectPtr, defaultObjectPtr, classId, saveSettings); - - if (jsonResult.GetProcessing() != JsonSerializationResult::Processing::Completed) - { - return AZ::Failure(jsonResult.ToString("")); - } - - const SerializeContext::ClassData* classData = serializeContext->FindClassData(classId); - - jsonDocument.AddMember(rapidjson::StringRef(VersionTag), 1, jsonDocument.GetAllocator()); - jsonDocument.AddMember(rapidjson::StringRef(ClassNameTag), rapidjson::StringRef(classData->m_name), jsonDocument.GetAllocator()); - jsonDocument.AddMember(rapidjson::StringRef(ClassDataTag), AZStd::move(serializedObject), jsonDocument.GetAllocator()); - - AZ::IO::RapidJSONStreamWriter jsonStreamWriter(&stream); - rapidjson::PrettyWriter writer(jsonStreamWriter); - bool jsonWriteResult = jsonDocument.Accept(writer); - if (!jsonWriteResult) - { - return AZ::Failure(AZStd::string::format("Unable to write class %s with json serialization format'", - classId.ToString().data())); - } - - return AZ::Success(); + saveSettings.m_serializeContext = serializeContext; } - AZ::Outcome SaveObjectToFileByType(const void* classPtr, const Uuid& classId, const AZStd::string& filePath, - const void* defaultClassPtr, const JsonSerializerSettings* settings) + rapidjson::Document jsonDocument; + jsonDocument.SetObject(); + jsonDocument.AddMember(rapidjson::StringRef(FileTypeTag), rapidjson::StringRef(FileType), jsonDocument.GetAllocator()); + + rapidjson::Value serializedObject; + + JsonSerializationResult::ResultCode jsonResult = JsonSerialization::Store(serializedObject, jsonDocument.GetAllocator(), + objectPtr, defaultObjectPtr, classId, saveSettings); + + if (jsonResult.GetProcessing() != JsonSerializationResult::Processing::Completed) { - AZStd::vector buffer; - buffer.reserve(1024); - AZ::IO::ByteContainerStream > byteStream(&buffer); - auto saveResult = SaveObjectToStreamByType(classPtr, classId, byteStream, defaultClassPtr, settings); - if (saveResult.IsSuccess()) - { - AZ::IO::FileIOStream outputFileStream; - if (!outputFileStream.Open(filePath.c_str(), AZ::IO::OpenMode::ModeWrite | AZ::IO::OpenMode::ModeCreatePath | AZ::IO::OpenMode::ModeText)) - { - return AZ::Failure(AZStd::string::format("Error opening file '%s' for writing", filePath.c_str())); - } - outputFileStream.Write(buffer.size(), buffer.data()); - } - return saveResult; + return AZ::Failure(jsonResult.ToString("")); } - // Helper function to check whether the load outcome was success (for loading json serialization file) - bool WasLoadSuccess(JsonSerializationResult::Outcomes outcome) - { - return (outcome == JsonSerializationResult::Outcomes::Success - || outcome == JsonSerializationResult::Outcomes::DefaultsUsed - || outcome == JsonSerializationResult::Outcomes::PartialDefaults); - } - - AZ::Outcome PrepareDeserializerSettings(const JsonDeserializerSettings* inputSettings, JsonDeserializerSettings& returnSettings - , AZStd::string& deserializeError) - { - if (inputSettings) - { - returnSettings = *inputSettings; - } + const SerializeContext::ClassData* classData = serializeContext->FindClassData(classId); + jsonDocument.AddMember(rapidjson::StringRef(VersionTag), 1, jsonDocument.GetAllocator()); + jsonDocument.AddMember(rapidjson::StringRef(ClassNameTag), rapidjson::StringRef(classData->m_name), jsonDocument.GetAllocator()); + jsonDocument.AddMember(rapidjson::StringRef(ClassDataTag), AZStd::move(serializedObject), jsonDocument.GetAllocator()); + + AZ::IO::RapidJSONStreamWriter jsonStreamWriter(&stream); + rapidjson::PrettyWriter writer(jsonStreamWriter); + bool jsonWriteResult = jsonDocument.Accept(writer); + if (!jsonWriteResult) + { + return AZ::Failure(AZStd::string::format("Unable to write class %s with json serialization format'", + classId.ToString().data())); + } + + return AZ::Success(); + } + + AZ::Outcome SaveObjectToFileByType(const void* classPtr, const Uuid& classId, const AZStd::string& filePath, + const void* defaultClassPtr, const JsonSerializerSettings* settings) + { + AZStd::vector buffer; + buffer.reserve(1024); + AZ::IO::ByteContainerStream > byteStream(&buffer); + auto saveResult = SaveObjectToStreamByType(classPtr, classId, byteStream, defaultClassPtr, settings); + if (saveResult.IsSuccess()) + { + AZ::IO::FileIOStream outputFileStream; + if (!outputFileStream.Open(filePath.c_str(), AZ::IO::OpenMode::ModeWrite | AZ::IO::OpenMode::ModeCreatePath | AZ::IO::OpenMode::ModeText)) + { + return AZ::Failure(AZStd::string::format("Error opening file '%s' for writing", filePath.c_str())); + } + outputFileStream.Write(buffer.size(), buffer.data()); + } + return saveResult; + } + + // Helper function to check whether the load outcome was success (for loading json serialization file) + bool WasLoadSuccess(JsonSerializationResult::Outcomes outcome) + { + return (outcome == JsonSerializationResult::Outcomes::Success + || outcome == JsonSerializationResult::Outcomes::DefaultsUsed + || outcome == JsonSerializationResult::Outcomes::PartialDefaults); + } + + AZ::Outcome PrepareDeserializerSettings(const JsonDeserializerSettings* inputSettings, JsonDeserializerSettings& returnSettings + , AZStd::string& deserializeError) + { + if (inputSettings) + { + returnSettings = *inputSettings; + } + + if (!returnSettings.m_serializeContext) + { + AZ::ComponentApplicationBus::BroadcastResult(returnSettings.m_serializeContext, &AZ::ComponentApplicationBus::Events::GetSerializeContext); if (!returnSettings.m_serializeContext) { - AZ::ComponentApplicationBus::BroadcastResult(returnSettings.m_serializeContext, &AZ::ComponentApplicationBus::Events::GetSerializeContext); - if (!returnSettings.m_serializeContext) + return AZ::Failure(AZStd::string("Need SerializeContext for loading")); + } + } + + // Report unused data field as error by default + auto reporting = returnSettings.m_reporting; + auto issueReportingCallback = [&deserializeError, reporting](AZStd::string_view message, JsonSerializationResult::ResultCode result, AZStd::string_view target) -> JsonSerializationResult::ResultCode + { + using namespace JsonSerializationResult; + + if (!WasLoadSuccess(result.GetOutcome())) + { + // This if is a hack around fault in the JSON serialization system + // Jira: LY-106587 + if (message != "No part of the string could be interpreted as a uuid.") { - return AZ::Failure(AZStd::string("Need SerializeContext for loading")); + deserializeError.append(message); + deserializeError.append(AZStd::string::format(" '%s' \n", target.data())); } } - // Report unused data field as error by default - auto reporting = returnSettings.m_reporting; - auto issueReportingCallback = [&deserializeError, reporting](AZStd::string_view message, JsonSerializationResult::ResultCode result, AZStd::string_view target) -> JsonSerializationResult::ResultCode + if (reporting) { - using namespace JsonSerializationResult; + result = reporting(message, result, target); + } - if (!WasLoadSuccess(result.GetOutcome())) - { - // This if is a hack around fault in the JSON serialization system - // Jira: LY-106587 - if (message != "No part of the string could be interpreted as a uuid.") - { - deserializeError.append(message); - deserializeError.append(AZStd::string::format(" '%s' \n", target.data())); - } - } + return result; + }; - if (reporting) - { - result = reporting(message, result, target); - } + returnSettings.m_reporting = issueReportingCallback; - return result; - }; + return AZ::Success(); + } - returnSettings.m_reporting = issueReportingCallback; - return AZ::Success(); + AZ::Outcome ReadJsonString(AZStd::string_view jsonText) + { + if (jsonText.empty()) + { + return AZ::Failure(AZStd::string("Failed to parse JSON: input string is empty.")); } - - AZ::Outcome ReadJsonString(AZStd::string_view jsonText) + rapidjson::Document jsonDocument; + jsonDocument.Parse(jsonText.data(), jsonText.size()); + if (jsonDocument.HasParseError()) { - if (jsonText.empty()) + size_t lineNumber = 1; + + const size_t errorOffset = jsonDocument.GetErrorOffset(); + for (size_t searchOffset = jsonText.find('\n'); + searchOffset < errorOffset && searchOffset < AZStd::string::npos; + searchOffset = jsonText.find('\n', searchOffset + 1)) { - return AZ::Failure(AZStd::string("Failed to parse JSON: input string is empty.")); + lineNumber++; } - rapidjson::Document jsonDocument; - jsonDocument.Parse(jsonText.data(), jsonText.size()); - if (jsonDocument.HasParseError()) - { - size_t lineNumber = 1; + return AZ::Failure(AZStd::string::format("JSON parse error at line %zu: %s", lineNumber, rapidjson::GetParseError_En(jsonDocument.GetParseError()))); + } + else + { + return AZ::Success(AZStd::move(jsonDocument)); + } + } - const size_t errorOffset = jsonDocument.GetErrorOffset(); - for (size_t searchOffset = jsonText.find('\n'); - searchOffset < errorOffset && searchOffset < AZStd::string::npos; - searchOffset = jsonText.find('\n', searchOffset + 1)) - { - lineNumber++; - } - - return AZ::Failure(AZStd::string::format("JSON parse error at line %zu: %s", lineNumber, rapidjson::GetParseError_En(jsonDocument.GetParseError()))); - } - else - { - return AZ::Success(AZStd::move(jsonDocument)); - } + AZ::Outcome ReadJsonStream(IO::GenericStream& stream) + { + IO::SizeType length = stream.GetLength(); + + AZStd::vector memoryBuffer; + memoryBuffer.resize_no_construct(static_cast::size_type>(static_cast::size_type>(length) + 1)); + + IO::SizeType bytesRead = stream.Read(length, memoryBuffer.data()); + if (bytesRead != length) + { + return AZ::Failure(AZStd::string{"Cannot to read input stream."}); } - AZ::Outcome ReadJsonStream(IO::GenericStream& stream) + memoryBuffer.back() = 0; + + return ReadJsonString(AZStd::string_view{memoryBuffer.data(), memoryBuffer.size()}); + } + + AZ::Outcome ReadJsonFile(AZStd::string_view filePath, size_t maxFileSize) + { + // Read into memory first and then parse the json, rather than passing a file stream to rapidjson. + // This should avoid creating a large number of micro-reads from the file. + + auto readResult = AZ::Utils::ReadFile(filePath, maxFileSize); + if(!readResult.IsSuccess()) { - IO::SizeType length = stream.GetLength(); - - AZStd::vector memoryBuffer; - memoryBuffer.resize_no_construct(static_cast::size_type>(static_cast::size_type>(length) + 1)); - - IO::SizeType bytesRead = stream.Read(length, memoryBuffer.data()); - if (bytesRead != length) - { - return AZ::Failure(AZStd::string{"Cannot to read input stream."}); - } - - memoryBuffer.back() = 0; - - return ReadJsonString(AZStd::string_view{memoryBuffer.data(), memoryBuffer.size()}); + return AZ::Failure(readResult.GetError()); } - AZ::Outcome ReadJsonFile(AZStd::string_view filePath, size_t maxFileSize) + AZStd::string jsonContent = readResult.TakeValue(); + + auto result = ReadJsonString(jsonContent); + if (!result.IsSuccess()) { - // Read into memory first and then parse the json, rather than passing a file stream to rapidjson. - // This should avoid creating a large number of micro-reads from the file. + return AZ::Failure(AZStd::string::format("Failed to load '%.*s'. %s", AZ_STRING_ARG(filePath), result.GetError().c_str())); + } + else + { + return result; + } + } - auto readResult = AZ::Utils::ReadFile(filePath, maxFileSize); - if(!readResult.IsSuccess()) - { - return AZ::Failure(readResult.GetError()); - } - - AZStd::string jsonContent = readResult.TakeValue(); - - auto result = ReadJsonString(jsonContent); - if (!result.IsSuccess()) - { - return AZ::Failure(AZStd::string::format("Failed to load '%.*s'. %s", AZ_STRING_ARG(filePath), result.GetError().c_str())); - } - else - { - return result; - } + // Helper function to validate the JSON is structured with the standard header for a generic class + AZ::Outcome ValidateJsonClassHeader(const rapidjson::Document& jsonDocument) + { + auto typeItr = jsonDocument.FindMember(FileTypeTag); + if (typeItr == jsonDocument.MemberEnd() || !typeItr->value.IsString() || azstricmp(typeItr->value.GetString(), FileType) != 0) + { + return AZ::Failure(AZStd::string::format("Not a valid JsonSerialization file")); } - // Helper function to validate the JSON is structured with the standard header for a generic class - AZ::Outcome ValidateJsonClassHeader(const rapidjson::Document& jsonDocument) + auto nameItr = jsonDocument.FindMember(ClassNameTag); + if (nameItr == jsonDocument.MemberEnd() || !nameItr->value.IsString()) { - auto typeItr = jsonDocument.FindMember(FileTypeTag); - if (typeItr == jsonDocument.MemberEnd() || !typeItr->value.IsString() || azstricmp(typeItr->value.GetString(), FileType) != 0) - { - return AZ::Failure(AZStd::string::format("Not a valid JsonSerialization file")); - } - - auto nameItr = jsonDocument.FindMember(ClassNameTag); - if (nameItr == jsonDocument.MemberEnd() || !nameItr->value.IsString()) - { - return AZ::Failure(AZStd::string::format("File should contain ClassName")); - } - - auto dataItr = jsonDocument.FindMember(ClassDataTag); - // data can be empty but it should be an object - if (dataItr != jsonDocument.MemberEnd() && !dataItr->value.IsObject()) - { - return AZ::Failure(AZStd::string::format("ClassData should be an object")); - } - - return AZ::Success(); + return AZ::Failure(AZStd::string::format("File should contain ClassName")); } - AZ::Outcome LoadObjectFromStringByType(void* objectToLoad, const Uuid& classId, AZStd::string_view stream, - const JsonDeserializerSettings* settings) + auto dataItr = jsonDocument.FindMember(ClassDataTag); + // data can be empty but it should be an object + if (dataItr != jsonDocument.MemberEnd() && !dataItr->value.IsObject()) { - JsonDeserializerSettings loadSettings; - AZStd::string deserializeErrors; - auto prepare = PrepareDeserializerSettings(settings, loadSettings, deserializeErrors); - if (!prepare.IsSuccess()) - { - return AZ::Failure(prepare.GetError()); - } + return AZ::Failure(AZStd::string::format("ClassData should be an object")); + } - auto parseResult = ReadJsonString(stream); - if (!parseResult.IsSuccess()) - { - return AZ::Failure(parseResult.GetError()); - } + return AZ::Success(); + } - const rapidjson::Document& jsonDocument = parseResult.GetValue(); + AZ::Outcome LoadObjectFromStringByType(void* objectToLoad, const Uuid& classId, AZStd::string_view stream, + const JsonDeserializerSettings* settings) + { + JsonDeserializerSettings loadSettings; + AZStd::string deserializeErrors; + auto prepare = PrepareDeserializerSettings(settings, loadSettings, deserializeErrors); + if (!prepare.IsSuccess()) + { + return AZ::Failure(prepare.GetError()); + } - auto validateResult = ValidateJsonClassHeader(jsonDocument); - if (!validateResult.IsSuccess()) - { - return AZ::Failure(validateResult.GetError()); - } + auto parseResult = ReadJsonString(stream); + if (!parseResult.IsSuccess()) + { + return AZ::Failure(parseResult.GetError()); + } - const char* className = jsonDocument.FindMember(ClassNameTag)->value.GetString(); + const rapidjson::Document& jsonDocument = parseResult.GetValue(); - // validate class name - auto classData = loadSettings.m_serializeContext->FindClassData(classId); - if (!classData) - { - return AZ::Failure(AZStd::string::format("Try to load class from Id %s", classId.ToString().c_str())); - } + auto validateResult = ValidateJsonClassHeader(jsonDocument); + if (!validateResult.IsSuccess()) + { + return AZ::Failure(validateResult.GetError()); + } - if (azstricmp(classData->m_name, className) != 0) - { - return AZ::Failure(AZStd::string::format("Try to load class %s from class %s data", classData->m_name, className)); - } + const char* className = jsonDocument.FindMember(ClassNameTag)->value.GetString(); - JsonSerializationResult::ResultCode result = JsonSerialization::Load(objectToLoad, classId, jsonDocument.FindMember(ClassDataTag)->value, loadSettings); + // validate class name + auto classData = loadSettings.m_serializeContext->FindClassData(classId); + if (!classData) + { + return AZ::Failure(AZStd::string::format("Try to load class from Id %s", classId.ToString().c_str())); + } + + if (azstricmp(classData->m_name, className) != 0) + { + return AZ::Failure(AZStd::string::format("Try to load class %s from class %s data", classData->m_name, className)); + } + + JsonSerializationResult::ResultCode result = JsonSerialization::Load(objectToLoad, classId, jsonDocument.FindMember(ClassDataTag)->value, loadSettings); + + if (!WasLoadSuccess(result.GetOutcome()) || !deserializeErrors.empty()) + { + return AZ::Failure(deserializeErrors); + } + + return AZ::Success(); + } + + AZ::Outcome LoadObjectFromStreamByType(void* objectToLoad, const Uuid& classId, IO::GenericStream& stream, + const JsonDeserializerSettings* settings) + { + JsonDeserializerSettings loadSettings; + AZStd::string deserializeErrors; + auto prepare = PrepareDeserializerSettings(settings, loadSettings, deserializeErrors); + if (!prepare.IsSuccess()) + { + return AZ::Failure(prepare.GetError()); + } + + auto parseResult = ReadJsonStream(stream); + if (!parseResult.IsSuccess()) + { + return AZ::Failure(parseResult.GetError()); + } + + const rapidjson::Document& jsonDocument = parseResult.GetValue(); + + auto validateResult = ValidateJsonClassHeader(jsonDocument); + if (!validateResult.IsSuccess()) + { + return AZ::Failure(validateResult.GetError()); + } + + const char* className = jsonDocument.FindMember(ClassNameTag)->value.GetString(); + + // validate class name + auto classData = loadSettings.m_serializeContext->FindClassData(classId); + if (!classData) + { + return AZ::Failure(AZStd::string::format("Try to load class from Id %s", classId.ToString().c_str())); + } + + if (azstricmp(classData->m_name, className) != 0) + { + return AZ::Failure(AZStd::string::format("Try to load class %s from class %s data", classData->m_name, className)); + } + + JsonSerializationResult::ResultCode result = JsonSerialization::Load(objectToLoad, classId, jsonDocument.FindMember(ClassDataTag)->value, loadSettings); + + if (!WasLoadSuccess(result.GetOutcome()) || !deserializeErrors.empty()) + { + return AZ::Failure(deserializeErrors); + } + + return AZ::Success(); + } + + AZ::Outcome LoadAnyObjectFromStream(IO::GenericStream& stream, const JsonDeserializerSettings* settings) + { + JsonDeserializerSettings loadSettings; + AZStd::string deserializeErrors; + auto prepare = PrepareDeserializerSettings(settings, loadSettings, deserializeErrors); + if (!prepare.IsSuccess()) + { + return AZ::Failure(prepare.GetError()); + } + + auto parseResult = ReadJsonStream(stream); + if (!parseResult.IsSuccess()) + { + return AZ::Failure(parseResult.GetError()); + } + + const rapidjson::Document& jsonDocument = parseResult.GetValue(); + + auto validateResult = ValidateJsonClassHeader(jsonDocument); + if (!parseResult.IsSuccess()) + { + return AZ::Failure(parseResult.GetError()); + } + + const char* className = jsonDocument.FindMember(ClassNameTag)->value.GetString(); + AZStd::vector ids = loadSettings.m_serializeContext->FindClassId(AZ::Crc32(className)); + + // Load with first found class id + if (ids.size() >= 1) + { + auto classId = ids[0]; + AZStd::any anyData = loadSettings.m_serializeContext->CreateAny(classId); + auto& objectData = jsonDocument.FindMember(ClassDataTag)->value; + JsonSerializationResult::ResultCode result = JsonSerialization::Load(AZStd::any_cast(&anyData), classId, objectData, loadSettings); if (!WasLoadSuccess(result.GetOutcome()) || !deserializeErrors.empty()) { return AZ::Failure(deserializeErrors); } - return AZ::Success(); + return AZ::Success(anyData); } - AZ::Outcome LoadObjectFromStreamByType(void* objectToLoad, const Uuid& classId, IO::GenericStream& stream, - const JsonDeserializerSettings* settings) + return AZ::Failure(AZStd::string::format("Can't find serialize context for class %s", className)); + } + + AZ::Outcome LoadAnyObjectFromFile(const AZStd::string& filePath, const JsonDeserializerSettings* settings) + { + AZ::IO::FileIOStream inputFileStream; + if (!inputFileStream.Open(filePath.c_str(), AZ::IO::OpenMode::ModeRead | AZ::IO::OpenMode::ModeText)) { - JsonDeserializerSettings loadSettings; - AZStd::string deserializeErrors; - auto prepare = PrepareDeserializerSettings(settings, loadSettings, deserializeErrors); - if (!prepare.IsSuccess()) - { - return AZ::Failure(prepare.GetError()); - } - - auto parseResult = ReadJsonStream(stream); - if (!parseResult.IsSuccess()) - { - return AZ::Failure(parseResult.GetError()); - } - - const rapidjson::Document& jsonDocument = parseResult.GetValue(); - - auto validateResult = ValidateJsonClassHeader(jsonDocument); - if (!validateResult.IsSuccess()) - { - return AZ::Failure(validateResult.GetError()); - } - - const char* className = jsonDocument.FindMember(ClassNameTag)->value.GetString(); - - // validate class name - auto classData = loadSettings.m_serializeContext->FindClassData(classId); - if (!classData) - { - return AZ::Failure(AZStd::string::format("Try to load class from Id %s", classId.ToString().c_str())); - } - - if (azstricmp(classData->m_name, className) != 0) - { - return AZ::Failure(AZStd::string::format("Try to load class %s from class %s data", classData->m_name, className)); - } - - JsonSerializationResult::ResultCode result = JsonSerialization::Load(objectToLoad, classId, jsonDocument.FindMember(ClassDataTag)->value, loadSettings); - - if (!WasLoadSuccess(result.GetOutcome()) || !deserializeErrors.empty()) - { - return AZ::Failure(deserializeErrors); - } - - return AZ::Success(); - } - - AZ::Outcome LoadAnyObjectFromStream(IO::GenericStream& stream, const JsonDeserializerSettings* settings) - { - JsonDeserializerSettings loadSettings; - AZStd::string deserializeErrors; - auto prepare = PrepareDeserializerSettings(settings, loadSettings, deserializeErrors); - if (!prepare.IsSuccess()) - { - return AZ::Failure(prepare.GetError()); - } - - auto parseResult = ReadJsonStream(stream); - if (!parseResult.IsSuccess()) - { - return AZ::Failure(parseResult.GetError()); - } - - const rapidjson::Document& jsonDocument = parseResult.GetValue(); - - auto validateResult = ValidateJsonClassHeader(jsonDocument); - if (!parseResult.IsSuccess()) - { - return AZ::Failure(parseResult.GetError()); - } - - const char* className = jsonDocument.FindMember(ClassNameTag)->value.GetString(); - AZStd::vector ids = loadSettings.m_serializeContext->FindClassId(AZ::Crc32(className)); - - // Load with first found class id - if (ids.size() >= 1) - { - auto classId = ids[0]; - AZStd::any anyData = loadSettings.m_serializeContext->CreateAny(classId); - auto& objectData = jsonDocument.FindMember(ClassDataTag)->value; - JsonSerializationResult::ResultCode result = JsonSerialization::Load(AZStd::any_cast(&anyData), classId, objectData, loadSettings); - - if (!WasLoadSuccess(result.GetOutcome()) || !deserializeErrors.empty()) - { - return AZ::Failure(deserializeErrors); - } - - return AZ::Success(anyData); - } - - return AZ::Failure(AZStd::string::format("Can't find serialize context for class %s", className)); + return AZ::Failure(AZStd::string::format("Error opening file '%s' for reading", filePath.c_str())); } + return LoadAnyObjectFromStream(inputFileStream, settings); + } - AZ::Outcome LoadAnyObjectFromFile(const AZStd::string& filePath, const JsonDeserializerSettings* settings) - { - AZ::IO::FileIOStream inputFileStream; - if (!inputFileStream.Open(filePath.c_str(), AZ::IO::OpenMode::ModeRead | AZ::IO::OpenMode::ModeText)) - { - return AZ::Failure(AZStd::string::format("Error opening file '%s' for reading", filePath.c_str())); - } - return LoadAnyObjectFromStream(inputFileStream, settings); - } - - } // namespace JsonSerializationUtils -} // namespace AZ +} // namespace AZ::JsonSerializationUtils diff --git a/Code/Framework/AzCore/AzCore/Serialization/ObjectStream.cpp b/Code/Framework/AzCore/AzCore/Serialization/ObjectStream.cpp index 746c80f3ea..b53515c071 100644 --- a/Code/Framework/AzCore/AzCore/Serialization/ObjectStream.cpp +++ b/Code/Framework/AzCore/AzCore/Serialization/ObjectStream.cpp @@ -1050,7 +1050,7 @@ namespace AZ } m_xmlNode = next; - Uuid specializedId; + Uuid specializedId = Uuid::CreateNull(); // now parse the node rapidxml::xml_attribute* attr = m_xmlNode->first_attribute(); while (attr) diff --git a/Code/Framework/AzCore/AzCore/Serialization/SerializationUtils.cpp b/Code/Framework/AzCore/AzCore/Serialization/SerializationUtils.cpp index 37eacd940e..8439ff8e2c 100644 --- a/Code/Framework/AzCore/AzCore/Serialization/SerializationUtils.cpp +++ b/Code/Framework/AzCore/AzCore/Serialization/SerializationUtils.cpp @@ -18,502 +18,499 @@ #include -namespace AZ +namespace AZ::Utils { - namespace Utils + bool LoadObjectFromStreamInPlace(IO::GenericStream& stream, AZ::SerializeContext* context, const SerializeContext::ClassData* objectClassData, void* targetPointer, const FilterDescriptor& filterDesc) { - bool LoadObjectFromStreamInPlace(IO::GenericStream& stream, AZ::SerializeContext* context, const SerializeContext::ClassData* objectClassData, void* targetPointer, const FilterDescriptor& filterDesc) + AZ_PROFILE_FUNCTION(AzCore); + + AZ_Assert(objectClassData, "Class data is required."); + + if (!context) { - AZ_PROFILE_FUNCTION(AzCore); - - AZ_Assert(objectClassData, "Class data is required."); - - if (!context) - { - EBUS_EVENT_RESULT(context, ComponentApplicationBus, GetSerializeContext); - AZ_Assert(context, "No serialize context"); - } - - if (!context) - { - return false; - } - - AZ_Assert(targetPointer, "You must provide a target pointer"); - - bool foundSuccess = false; - using CreationCallback = AZStd::function; - auto handler = [&targetPointer, objectClassData, &foundSuccess](void** instance, const SerializeContext::ClassData** classData, const Uuid& classId, SerializeContext* context) - { - void* convertibleInstance{}; - if (objectClassData->ConvertFromType(convertibleInstance, classId, targetPointer, *context)) - { - foundSuccess = true; - if (instance) - { - // The ObjectStream will ask us for the address of the target to load into, so provide it. - *instance = convertibleInstance; - } - if (classData) - { - // The ObjectStream will ask us for the class data of the target being loaded into, so provide it if needed. - // This allows us to load directly into a generic object (templated containers, strings, etc). - *classData = objectClassData; - } - } - }; - bool readSuccess = ObjectStream::LoadBlocking(&stream, *context, ObjectStream::ClassReadyCB(), filterDesc, CreationCallback(handler, AZ::OSStdAllocator())); - - AZ_Warning("Serialization", readSuccess, "LoadObjectFromStreamInPlace: Stream did not deserialize correctly"); - AZ_Warning("Serialization", foundSuccess, "LoadObjectFromStreamInPlace: Did not find the expected type in the stream"); - - return readSuccess && foundSuccess; + EBUS_EVENT_RESULT(context, ComponentApplicationBus, GetSerializeContext); + AZ_Assert(context, "No serialize context"); } - bool LoadObjectFromStreamInPlace(IO::GenericStream& stream, AZ::SerializeContext* context, const Uuid& targetClassId, void* targetPointer, const FilterDescriptor& filterDesc) + if (!context) { - AZ_PROFILE_FUNCTION(AzCore); - - if (!context) - { - EBUS_EVENT_RESULT(context, ComponentApplicationBus, GetSerializeContext); - AZ_Assert(context, "No serialize context"); - } - - if (!context) - { - return false; - } - - const SerializeContext::ClassData* classData = context->FindClassData(targetClassId); - if (!classData) - { - AZ_Error("Serialization", false, - "Unable to locate class data for uuid \"%s\". This object cannot be serialized as a root element. " - "Make sure the Uuid is valid, or if this is a generic type, use the override that takes a ClassData pointer instead.", - targetClassId.ToString().c_str()); - return false; - } - - return LoadObjectFromStreamInPlace(stream, context, classData, targetPointer, filterDesc); - } - - bool LoadObjectFromFileInPlace(const AZStd::string& filePath, const Uuid& targetClassId, void* destination, AZ::SerializeContext* context /*= nullptr*/, const FilterDescriptor& filterDesc /*= FilterDescriptor()*/) - { - AZ::IO::FileIOStream fileStream; - if (!fileStream.Open(filePath.c_str(), IO::OpenMode::ModeRead | IO::OpenMode::ModeBinary)) - { - return false; - } - - return LoadObjectFromStreamInPlace(fileStream, context, targetClassId, destination, filterDesc); - } - - void* LoadObjectFromStream(IO::GenericStream& stream, AZ::SerializeContext* context, const Uuid* targetClassId, const FilterDescriptor& filterDesc) - { - AZ_PROFILE_FUNCTION(AzCore); - - if (!context) - { - EBUS_EVENT_RESULT(context, ComponentApplicationBus, GetSerializeContext); - AZ_Assert(context, "No serialize context"); - } - - if (!context) - { - return nullptr; - } - - void* loadedInstance = nullptr; - bool success = ObjectStream::LoadBlocking(&stream, *context, - [&loadedInstance, targetClassId](void* classPtr, const Uuid& classId, const SerializeContext* serializeContext) - { - if (targetClassId) - { - void* instance = serializeContext->DownCast(classPtr, classId, *targetClassId); - - // Given a valid object - if (instance) - { - AZ_Assert(!loadedInstance, "loadedInstance must be NULL, otherwise we are being invoked with multiple valid objects"); - loadedInstance = instance; - return; - } - } - else - { - if (!loadedInstance) - { - loadedInstance = classPtr; - return; - } - } - - auto classData = serializeContext->FindClassData(classId); - if (classData && classData->m_factory) - { - classData->m_factory->Destroy(classPtr); - } - }, - filterDesc, - ObjectStream::InplaceLoadRootInfoCB() - ); - - if (!success) - { - return nullptr; - } - - return loadedInstance; - } - - void* LoadObjectFromFile(const AZStd::string& filePath, const Uuid& targetClassId, SerializeContext* context, const FilterDescriptor& filterDesc, int /*platformFlags*/) - { - AZ_PROFILE_FUNCTION(AzCore); - - AZ::IO::FileIOStream fileStream; - if (!fileStream.Open(filePath.c_str(), IO::OpenMode::ModeRead | IO::OpenMode::ModeBinary)) - { - return nullptr; - } - - void* loadedObject = LoadObjectFromStream(fileStream, context, &targetClassId, filterDesc); - return loadedObject; - } - - bool SaveObjectToStream(IO::GenericStream& stream, DataStream::StreamType streamType, const void* classPtr, const Uuid& classId, SerializeContext* context, const SerializeContext::ClassData* classData) - { - AZ_PROFILE_FUNCTION(AzCore); - - if (!context) - { - EBUS_EVENT_RESULT(context, ComponentApplicationBus, GetSerializeContext); - - if(!context) - { - AZ_Assert(false, "No serialize context"); - return false; - } - } - - if (!classPtr) - { - AZ_Assert(false, "SaveObjectToStream: classPtr is null, object cannot be serialized."); - return false; - } - - AZ::ObjectStream* objectStream = AZ::ObjectStream::Create(&stream, *context, streamType); - if (!objectStream) - { - return false; - } - - if (!objectStream->WriteClass(classPtr, classId, classData)) - { - objectStream->Finalize(); - return false; - } - - if (!objectStream->Finalize()) - { - return false; - } - - return true; - } - - bool SaveStreamToFile(const AZStd::string& filePath, const AZStd::vector& streamData, int platformFlags) - { - AZ::IO::FileIOBase* fileIo = AZ::IO::FileIOBase::GetInstance(); - AZ::IO::FixedMaxPathString resolvedPath; - if (fileIo == nullptr || !fileIo->ResolvePath(filePath.c_str(), resolvedPath.data(), resolvedPath.capacity() + 1)) - { - resolvedPath = filePath; - } - if (AZ::IO::SystemFile fileHandle; fileHandle.Open(resolvedPath.c_str(), - AZ::IO::SystemFile::SF_OPEN_CREATE | AZ::IO::SystemFile::SF_OPEN_CREATE_PATH | AZ::IO::SystemFile::SF_OPEN_WRITE_ONLY, - platformFlags)) - { - AZ::IO::SizeType bytesWritten = fileHandle.Write(streamData.data(), streamData.size()); - return bytesWritten == streamData.size(); - } - return false; } - bool SaveObjectToFile(const AZStd::string& filePath, DataStream::StreamType fileType, const void* classPtr, const Uuid& classId, SerializeContext* context, int platformFlags) - { - AZ_PROFILE_FUNCTION(AzCore); + AZ_Assert(targetPointer, "You must provide a target pointer"); - // \note This is ok for tools, but we should use the streamer to write objects directly (no memory store) - AZStd::vector dstData; - AZ::IO::ByteContainerStream > dstByteStream(&dstData); - - if (!SaveObjectToStream(dstByteStream, fileType, classPtr, classId, context)) + bool foundSuccess = false; + using CreationCallback = AZStd::function; + auto handler = [&targetPointer, objectClassData, &foundSuccess](void** instance, const SerializeContext::ClassData** classData, const Uuid& classId, SerializeContext* context) { - return false; - } - - return SaveStreamToFile(filePath, dstData, platformFlags); - } - - /*! - \brief Finds any descendant DataElementNodes of the @classElement which match each Crc32 values in the supplied elementCrcQueue - \param context SerializeContext used for looking up ClassData - \param classElement Top level DataElementNode to begin comparison each the Crc32 queue - \param elementCrcQueue Container of Crc32 values in the order in which DataElementNodes should be matched as the DataElementNode tree is traversed - \return Vector of valid pointers to DataElementNodes which match the entire element Crc32 queue - */ - AZStd::vector FindDescendantElements(AZ::SerializeContext& context, AZ::SerializeContext::DataElementNode& classElement, - const AZStd::vector& elementCrcQueue) - { - AZStd::vector dataElementNodes; - FindDescendantElements(context, classElement, dataElementNodes, elementCrcQueue.begin(), elementCrcQueue.end()); - - return dataElementNodes; - } - - /*! - \brief Finds any descendant DataElementNodes of the @classElement which match each Crc32 values in the supplied elementCrcQueue - \param context SerializeContext used for looking up ClassData - \param classElement The current DataElementNode which will be compared against be to current top Crc32 value in the Crc32 queue - \param dataElementNodes[out] Array to populate with a DataElementNode which was found by matching all Crc32 values in the Crc32 queue - \param first The current front of the Crc32 queue - \param last The end of the Crc32 queue - */ - void FindDescendantElements(AZ::SerializeContext& context, AZ::SerializeContext::DataElementNode& classElement, - AZStd::vector& dataElementNodes, AZStd::vector::const_iterator first, AZStd::vector::const_iterator last) - { - if (first == last) - { - return; - } - - for (int i = 0; i < classElement.GetNumSubElements(); ++i) - { - auto& childElement = classElement.GetSubElement(i); - if (*first == AZ::Crc32(childElement.GetName())) + void* convertibleInstance{}; + if (objectClassData->ConvertFromType(convertibleInstance, classId, targetPointer, *context)) { - if (AZStd::distance(first, last) == 1) + foundSuccess = true; + if (instance) { - dataElementNodes.push_back(&childElement); + // The ObjectStream will ask us for the address of the target to load into, so provide it. + *instance = convertibleInstance; + } + if (classData) + { + // The ObjectStream will ask us for the class data of the target being loaded into, so provide it if needed. + // This allows us to load directly into a generic object (templated containers, strings, etc). + *classData = objectClassData; + } + } + }; + bool readSuccess = ObjectStream::LoadBlocking(&stream, *context, ObjectStream::ClassReadyCB(), filterDesc, CreationCallback(handler, AZ::OSStdAllocator())); + + AZ_Warning("Serialization", readSuccess, "LoadObjectFromStreamInPlace: Stream did not deserialize correctly"); + AZ_Warning("Serialization", foundSuccess, "LoadObjectFromStreamInPlace: Did not find the expected type in the stream"); + + return readSuccess && foundSuccess; + } + + bool LoadObjectFromStreamInPlace(IO::GenericStream& stream, AZ::SerializeContext* context, const Uuid& targetClassId, void* targetPointer, const FilterDescriptor& filterDesc) + { + AZ_PROFILE_FUNCTION(AzCore); + + if (!context) + { + EBUS_EVENT_RESULT(context, ComponentApplicationBus, GetSerializeContext); + AZ_Assert(context, "No serialize context"); + } + + if (!context) + { + return false; + } + + const SerializeContext::ClassData* classData = context->FindClassData(targetClassId); + if (!classData) + { + AZ_Error("Serialization", false, + "Unable to locate class data for uuid \"%s\". This object cannot be serialized as a root element. " + "Make sure the Uuid is valid, or if this is a generic type, use the override that takes a ClassData pointer instead.", + targetClassId.ToString().c_str()); + return false; + } + + return LoadObjectFromStreamInPlace(stream, context, classData, targetPointer, filterDesc); + } + + bool LoadObjectFromFileInPlace(const AZStd::string& filePath, const Uuid& targetClassId, void* destination, AZ::SerializeContext* context /*= nullptr*/, const FilterDescriptor& filterDesc /*= FilterDescriptor()*/) + { + AZ::IO::FileIOStream fileStream; + if (!fileStream.Open(filePath.c_str(), IO::OpenMode::ModeRead | IO::OpenMode::ModeBinary)) + { + return false; + } + + return LoadObjectFromStreamInPlace(fileStream, context, targetClassId, destination, filterDesc); + } + + void* LoadObjectFromStream(IO::GenericStream& stream, AZ::SerializeContext* context, const Uuid* targetClassId, const FilterDescriptor& filterDesc) + { + AZ_PROFILE_FUNCTION(AzCore); + + if (!context) + { + EBUS_EVENT_RESULT(context, ComponentApplicationBus, GetSerializeContext); + AZ_Assert(context, "No serialize context"); + } + + if (!context) + { + return nullptr; + } + + void* loadedInstance = nullptr; + bool success = ObjectStream::LoadBlocking(&stream, *context, + [&loadedInstance, targetClassId](void* classPtr, const Uuid& classId, const SerializeContext* serializeContext) + { + if (targetClassId) + { + void* instance = serializeContext->DownCast(classPtr, classId, *targetClassId); + + // Given a valid object + if (instance) + { + AZ_Assert(!loadedInstance, "loadedInstance must be NULL, otherwise we are being invoked with multiple valid objects"); + loadedInstance = instance; + return; + } } else { - FindDescendantElements(context, childElement, dataElementNodes, AZStd::next(first), last); - } - } - } - } - - bool IsVectorContainerType(const AZ::Uuid& type) - { - AZ::SerializeContext* serializeContext = nullptr; - AZ::ComponentApplicationBus::BroadcastResult(serializeContext, &AZ::ComponentApplicationRequests::GetSerializeContext); - - if (serializeContext) - { - AZ::TypeId containerTypeId = type; - - if (GenericClassInfo* classInfo = serializeContext->FindGenericClassInfo(type)) - { - // This type is a container - containerTypeId = classInfo->GetGenericTypeId(); - } - - if (containerTypeId == AZ::GetGenericClassInfoVectorTypeId() - || containerTypeId == AZ::GetGenericClassInfoFixedVectorTypeId() - || containerTypeId == AZ::GetGenericClassInfoArrayTypeId() - ) - { - return true; - } - } - - return false; - } - - bool IsSetContainerType(const AZ::Uuid& type) - { - AZ::SerializeContext* serializeContext = nullptr; - AZ::ComponentApplicationBus::BroadcastResult(serializeContext, &AZ::ComponentApplicationRequests::GetSerializeContext); - - if (serializeContext) - { - AZ::TypeId containerTypeId = type; - - if (GenericClassInfo* classInfo = serializeContext->FindGenericClassInfo(type)) - { - // This type is a container - containerTypeId = classInfo->GetGenericTypeId(); - } - - if (containerTypeId == AZ::GetGenericClassSetTypeId() - || containerTypeId == AZ::GetGenericClassUnorderedSetTypeId() - ) - { - return true; - } - } - - return false; - } - - - bool IsMapContainerType(const AZ::Uuid& type) - { - AZ::SerializeContext* serializeContext = nullptr; - AZ::ComponentApplicationBus::BroadcastResult(serializeContext, &AZ::ComponentApplicationRequests::GetSerializeContext); - - if (serializeContext) - { - AZ::Uuid containerTypeId = type; - - if (GenericClassInfo* classInfo = serializeContext->FindGenericClassInfo(type)) - { - containerTypeId = classInfo->GetGenericTypeId(); - } - - if (containerTypeId == AZ::GetGenericClassMapTypeId() - || containerTypeId == AZ::GetGenericClassUnorderedMapTypeId() - ) - { - return true; - } - } - - return false; - } - - bool IsContainerType(const AZ::Uuid& type) - { - return IsVectorContainerType(type) || IsSetContainerType(type) || IsMapContainerType(type); - } - - AZStd::vector GetContainedTypes(const AZ::Uuid& type) - { - AZStd::vector types; - - AZ::SerializeContext* serializeContext = nullptr; - AZ::ComponentApplicationBus::BroadcastResult(serializeContext, &AZ::ComponentApplicationRequests::GetSerializeContext); - - if (serializeContext) - { - if (GenericClassInfo* classInfo = serializeContext->FindGenericClassInfo(type)) - { - for (int i = 0; i < classInfo->GetNumTemplatedArguments(); ++i) - { - types.push_back(classInfo->GetTemplatedTypeId(i)); - } - } - } - - return types; - } - - bool IsOutcomeType(const AZ::Uuid& type) - { - AZ::SerializeContext* serializeContext = nullptr; - AZ::ComponentApplicationBus::BroadcastResult(serializeContext, &AZ::ComponentApplicationRequests::GetSerializeContext); - - if (serializeContext) - { - GenericClassInfo* classInfo = serializeContext->FindGenericClassInfo(type); - return classInfo && classInfo->GetGenericTypeId() == AZ::GetGenericOutcomeTypeId(); - } - - return false; - } - - bool IsPairContainerType(const AZ::Uuid& type) - { - AZ::SerializeContext* serializeContext = nullptr; - AZ::ComponentApplicationBus::BroadcastResult(serializeContext, &AZ::ComponentApplicationRequests::GetSerializeContext); - - if (serializeContext) - { - AZ::Uuid containerTypeId = type; - - if (GenericClassInfo* classInfo = serializeContext->FindGenericClassInfo(type)) - { - containerTypeId = classInfo->GetGenericTypeId(); - } - - if (containerTypeId == AZ::GetGenericClassPairTypeId()) - { - return true; - } - } - - return false; - } - - AZ::TypeId GetGenericContainerType(const AZ::TypeId& type) - { - AZ::SerializeContext* serializeContext = nullptr; - AZ::ComponentApplicationBus::BroadcastResult(serializeContext, &AZ::ComponentApplicationRequests::GetSerializeContext); - - if (serializeContext) - { - if (GenericClassInfo* classInfo = serializeContext->FindGenericClassInfo(type)) - { - // This type is a container - return classInfo->GetGenericTypeId(); - } - } - - return azrtti_typeid(); - } - - bool IsGenericContainerType(const AZ::TypeId& type) - { - return IsContainerType(type) && GetGenericContainerType(type) == azrtti_typeid(); - } - - AZStd::pair GetOutcomeTypes(const AZ::Uuid& type) - { - AZStd::vector types; - - AZ::SerializeContext* serializeContext = nullptr; - AZ::ComponentApplicationBus::BroadcastResult(serializeContext, &AZ::ComponentApplicationRequests::GetSerializeContext); - - if (serializeContext) - { - if (GenericClassInfo* classInfo = serializeContext->FindGenericClassInfo(type)) - { - AZ_Assert(classInfo->GetNumTemplatedArguments() == 2, "Outcome template arguments must be 2, even if void, void"); - return AZStd::make_pair(classInfo->GetTemplatedTypeId(0), classInfo->GetTemplatedTypeId(1)); - } - } - - return AZStd::make_pair(azrtti_typeid(), azrtti_typeid()); - } - - void* ResolvePointer(void* ptr, const SerializeContext::ClassElement& classElement, const SerializeContext& context) - { - if (classElement.m_flags & SerializeContext::ClassElement::FLG_POINTER) - { - // In the case of pointer-to-pointer, we'll deference. - ptr = *(void**)(ptr); - - // Pointer-to-pointer fields may be base class / polymorphic, so cast pointer to actual type, - // safe for passing as 'this' to member functions. - if (ptr && classElement.m_azRtti) - { - Uuid actualClassId = classElement.m_azRtti->GetActualUuid(ptr); - if (actualClassId != classElement.m_typeId) - { - const SerializeContext::ClassData* classData = context.FindClassData(actualClassId); - if (classData) + if (!loadedInstance) { - ptr = classElement.m_azRtti->Cast(ptr, classData->m_azRtti->GetTypeId()); + loadedInstance = classPtr; + return; } } - } - } - return ptr; + auto classData = serializeContext->FindClassData(classId); + if (classData && classData->m_factory) + { + classData->m_factory->Destroy(classPtr); + } + }, + filterDesc, + ObjectStream::InplaceLoadRootInfoCB() + ); + + if (!success) + { + return nullptr; } - } // namespace Utils -} // namespace AZ + return loadedInstance; + } + + void* LoadObjectFromFile(const AZStd::string& filePath, const Uuid& targetClassId, SerializeContext* context, const FilterDescriptor& filterDesc, int /*platformFlags*/) + { + AZ_PROFILE_FUNCTION(AzCore); + + AZ::IO::FileIOStream fileStream; + if (!fileStream.Open(filePath.c_str(), IO::OpenMode::ModeRead | IO::OpenMode::ModeBinary)) + { + return nullptr; + } + + void* loadedObject = LoadObjectFromStream(fileStream, context, &targetClassId, filterDesc); + return loadedObject; + } + + bool SaveObjectToStream(IO::GenericStream& stream, DataStream::StreamType streamType, const void* classPtr, const Uuid& classId, SerializeContext* context, const SerializeContext::ClassData* classData) + { + AZ_PROFILE_FUNCTION(AzCore); + + if (!context) + { + EBUS_EVENT_RESULT(context, ComponentApplicationBus, GetSerializeContext); + + if(!context) + { + AZ_Assert(false, "No serialize context"); + return false; + } + } + + if (!classPtr) + { + AZ_Assert(false, "SaveObjectToStream: classPtr is null, object cannot be serialized."); + return false; + } + + AZ::ObjectStream* objectStream = AZ::ObjectStream::Create(&stream, *context, streamType); + if (!objectStream) + { + return false; + } + + if (!objectStream->WriteClass(classPtr, classId, classData)) + { + objectStream->Finalize(); + return false; + } + + if (!objectStream->Finalize()) + { + return false; + } + + return true; + } + + bool SaveStreamToFile(const AZStd::string& filePath, const AZStd::vector& streamData, int platformFlags) + { + AZ::IO::FileIOBase* fileIo = AZ::IO::FileIOBase::GetInstance(); + AZ::IO::FixedMaxPathString resolvedPath; + if (fileIo == nullptr || !fileIo->ResolvePath(filePath.c_str(), resolvedPath.data(), resolvedPath.capacity() + 1)) + { + resolvedPath = filePath; + } + if (AZ::IO::SystemFile fileHandle; fileHandle.Open(resolvedPath.c_str(), + AZ::IO::SystemFile::SF_OPEN_CREATE | AZ::IO::SystemFile::SF_OPEN_CREATE_PATH | AZ::IO::SystemFile::SF_OPEN_WRITE_ONLY, + platformFlags)) + { + AZ::IO::SizeType bytesWritten = fileHandle.Write(streamData.data(), streamData.size()); + return bytesWritten == streamData.size(); + } + + return false; + } + + bool SaveObjectToFile(const AZStd::string& filePath, DataStream::StreamType fileType, const void* classPtr, const Uuid& classId, SerializeContext* context, int platformFlags) + { + AZ_PROFILE_FUNCTION(AzCore); + + // \note This is ok for tools, but we should use the streamer to write objects directly (no memory store) + AZStd::vector dstData; + AZ::IO::ByteContainerStream > dstByteStream(&dstData); + + if (!SaveObjectToStream(dstByteStream, fileType, classPtr, classId, context)) + { + return false; + } + + return SaveStreamToFile(filePath, dstData, platformFlags); + } + + /*! + \brief Finds any descendant DataElementNodes of the @classElement which match each Crc32 values in the supplied elementCrcQueue + \param context SerializeContext used for looking up ClassData + \param classElement Top level DataElementNode to begin comparison each the Crc32 queue + \param elementCrcQueue Container of Crc32 values in the order in which DataElementNodes should be matched as the DataElementNode tree is traversed + \return Vector of valid pointers to DataElementNodes which match the entire element Crc32 queue + */ + AZStd::vector FindDescendantElements(AZ::SerializeContext& context, AZ::SerializeContext::DataElementNode& classElement, + const AZStd::vector& elementCrcQueue) + { + AZStd::vector dataElementNodes; + FindDescendantElements(context, classElement, dataElementNodes, elementCrcQueue.begin(), elementCrcQueue.end()); + + return dataElementNodes; + } + + /*! + \brief Finds any descendant DataElementNodes of the @classElement which match each Crc32 values in the supplied elementCrcQueue + \param context SerializeContext used for looking up ClassData + \param classElement The current DataElementNode which will be compared against be to current top Crc32 value in the Crc32 queue + \param dataElementNodes[out] Array to populate with a DataElementNode which was found by matching all Crc32 values in the Crc32 queue + \param first The current front of the Crc32 queue + \param last The end of the Crc32 queue + */ + void FindDescendantElements(AZ::SerializeContext& context, AZ::SerializeContext::DataElementNode& classElement, + AZStd::vector& dataElementNodes, AZStd::vector::const_iterator first, AZStd::vector::const_iterator last) + { + if (first == last) + { + return; + } + + for (int i = 0; i < classElement.GetNumSubElements(); ++i) + { + auto& childElement = classElement.GetSubElement(i); + if (*first == AZ::Crc32(childElement.GetName())) + { + if (AZStd::distance(first, last) == 1) + { + dataElementNodes.push_back(&childElement); + } + else + { + FindDescendantElements(context, childElement, dataElementNodes, AZStd::next(first), last); + } + } + } + } + + bool IsVectorContainerType(const AZ::Uuid& type) + { + AZ::SerializeContext* serializeContext = nullptr; + AZ::ComponentApplicationBus::BroadcastResult(serializeContext, &AZ::ComponentApplicationRequests::GetSerializeContext); + + if (serializeContext) + { + AZ::TypeId containerTypeId = type; + + if (GenericClassInfo* classInfo = serializeContext->FindGenericClassInfo(type)) + { + // This type is a container + containerTypeId = classInfo->GetGenericTypeId(); + } + + if (containerTypeId == AZ::GetGenericClassInfoVectorTypeId() + || containerTypeId == AZ::GetGenericClassInfoFixedVectorTypeId() + || containerTypeId == AZ::GetGenericClassInfoArrayTypeId() + ) + { + return true; + } + } + + return false; + } + + bool IsSetContainerType(const AZ::Uuid& type) + { + AZ::SerializeContext* serializeContext = nullptr; + AZ::ComponentApplicationBus::BroadcastResult(serializeContext, &AZ::ComponentApplicationRequests::GetSerializeContext); + + if (serializeContext) + { + AZ::TypeId containerTypeId = type; + + if (GenericClassInfo* classInfo = serializeContext->FindGenericClassInfo(type)) + { + // This type is a container + containerTypeId = classInfo->GetGenericTypeId(); + } + + if (containerTypeId == AZ::GetGenericClassSetTypeId() + || containerTypeId == AZ::GetGenericClassUnorderedSetTypeId() + ) + { + return true; + } + } + + return false; + } + + + bool IsMapContainerType(const AZ::Uuid& type) + { + AZ::SerializeContext* serializeContext = nullptr; + AZ::ComponentApplicationBus::BroadcastResult(serializeContext, &AZ::ComponentApplicationRequests::GetSerializeContext); + + if (serializeContext) + { + AZ::Uuid containerTypeId = type; + + if (GenericClassInfo* classInfo = serializeContext->FindGenericClassInfo(type)) + { + containerTypeId = classInfo->GetGenericTypeId(); + } + + if (containerTypeId == AZ::GetGenericClassMapTypeId() + || containerTypeId == AZ::GetGenericClassUnorderedMapTypeId() + ) + { + return true; + } + } + + return false; + } + + bool IsContainerType(const AZ::Uuid& type) + { + return IsVectorContainerType(type) || IsSetContainerType(type) || IsMapContainerType(type); + } + + AZStd::vector GetContainedTypes(const AZ::Uuid& type) + { + AZStd::vector types; + + AZ::SerializeContext* serializeContext = nullptr; + AZ::ComponentApplicationBus::BroadcastResult(serializeContext, &AZ::ComponentApplicationRequests::GetSerializeContext); + + if (serializeContext) + { + if (GenericClassInfo* classInfo = serializeContext->FindGenericClassInfo(type)) + { + for (int i = 0; i < classInfo->GetNumTemplatedArguments(); ++i) + { + types.push_back(classInfo->GetTemplatedTypeId(i)); + } + } + } + + return types; + } + + bool IsOutcomeType(const AZ::Uuid& type) + { + AZ::SerializeContext* serializeContext = nullptr; + AZ::ComponentApplicationBus::BroadcastResult(serializeContext, &AZ::ComponentApplicationRequests::GetSerializeContext); + + if (serializeContext) + { + GenericClassInfo* classInfo = serializeContext->FindGenericClassInfo(type); + return classInfo && classInfo->GetGenericTypeId() == AZ::GetGenericOutcomeTypeId(); + } + + return false; + } + + bool IsPairContainerType(const AZ::Uuid& type) + { + AZ::SerializeContext* serializeContext = nullptr; + AZ::ComponentApplicationBus::BroadcastResult(serializeContext, &AZ::ComponentApplicationRequests::GetSerializeContext); + + if (serializeContext) + { + AZ::Uuid containerTypeId = type; + + if (GenericClassInfo* classInfo = serializeContext->FindGenericClassInfo(type)) + { + containerTypeId = classInfo->GetGenericTypeId(); + } + + if (containerTypeId == AZ::GetGenericClassPairTypeId()) + { + return true; + } + } + + return false; + } + + AZ::TypeId GetGenericContainerType(const AZ::TypeId& type) + { + AZ::SerializeContext* serializeContext = nullptr; + AZ::ComponentApplicationBus::BroadcastResult(serializeContext, &AZ::ComponentApplicationRequests::GetSerializeContext); + + if (serializeContext) + { + if (GenericClassInfo* classInfo = serializeContext->FindGenericClassInfo(type)) + { + // This type is a container + return classInfo->GetGenericTypeId(); + } + } + + return azrtti_typeid(); + } + + bool IsGenericContainerType(const AZ::TypeId& type) + { + return IsContainerType(type) && GetGenericContainerType(type) == azrtti_typeid(); + } + + AZStd::pair GetOutcomeTypes(const AZ::Uuid& type) + { + AZStd::vector types; + + AZ::SerializeContext* serializeContext = nullptr; + AZ::ComponentApplicationBus::BroadcastResult(serializeContext, &AZ::ComponentApplicationRequests::GetSerializeContext); + + if (serializeContext) + { + if (GenericClassInfo* classInfo = serializeContext->FindGenericClassInfo(type)) + { + AZ_Assert(classInfo->GetNumTemplatedArguments() == 2, "Outcome template arguments must be 2, even if void, void"); + return AZStd::make_pair(classInfo->GetTemplatedTypeId(0), classInfo->GetTemplatedTypeId(1)); + } + } + + return AZStd::make_pair(azrtti_typeid(), azrtti_typeid()); + } + + void* ResolvePointer(void* ptr, const SerializeContext::ClassElement& classElement, const SerializeContext& context) + { + if (classElement.m_flags & SerializeContext::ClassElement::FLG_POINTER) + { + // In the case of pointer-to-pointer, we'll deference. + ptr = *(void**)(ptr); + + // Pointer-to-pointer fields may be base class / polymorphic, so cast pointer to actual type, + // safe for passing as 'this' to member functions. + if (ptr && classElement.m_azRtti) + { + Uuid actualClassId = classElement.m_azRtti->GetActualUuid(ptr); + if (actualClassId != classElement.m_typeId) + { + const SerializeContext::ClassData* classData = context.FindClassData(actualClassId); + if (classData) + { + ptr = classElement.m_azRtti->Cast(ptr, classData->m_azRtti->GetTypeId()); + } + } + } + } + + return ptr; + } + +} // namespace AZ::Utils diff --git a/Code/Framework/AzCore/AzCore/Serialization/SerializeContext.cpp b/Code/Framework/AzCore/AzCore/Serialization/SerializeContext.cpp index 81546f4f28..ff0ad571f7 100644 --- a/Code/Framework/AzCore/AzCore/Serialization/SerializeContext.cpp +++ b/Code/Framework/AzCore/AzCore/Serialization/SerializeContext.cpp @@ -2065,11 +2065,15 @@ namespace AZ } EnumerateInstanceCallContext callContext( - AZStd::bind(&SerializeContext::BeginCloneElement, this, AZStd::placeholders::_1, AZStd::placeholders::_2, AZStd::placeholders::_3, &cloneData, &m_errorLogger, &scratchBuffer), - AZStd::bind(&SerializeContext::EndCloneElement, this, &cloneData), - this, - SerializeContext::ENUM_ACCESS_FOR_READ, - &m_errorLogger); + [&](void* ptr, const ClassData* classData, const ClassElement* elementData) -> bool + { + return BeginCloneElement(ptr, classData, elementData, &cloneData, &m_errorLogger, &scratchBuffer); + }, + [&]() -> bool + { + return EndCloneElement(&cloneData); + }, + this, SerializeContext::ENUM_ACCESS_FOR_READ, &m_errorLogger); EnumerateInstance( &callContext @@ -2098,19 +2102,17 @@ namespace AZ if (ptr) { EnumerateInstanceCallContext callContext( - AZStd::bind(&SerializeContext::BeginCloneElementInplace, this, dest, AZStd::placeholders::_1, AZStd::placeholders::_2, AZStd::placeholders::_3, &cloneData, &m_errorLogger, &scratchBuffer), - AZStd::bind(&SerializeContext::EndCloneElement, this, &cloneData), - this, - SerializeContext::ENUM_ACCESS_FOR_READ, - &m_errorLogger); + [&](void* ptr, const ClassData* classData, const ClassElement* elementData) -> bool + { + return BeginCloneElementInplace(dest, ptr, classData, elementData, &cloneData, &m_errorLogger, &scratchBuffer); + }, + [&]() -> bool + { + return EndCloneElement(&cloneData); + }, + this, SerializeContext::ENUM_ACCESS_FOR_READ, &m_errorLogger); - EnumerateInstance( - &callContext - , const_cast(ptr) - , classId - , nullptr - , nullptr - ); + EnumerateInstance(&callContext, const_cast(ptr), classId, nullptr, nullptr); } } @@ -2941,14 +2943,10 @@ namespace AZ { m_errorHandler = errorHandler ? errorHandler : &m_defaultErrorHandler; - m_elementCallback = AZStd::bind(static_cast(&SerializeContext::EnumerateInstance) - , m_context - , this - , AZStd::placeholders::_1 - , AZStd::placeholders::_2 - , AZStd::placeholders::_3 - , AZStd::placeholders::_4 - ); + m_elementCallback = [this](void* ptr, const Uuid& classId, const ClassData* classData, const ClassElement* classElement)->bool + { + return m_context->EnumerateInstance(this, ptr, classId, classData, classElement); + }; } //========================================================================= diff --git a/Code/Framework/AzCore/AzCore/Settings/SettingsRegistryImpl.cpp b/Code/Framework/AzCore/AzCore/Settings/SettingsRegistryImpl.cpp index cbf25c29d4..43c8a64b93 100644 --- a/Code/Framework/AzCore/AzCore/Settings/SettingsRegistryImpl.cpp +++ b/Code/Framework/AzCore/AzCore/Settings/SettingsRegistryImpl.cpp @@ -327,7 +327,7 @@ namespace AZ while (!localNotifierQueue.empty()) { - for (SignalNotifierArgs notifierArgs : localNotifierQueue) + for (const SignalNotifierArgs& notifierArgs : localNotifierQueue) { localNotifierEvent.Signal(notifierArgs.m_jsonPath, notifierArgs.m_type); } diff --git a/Code/Framework/AzCore/AzCore/Slice/SliceComponent.cpp b/Code/Framework/AzCore/AzCore/Slice/SliceComponent.cpp index 0b410369f0..ca89e95162 100644 --- a/Code/Framework/AzCore/AzCore/Slice/SliceComponent.cpp +++ b/Code/Framework/AzCore/AzCore/Slice/SliceComponent.cpp @@ -840,7 +840,7 @@ namespace AZ } SliceComponent::SliceInstance* SliceComponent::SliceReference::CreateInstanceFromExistingEntities(AZStd::vector& entities, - const EntityIdToEntityIdMap assetToLiveIdMap, + const EntityIdToEntityIdMap& assetToLiveIdMap, SliceInstanceId sliceInstanceId) { AZ_PROFILE_FUNCTION(AzCore); diff --git a/Code/Framework/AzCore/AzCore/Slice/SliceComponent.h b/Code/Framework/AzCore/AzCore/Slice/SliceComponent.h index 7a66167a96..ae4ec155c7 100644 --- a/Code/Framework/AzCore/AzCore/Slice/SliceComponent.h +++ b/Code/Framework/AzCore/AzCore/Slice/SliceComponent.h @@ -443,7 +443,7 @@ namespace AZ * @return A pointer to the newly created slice instance. Returns nullptr on error or if the SliceComponent is not instantiated. */ SliceInstance* CreateInstanceFromExistingEntities(AZStd::vector& entities, - const EntityIdToEntityIdMap assetToLiveIdMap, + const EntityIdToEntityIdMap& assetToLiveIdMap, SliceInstanceId sliceInstanceId = SliceInstanceId::CreateRandom()); /** diff --git a/Code/Framework/AzCore/AzCore/Socket/AzSocket.cpp b/Code/Framework/AzCore/AzCore/Socket/AzSocket.cpp index f15c288ad8..71c0aeea6d 100644 --- a/Code/Framework/AzCore/AzCore/Socket/AzSocket.cpp +++ b/Code/Framework/AzCore/AzCore/Socket/AzSocket.cpp @@ -8,77 +8,74 @@ #include -namespace AZ +namespace AZ::AzSock { - namespace AzSock + AzSocketAddress::AzSocketAddress() { - AzSocketAddress::AzSocketAddress() - { - Reset(); - } - - AzSocketAddress& AzSocketAddress::operator=(const AZSOCKADDR& addr) - { - m_sockAddr = *reinterpret_cast(&addr); - return *this; - } - - bool AzSocketAddress::operator==(const AzSocketAddress& rhs) const - { - return m_sockAddr.sin_family == rhs.m_sockAddr.sin_family - && m_sockAddr.sin_addr.s_addr == rhs.m_sockAddr.sin_addr.s_addr - && m_sockAddr.sin_port == rhs.m_sockAddr.sin_port; - } - - const AZSOCKADDR* AzSocketAddress::GetTargetAddress() const - { - return reinterpret_cast(&m_sockAddr); - } - - AZStd::string AzSocketAddress::GetIP() const - { - char ip[INET_ADDRSTRLEN]; - inet_ntop(AF_INET, const_cast(&m_sockAddr.sin_addr), ip, AZ_ARRAY_SIZE(ip)); - return AZStd::string(ip); - } - - AZStd::string AzSocketAddress::GetAddress() const - { - char ip[INET_ADDRSTRLEN]; - inet_ntop(AF_INET, const_cast(&m_sockAddr.sin_addr), ip, AZ_ARRAY_SIZE(ip)); - return AZStd::string::format("%s:%d", ip, AZ::AzSock::NetToHostShort(m_sockAddr.sin_port)); - } - - AZ::u16 AzSocketAddress::GetAddrPort() const - { - return AZ::AzSock::NetToHostShort(m_sockAddr.sin_port); - } - - void AzSocketAddress::SetAddrPort(AZ::u16 port) - { - m_sockAddr.sin_port = AZ::AzSock::HostToNetShort(port); - } - - bool AzSocketAddress::SetAddress(const AZStd::string& ip, AZ::u16 port) - { - AZ_Assert(!ip.empty(), "Invalid address string!"); - Reset(); - return AZ::AzSock::ResolveAddress(ip, port, m_sockAddr); - } - - bool AzSocketAddress::SetAddress(AZ::u32 ip, AZ::u16 port) - { - Reset(); - m_sockAddr.sin_addr.s_addr = AZ::AzSock::HostToNetLong(ip); - m_sockAddr.sin_port = AZ::AzSock::HostToNetShort(port); - return true; - } - - void AzSocketAddress::Reset() - { - memset(&m_sockAddr, 0, sizeof(m_sockAddr)); - m_sockAddr.sin_family = AF_INET; - m_sockAddr.sin_addr.s_addr = INADDR_ANY; - } + Reset(); } -} + + AzSocketAddress& AzSocketAddress::operator=(const AZSOCKADDR& addr) + { + m_sockAddr = *reinterpret_cast(&addr); + return *this; + } + + bool AzSocketAddress::operator==(const AzSocketAddress& rhs) const + { + return m_sockAddr.sin_family == rhs.m_sockAddr.sin_family + && m_sockAddr.sin_addr.s_addr == rhs.m_sockAddr.sin_addr.s_addr + && m_sockAddr.sin_port == rhs.m_sockAddr.sin_port; + } + + const AZSOCKADDR* AzSocketAddress::GetTargetAddress() const + { + return reinterpret_cast(&m_sockAddr); + } + + AZStd::string AzSocketAddress::GetIP() const + { + char ip[INET_ADDRSTRLEN]; + inet_ntop(AF_INET, const_cast(&m_sockAddr.sin_addr), ip, AZ_ARRAY_SIZE(ip)); + return AZStd::string(ip); + } + + AZStd::string AzSocketAddress::GetAddress() const + { + char ip[INET_ADDRSTRLEN]; + inet_ntop(AF_INET, const_cast(&m_sockAddr.sin_addr), ip, AZ_ARRAY_SIZE(ip)); + return AZStd::string::format("%s:%d", ip, AZ::AzSock::NetToHostShort(m_sockAddr.sin_port)); + } + + AZ::u16 AzSocketAddress::GetAddrPort() const + { + return AZ::AzSock::NetToHostShort(m_sockAddr.sin_port); + } + + void AzSocketAddress::SetAddrPort(AZ::u16 port) + { + m_sockAddr.sin_port = AZ::AzSock::HostToNetShort(port); + } + + bool AzSocketAddress::SetAddress(const AZStd::string& ip, AZ::u16 port) + { + AZ_Assert(!ip.empty(), "Invalid address string!"); + Reset(); + return AZ::AzSock::ResolveAddress(ip, port, m_sockAddr); + } + + bool AzSocketAddress::SetAddress(AZ::u32 ip, AZ::u16 port) + { + Reset(); + m_sockAddr.sin_addr.s_addr = AZ::AzSock::HostToNetLong(ip); + m_sockAddr.sin_port = AZ::AzSock::HostToNetShort(port); + return true; + } + + void AzSocketAddress::Reset() + { + memset(&m_sockAddr, 0, sizeof(m_sockAddr)); + m_sockAddr.sin_family = AF_INET; + m_sockAddr.sin_addr.s_addr = INADDR_ANY; + } +} // namespace AZ::AzSock diff --git a/Code/Framework/AzCore/AzCore/Statistics/RunningStatistic.cpp b/Code/Framework/AzCore/AzCore/Statistics/RunningStatistic.cpp index 47f85c8309..3093d38a10 100644 --- a/Code/Framework/AzCore/AzCore/Statistics/RunningStatistic.cpp +++ b/Code/Framework/AzCore/AzCore/Statistics/RunningStatistic.cpp @@ -10,66 +10,63 @@ #include "RunningStatistic.h" -namespace AZ +namespace AZ::Statistics { - namespace Statistics + void RunningStatistic::Reset() { - void RunningStatistic::Reset() + m_numSamples = 0; + m_mostRecentSample = 0.0; + m_minimum = 0.0; + m_maximum = 0.0; + m_sum = 0.0; + m_average = 0.0; + m_varianceTracking = 0.0; + } + + void RunningStatistic::PushSample(double value) + { + m_numSamples++; + m_mostRecentSample = value; + m_sum += value; + + if (m_numSamples == 1) { - m_numSamples = 0; - m_mostRecentSample = 0.0; - m_minimum = 0.0; - m_maximum = 0.0; - m_sum = 0.0; - m_average = 0.0; - m_varianceTracking = 0.0; + m_minimum = value; + m_maximum = value; + m_average = value; + return; } - void RunningStatistic::PushSample(double value) + if (value < m_minimum) { - m_numSamples++; - m_mostRecentSample = value; - m_sum += value; - - if (m_numSamples == 1) - { - m_minimum = value; - m_maximum = value; - m_average = value; - return; - } - - if (value < m_minimum) - { - m_minimum = value; - } - else if (value > m_maximum) - { - m_maximum = value; - } - - //See header notes and references to understand this way of calculating - //running average & variance. - const double newAverage = m_average + (value - m_average) / m_numSamples; - m_varianceTracking = m_varianceTracking + (value - m_average)*(value - newAverage); - - m_average = newAverage; + m_minimum = value; + } + else if (value > m_maximum) + { + m_maximum = value; } - double RunningStatistic::GetVariance(VarianceType varianceType) const - { - if (m_numSamples > 1) - { - const AZ::u64 varianceDivisor = (varianceType == VarianceType::S) ? m_numSamples - 1 : m_numSamples; - return m_varianceTracking / varianceDivisor; - } - return 0.0; - } + //See header notes and references to understand this way of calculating + //running average & variance. + const double newAverage = m_average + (value - m_average) / m_numSamples; + m_varianceTracking = m_varianceTracking + (value - m_average)*(value - newAverage); - double RunningStatistic::GetStdev(VarianceType varianceType) const + m_average = newAverage; + } + + double RunningStatistic::GetVariance(VarianceType varianceType) const + { + if (m_numSamples > 1) { - return sqrt(GetVariance(varianceType)); + const AZ::u64 varianceDivisor = (varianceType == VarianceType::S) ? m_numSamples - 1 : m_numSamples; + return m_varianceTracking / varianceDivisor; } - - }//namespace Statistics -}//namespace AZ + return 0.0; + } + + double RunningStatistic::GetStdev(VarianceType varianceType) const + { + return sqrt(GetVariance(varianceType)); + } + +} // namespace AZ::Statistics diff --git a/Code/Framework/AzCore/AzCore/Statistics/StatisticalProfilerProxySystemComponent.cpp b/Code/Framework/AzCore/AzCore/Statistics/StatisticalProfilerProxySystemComponent.cpp index 00bb97b745..ef87307624 100644 --- a/Code/Framework/AzCore/AzCore/Statistics/StatisticalProfilerProxySystemComponent.cpp +++ b/Code/Framework/AzCore/AzCore/Statistics/StatisticalProfilerProxySystemComponent.cpp @@ -12,55 +12,52 @@ #include "StatisticalProfilerProxySystemComponent.h" //////////////////////////////////////////////////////////////////////////////////////////////////// -namespace AZ +namespace AZ::Statistics { - namespace Statistics + StatisticalProfilerProxy* StatisticalProfilerProxy::TimedScope::m_profilerProxy = nullptr; + + //////////////////////////////////////////////////////////////////////////////////////////////// + void StatisticalProfilerProxySystemComponent::Reflect(AZ::ReflectContext* context) { - StatisticalProfilerProxy* StatisticalProfilerProxy::TimedScope::m_profilerProxy = nullptr; - - //////////////////////////////////////////////////////////////////////////////////////////////// - void StatisticalProfilerProxySystemComponent::Reflect(AZ::ReflectContext* context) + if (AZ::SerializeContext* serializeContext = azrtti_cast(context)) { - if (AZ::SerializeContext* serializeContext = azrtti_cast(context)) - { - serializeContext->Class() - ->Version(1); - } + serializeContext->Class() + ->Version(1); } + } - //////////////////////////////////////////////////////////////////////////////////////////////// - void StatisticalProfilerProxySystemComponent::GetProvidedServices(AZ::ComponentDescriptor::DependencyArrayType& provided) - { - provided.push_back(AZ_CRC("StatisticalProfilerService", 0x20066f73)); - } + //////////////////////////////////////////////////////////////////////////////////////////////// + void StatisticalProfilerProxySystemComponent::GetProvidedServices(AZ::ComponentDescriptor::DependencyArrayType& provided) + { + provided.push_back(AZ_CRC("StatisticalProfilerService", 0x20066f73)); + } - //////////////////////////////////////////////////////////////////////////////////////////////// - void StatisticalProfilerProxySystemComponent::GetIncompatibleServices(AZ::ComponentDescriptor::DependencyArrayType& incompatible) - { - incompatible.push_back(AZ_CRC("StatisticalProfilerService", 0x20066f73)); - } + //////////////////////////////////////////////////////////////////////////////////////////////// + void StatisticalProfilerProxySystemComponent::GetIncompatibleServices(AZ::ComponentDescriptor::DependencyArrayType& incompatible) + { + incompatible.push_back(AZ_CRC("StatisticalProfilerService", 0x20066f73)); + } - //////////////////////////////////////////////////////////////////////////////////////////////// - StatisticalProfilerProxySystemComponent::StatisticalProfilerProxySystemComponent() - : m_StatisticalProfilerProxy(nullptr) - { - } + //////////////////////////////////////////////////////////////////////////////////////////////// + StatisticalProfilerProxySystemComponent::StatisticalProfilerProxySystemComponent() + : m_StatisticalProfilerProxy(nullptr) + { + } - //////////////////////////////////////////////////////////////////////////////////////////////// - StatisticalProfilerProxySystemComponent::~StatisticalProfilerProxySystemComponent() - { - } + //////////////////////////////////////////////////////////////////////////////////////////////// + StatisticalProfilerProxySystemComponent::~StatisticalProfilerProxySystemComponent() + { + } - //////////////////////////////////////////////////////////////////////////////////////////////// - void StatisticalProfilerProxySystemComponent::Activate() - { - m_StatisticalProfilerProxy = new StatisticalProfilerProxy; - } + //////////////////////////////////////////////////////////////////////////////////////////////// + void StatisticalProfilerProxySystemComponent::Activate() + { + m_StatisticalProfilerProxy = new StatisticalProfilerProxy; + } - //////////////////////////////////////////////////////////////////////////////////////////////// - void StatisticalProfilerProxySystemComponent::Deactivate() - { - delete m_StatisticalProfilerProxy; - } - } //namespace Statistics -} // namespace AZ + //////////////////////////////////////////////////////////////////////////////////////////////// + void StatisticalProfilerProxySystemComponent::Deactivate() + { + delete m_StatisticalProfilerProxy; + } +} // namespace AZ::Statistics diff --git a/Code/Framework/AzCore/AzCore/StringFunc/StringFunc.cpp b/Code/Framework/AzCore/AzCore/StringFunc/StringFunc.cpp index a953d53c33..8405424f7d 100644 --- a/Code/Framework/AzCore/AzCore/StringFunc/StringFunc.cpp +++ b/Code/Framework/AzCore/AzCore/StringFunc/StringFunc.cpp @@ -416,2157 +416,2155 @@ namespace AZ::StringFunc::Internal } -namespace AZ +namespace AZ::StringFunc { - namespace StringFunc + AZStd::string_view LStrip(AZStd::string_view in, AZStd::string_view stripCharacters) { - AZStd::string_view LStrip(AZStd::string_view in, AZStd::string_view stripCharacters) + if (size_t pos = in.find_first_not_of(stripCharacters); pos != AZStd::string_view::npos) { - if (size_t pos = in.find_first_not_of(stripCharacters); pos != AZStd::string_view::npos) - { - return in.substr(pos); - } + return in.substr(pos); + } - return {}; - }; + return {}; + }; - AZStd::string_view RStrip(AZStd::string_view in, AZStd::string_view stripCharacters) + AZStd::string_view RStrip(AZStd::string_view in, AZStd::string_view stripCharacters) + { + if (size_t pos = in.find_last_not_of(stripCharacters); pos != AZStd::string_view::npos) { - if (size_t pos = in.find_last_not_of(stripCharacters); pos != AZStd::string_view::npos) - { - return in.substr(0, pos < in.size() ? pos + 1 : pos); - } + return in.substr(0, pos < in.size() ? pos + 1 : pos); + } - return {}; - }; + return {}; + }; - AZStd::string_view StripEnds(AZStd::string_view in, AZStd::string_view stripCharacters) + AZStd::string_view StripEnds(AZStd::string_view in, AZStd::string_view stripCharacters) + { + return LStrip(RStrip(in, stripCharacters), stripCharacters); + }; + + bool Equal(const char* inA, const char* inB, bool bCaseSensitive /*= false*/, size_t n /*= 0*/) + { + if (!inA || !inB) { - return LStrip(RStrip(in, stripCharacters), stripCharacters); - }; + return false; + } - bool Equal(const char* inA, const char* inB, bool bCaseSensitive /*= false*/, size_t n /*= 0*/) + if (inA == inB) { - if (!inA || !inB) - { - return false; - } + return true; + } - if (inA == inB) + if (bCaseSensitive) + { + if (n) { - return true; - } - - if (bCaseSensitive) - { - if (n) - { - return !strncmp(inA, inB, n); - } - else - { - return !strcmp(inA, inB); - } + return !strncmp(inA, inB, n); } else { - if (n) - { - return !azstrnicmp(inA, inB, n); - } - else - { - return !azstricmp(inA, inB); - } + return !strcmp(inA, inB); } } - bool Equal(AZStd::string_view inA, AZStd::string_view inB, bool bCaseSensitive) + else { - const size_t maxCharsToCompare = inA.size(); - - return inA.size() == inB.size() && (bCaseSensitive - ? strncmp(inA.data(), inB.data(), maxCharsToCompare) == 0 - : azstrnicmp(inA.data(), inB.data(), maxCharsToCompare) == 0); - } - - bool StartsWith(AZStd::string_view sourceValue, AZStd::string_view prefixValue, bool bCaseSensitive) - { - return sourceValue.size() >= prefixValue.size() - && Equal(sourceValue.data(), prefixValue.data(), bCaseSensitive, prefixValue.size()); - } - - bool EndsWith(AZStd::string_view sourceValue, AZStd::string_view suffixValue, bool bCaseSensitive) - { - return sourceValue.size() >= suffixValue.size() - && Equal(sourceValue.substr(sourceValue.size() - suffixValue.size(), AZStd::string_view::npos).data(), suffixValue.data(), bCaseSensitive, suffixValue.size()); - } - - bool Contains(AZStd::string_view in, char ch, bool bCaseSensitive) - { - return Find(in, ch, 0, false, bCaseSensitive) != AZStd::string_view::npos; - } - bool Contains(AZStd::string_view in, AZStd::string_view sv, bool bCaseSensitive) - { - return Find(in, sv, 0, false, bCaseSensitive) != AZStd::string_view::npos; - } - - size_t Find(AZStd::string_view in, char c, size_t pos /*= 0*/, bool bReverse /*= false*/, bool bCaseSensitive /*= false*/) - { - if (in.empty()) + if (n) { - return AZStd::string::npos; + return !azstrnicmp(inA, inB, n); } - - if (pos == AZStd::string::npos) + else { - pos = 0; + return !azstricmp(inA, inB); } + } + } + bool Equal(AZStd::string_view inA, AZStd::string_view inB, bool bCaseSensitive) + { + const size_t maxCharsToCompare = inA.size(); - size_t inLen = in.size(); - if (inLen < pos) - { - return AZStd::string::npos; - } + return inA.size() == inB.size() && (bCaseSensitive + ? strncmp(inA.data(), inB.data(), maxCharsToCompare) == 0 + : azstrnicmp(inA.data(), inB.data(), maxCharsToCompare) == 0); + } + bool StartsWith(AZStd::string_view sourceValue, AZStd::string_view prefixValue, bool bCaseSensitive) + { + return sourceValue.size() >= prefixValue.size() + && Equal(sourceValue.data(), prefixValue.data(), bCaseSensitive, prefixValue.size()); + } + + bool EndsWith(AZStd::string_view sourceValue, AZStd::string_view suffixValue, bool bCaseSensitive) + { + return sourceValue.size() >= suffixValue.size() + && Equal(sourceValue.substr(sourceValue.size() - suffixValue.size(), AZStd::string_view::npos).data(), suffixValue.data(), bCaseSensitive, suffixValue.size()); + } + + bool Contains(AZStd::string_view in, char ch, bool bCaseSensitive) + { + return Find(in, ch, 0, false, bCaseSensitive) != AZStd::string_view::npos; + } + bool Contains(AZStd::string_view in, AZStd::string_view sv, bool bCaseSensitive) + { + return Find(in, sv, 0, false, bCaseSensitive) != AZStd::string_view::npos; + } + + size_t Find(AZStd::string_view in, char c, size_t pos /*= 0*/, bool bReverse /*= false*/, bool bCaseSensitive /*= false*/) + { + if (in.empty()) + { + return AZStd::string::npos; + } + + if (pos == AZStd::string::npos) + { + pos = 0; + } + + size_t inLen = in.size(); + if (inLen < pos) + { + return AZStd::string::npos; + } + + if (!bCaseSensitive) + { + c = (char)tolower(c); + } + + if (bReverse) + { + pos = inLen - pos - 1; + } + + char character; + + do + { if (!bCaseSensitive) { - c = (char)tolower(c); + character = (char)tolower(in[pos]); + } + else + { + character = in[pos]; + } + + if (character == c) + { + return pos; } if (bReverse) { - pos = inLen - pos - 1; + pos = pos > 0 ? pos-1 : pos; } - - char character; - - do + else { - if (!bCaseSensitive) - { - character = (char)tolower(in[pos]); - } - else - { - character = in[pos]; - } + pos++; + } + } while (bReverse ? pos : character != '\0'); - if (character == c) - { - return pos; - } + return AZStd::string::npos; + } - if (bReverse) - { - pos = pos > 0 ? pos-1 : pos; - } - else - { - pos++; - } - } while (bReverse ? pos : character != '\0'); + size_t Find(AZStd::string_view in, AZStd::string_view s, size_t offset /*= 0*/, bool bReverse /*= false*/, bool bCaseSensitive /*= false*/) + { + // Formally an empty string matches at the offset if it is <= to the size of the input string + if (s.empty() && offset <= in.size()) + { + return offset; + } + if (in.empty()) + { return AZStd::string::npos; } - size_t Find(AZStd::string_view in, AZStd::string_view s, size_t offset /*= 0*/, bool bReverse /*= false*/, bool bCaseSensitive /*= false*/) + const size_t inlen = in.size(); + const size_t slen = s.size(); + + if (offset == AZStd::string::npos) { - // Formally an empty string matches at the offset if it is <= to the size of the input string - if (s.empty() && offset <= in.size()) + offset = 0; + } + + if (offset + slen > inlen) + { + return AZStd::string::npos; + } + + const char* pCur; + + if (bReverse) + { + // Start at the end (- pos) + pCur = in.data() + inlen - slen - offset; + } + else + { + // Start at the beginning (+ pos) + pCur = in.data() + offset; + } + + do + { + if (bCaseSensitive) { - return offset; + if (!strncmp(pCur, s.data(), slen)) + { + return static_cast(pCur - in.data()); + } } - - if (in.empty()) + else { - return AZStd::string::npos; + if (!azstrnicmp(pCur, s.data(), slen)) + { + return static_cast(pCur - in.data()); + } } - const size_t inlen = in.size(); - const size_t slen = s.size(); - - if (offset == AZStd::string::npos) - { - offset = 0; - } - - if (offset + slen > inlen) - { - return AZStd::string::npos; - } - - const char* pCur; - if (bReverse) { - // Start at the end (- pos) - pCur = in.data() + inlen - slen - offset; + pCur--; } else { - // Start at the beginning (+ pos) - pCur = in.data() + offset; + pCur++; + } + } while (bReverse ? pCur >= in.data() : pCur - in.data() <= static_cast(inlen)); + + return AZStd::string::npos; + } + + char FirstCharacter(const char* in) + { + if (!in) + { + return '\0'; + } + if (in[0] == '\n') + { + return '\0'; + } + return in[0]; + } + + char LastCharacter(const char* in) + { + if (!in) + { + return '\0'; + } + size_t len = strlen(in); + if (!len) + { + return '\0'; + } + return in[len - 1]; + } + + AZStd::string& Append(AZStd::string& inout, const char s) + { + return inout.append(1, s); + } + + AZStd::string& Append(AZStd::string& inout, const char* str) + { + if (!str) + { + return inout; + } + return inout.append(str); + } + + AZStd::string& Prepend(AZStd::string& inout, const char s) + { + return inout.insert((size_t)0, 1, s); + } + + AZStd::string& Prepend(AZStd::string& inout, const char* str) + { + if (!str) + { + return inout; + } + return inout.insert(0, str); + } + + AZStd::string& LChop(AZStd::string& inout, size_t num) + { + return Internal::LChop(inout, num); + } + + AZStd::string_view LChop(AZStd::string_view in, size_t num) + { + return Internal::LChop(in, num); + } + + AZStd::string& RChop(AZStd::string& inout, size_t num) + { + return Internal::RChop(inout, num); + } + + AZStd::string_view RChop(AZStd::string_view in, size_t num) + { + return Internal::RChop(in, num); + } + + AZStd::string& LKeep(AZStd::string& inout, size_t pos, bool bKeepPosCharacter) + { + return Internal::LKeep(inout, pos, bKeepPosCharacter); + } + + AZStd::string& RKeep(AZStd::string& inout, size_t pos, bool bKeepPosCharacter) + { + return Internal::RKeep(inout, pos, bKeepPosCharacter); + } + + bool Replace(AZStd::string& inout, const char replaceA, const char withB, bool bCaseSensitive, bool bReplaceFirst, bool bReplaceLast) + { + return Internal::Replace(inout, replaceA, withB, bCaseSensitive, bReplaceFirst, bReplaceLast); + } + + bool Replace(AZStd::string& inout, const char* replaceA, const char* withB, bool bCaseSensitive, bool bReplaceFirst, bool bReplaceLast) + { + return Internal::Replace(inout, replaceA, withB, bCaseSensitive, bReplaceFirst, bReplaceLast); + } + + bool Strip(AZStd::string& inout, const char stripCharacter, bool bCaseSensitive, bool bStripBeginning, bool bStripEnding) + { + return Internal::Strip(inout, stripCharacter, bCaseSensitive, bStripBeginning, bStripEnding); + } + + bool Strip(AZStd::string& inout, const char* stripCharacters, bool bCaseSensitive, bool bStripBeginning, bool bStripEnding) + { + return Internal::Strip(inout, stripCharacters, bCaseSensitive, bStripBeginning, bStripEnding); + } + + AZStd::string& TrimWhiteSpace(AZStd::string& value, bool leading, bool trailing) + { + static const char* trimmable = " \t\r\n"; + if (value.length() > 0) + { + if (leading) + { + value.erase(0, value.find_first_not_of(trimmable)); + } + if (trailing) + { + value.erase(value.find_last_not_of(trimmable) + 1); + } + } + return value; + } + + void Tokenize(AZStd::string_view in, AZStd::vector& tokens, const char delimiter, bool keepEmptyStrings, bool keepSpaceStrings) + { + return Tokenize(in, tokens, { &delimiter, 1 }, keepEmptyStrings, keepSpaceStrings); + } + + void Tokenize(AZStd::string_view in, AZStd::vector& tokens, AZStd::string_view delimiters, bool keepEmptyStrings, bool keepSpaceStrings) + { + auto insertVisitor = [&tokens](AZStd::string_view token) + { + tokens.push_back(token); + }; + return TokenizeVisitor(in, insertVisitor, delimiters, keepEmptyStrings, keepSpaceStrings); + } + + void TokenizeVisitor(AZStd::string_view in, const TokenVisitor& tokenVisitor, const char delimiter, bool keepEmptyStrings, bool keepSpaceStrings) + { + return TokenizeVisitor(in, tokenVisitor, { &delimiter, 1 }, keepEmptyStrings, keepSpaceStrings); + } + + void TokenizeVisitor(AZStd::string_view in, const TokenVisitor& tokenVisitor, AZStd::string_view delimiters, + bool keepEmptyStrings, bool keepSpaceStrings) + { + if (delimiters.empty() || in.empty()) + { + return; + } + + while (AZStd::optional nextToken = TokenizeNext(in, delimiters)) + { + bool bIsEmpty = nextToken->empty(); + bool bIsSpaces = false; + if (!bIsEmpty) + { + AZStd::string_view strippedNextToken = StripEnds(*nextToken, " "); + bIsSpaces = strippedNextToken.empty(); } - do + if ((bIsEmpty && keepEmptyStrings) || + (bIsSpaces && keepSpaceStrings) || + (!bIsSpaces && !bIsEmpty)) { - if (bCaseSensitive) + tokenVisitor(*nextToken); + } + } + } + + void TokenizeVisitorReverse(AZStd::string_view in, const TokenVisitor& tokenVisitor, const char delimiter, bool keepEmptyStrings, bool keepSpaceStrings) + { + return TokenizeVisitorReverse(in, tokenVisitor, { &delimiter, 1 }, keepEmptyStrings, keepSpaceStrings); + } + + void TokenizeVisitorReverse(AZStd::string_view in, const TokenVisitor& tokenVisitor, AZStd::string_view delimiters, + bool keepEmptyStrings, bool keepSpaceStrings) + { + if (delimiters.empty() || in.empty()) + { + return; + } + + while (AZStd::optional nextToken = TokenizeLast(in, delimiters)) + { + bool bIsEmpty = nextToken->empty(); + bool bIsSpaces = false; + if (!bIsEmpty) + { + AZStd::string_view strippedNextToken = StripEnds(*nextToken, " "); + bIsSpaces = strippedNextToken.empty(); + } + + if ((bIsEmpty && keepEmptyStrings) || + (bIsSpaces && keepSpaceStrings) || + (!bIsSpaces && !bIsEmpty)) + { + tokenVisitor(*nextToken); + } + } + } + + AZStd::optional TokenizeNext(AZStd::string_view& inout, const char delimiter) + { + return TokenizeNext(inout, { &delimiter, 1 }); + } + AZStd::optional TokenizeNext(AZStd::string_view& inout, AZStd::string_view delimiters) + { + if (delimiters.empty() || inout.empty()) + { + return AZStd::nullopt; + } + + AZStd::string_view resultToken; + if (size_t pos = inout.find_first_of(delimiters); pos == AZStd::string_view::npos) + { + // The delimiter has not been found, a new view containing the entire + // string will be returned and the input parameter will be set to empty + resultToken.swap(inout); + } + else + { + resultToken = { inout.data(), pos }; + // Strip off all previous characters before the delimiter plus + // the delimiter itself from the input view + inout.remove_prefix(pos + 1); + } + + return resultToken; + } + + AZStd::optional TokenizeLast(AZStd::string_view& inout, const char delimiter) + { + return TokenizeLast(inout, { &delimiter, 1 }); + } + AZStd::optional TokenizeLast(AZStd::string_view& inout, AZStd::string_view delimiters) + { + if (delimiters.empty() || inout.empty()) + { + return AZStd::nullopt; + } + + AZStd::string_view resultToken; + if (size_t pos = inout.find_last_of(delimiters); pos == AZStd::string_view::npos) + { + // The delimiter has not been found, a new view containing the entire + // string will be returned and the input parameter will be set to empty + resultToken.swap(inout); + } + else + { + resultToken = inout.substr(pos + 1); + // Strip off all previous characters before the delimiter plus + // the delimiter itself from the input view + inout = inout.substr(0, pos); + } + + return resultToken; + } + + bool FindFirstOf(AZStd::string_view inString, size_t offset, const AZStd::vector& searchStrings, uint32_t& outIndex, size_t& outOffset) + { + bool found = false; + + outIndex = 0; + outOffset = AZStd::string::npos; + for (int32_t i = 0; i < searchStrings.size(); ++i) + { + const AZStd::string& search = searchStrings[i]; + + size_t entry = inString.find(search, offset); + if (entry != AZStd::string::npos) + { + if (!found || (entry < outOffset)) { - if (!strncmp(pCur, s.data(), slen)) - { - return static_cast(pCur - in.data()); - } - } - else - { - if (!azstrnicmp(pCur, s.data(), slen)) - { - return static_cast(pCur - in.data()); - } - } - - if (bReverse) - { - pCur--; - } - else - { - pCur++; - } - } while (bReverse ? pCur >= in.data() : pCur - in.data() <= static_cast(inlen)); - - return AZStd::string::npos; - } - - char FirstCharacter(const char* in) - { - if (!in) - { - return '\0'; - } - if (in[0] == '\n') - { - return '\0'; - } - return in[0]; - } - - char LastCharacter(const char* in) - { - if (!in) - { - return '\0'; - } - size_t len = strlen(in); - if (!len) - { - return '\0'; - } - return in[len - 1]; - } - - AZStd::string& Append(AZStd::string& inout, const char s) - { - return inout.append(1, s); - } - - AZStd::string& Append(AZStd::string& inout, const char* str) - { - if (!str) - { - return inout; - } - return inout.append(str); - } - - AZStd::string& Prepend(AZStd::string& inout, const char s) - { - return inout.insert((size_t)0, 1, s); - } - - AZStd::string& Prepend(AZStd::string& inout, const char* str) - { - if (!str) - { - return inout; - } - return inout.insert(0, str); - } - - AZStd::string& LChop(AZStd::string& inout, size_t num) - { - return Internal::LChop(inout, num); - } - - AZStd::string_view LChop(AZStd::string_view in, size_t num) - { - return Internal::LChop(in, num); - } - - AZStd::string& RChop(AZStd::string& inout, size_t num) - { - return Internal::RChop(inout, num); - } - - AZStd::string_view RChop(AZStd::string_view in, size_t num) - { - return Internal::RChop(in, num); - } - - AZStd::string& LKeep(AZStd::string& inout, size_t pos, bool bKeepPosCharacter) - { - return Internal::LKeep(inout, pos, bKeepPosCharacter); - } - - AZStd::string& RKeep(AZStd::string& inout, size_t pos, bool bKeepPosCharacter) - { - return Internal::RKeep(inout, pos, bKeepPosCharacter); - } - - bool Replace(AZStd::string& inout, const char replaceA, const char withB, bool bCaseSensitive, bool bReplaceFirst, bool bReplaceLast) - { - return Internal::Replace(inout, replaceA, withB, bCaseSensitive, bReplaceFirst, bReplaceLast); - } - - bool Replace(AZStd::string& inout, const char* replaceA, const char* withB, bool bCaseSensitive, bool bReplaceFirst, bool bReplaceLast) - { - return Internal::Replace(inout, replaceA, withB, bCaseSensitive, bReplaceFirst, bReplaceLast); - } - - bool Strip(AZStd::string& inout, const char stripCharacter, bool bCaseSensitive, bool bStripBeginning, bool bStripEnding) - { - return Internal::Strip(inout, stripCharacter, bCaseSensitive, bStripBeginning, bStripEnding); - } - - bool Strip(AZStd::string& inout, const char* stripCharacters, bool bCaseSensitive, bool bStripBeginning, bool bStripEnding) - { - return Internal::Strip(inout, stripCharacters, bCaseSensitive, bStripBeginning, bStripEnding); - } - - AZStd::string& TrimWhiteSpace(AZStd::string& value, bool leading, bool trailing) - { - static const char* trimmable = " \t\r\n"; - if (value.length() > 0) - { - if (leading) - { - value.erase(0, value.find_first_not_of(trimmable)); - } - if (trailing) - { - value.erase(value.find_last_not_of(trimmable) + 1); - } - } - return value; - } - - void Tokenize(AZStd::string_view in, AZStd::vector& tokens, const char delimiter, bool keepEmptyStrings, bool keepSpaceStrings) - { - return Tokenize(in, tokens, { &delimiter, 1 }, keepEmptyStrings, keepSpaceStrings); - } - - void Tokenize(AZStd::string_view in, AZStd::vector& tokens, AZStd::string_view delimiters, bool keepEmptyStrings, bool keepSpaceStrings) - { - auto insertVisitor = [&tokens](AZStd::string_view token) - { - tokens.push_back(token); - }; - return TokenizeVisitor(in, insertVisitor, delimiters, keepEmptyStrings, keepSpaceStrings); - } - - void TokenizeVisitor(AZStd::string_view in, const TokenVisitor& tokenVisitor, const char delimiter, bool keepEmptyStrings, bool keepSpaceStrings) - { - return TokenizeVisitor(in, tokenVisitor, { &delimiter, 1 }, keepEmptyStrings, keepSpaceStrings); - } - - void TokenizeVisitor(AZStd::string_view in, const TokenVisitor& tokenVisitor, AZStd::string_view delimiters, - bool keepEmptyStrings, bool keepSpaceStrings) - { - if (delimiters.empty() || in.empty()) - { - return; - } - - while (AZStd::optional nextToken = TokenizeNext(in, delimiters)) - { - bool bIsEmpty = nextToken->empty(); - bool bIsSpaces = false; - if (!bIsEmpty) - { - AZStd::string_view strippedNextToken = StripEnds(*nextToken, " "); - bIsSpaces = strippedNextToken.empty(); - } - - if ((bIsEmpty && keepEmptyStrings) || - (bIsSpaces && keepSpaceStrings) || - (!bIsSpaces && !bIsEmpty)) - { - tokenVisitor(*nextToken); + found = true; + outIndex = i; + outOffset = entry; } } } - void TokenizeVisitorReverse(AZStd::string_view in, const TokenVisitor& tokenVisitor, const char delimiter, bool keepEmptyStrings, bool keepSpaceStrings) + return found; + } + + void Tokenize(AZStd::string_view input, AZStd::vector& tokens, const AZStd::vector& delimiters, bool keepEmptyStrings /*= false*/, bool keepSpaceStrings /*= false*/) + { + if (input.empty()) { - return TokenizeVisitorReverse(in, tokenVisitor, { &delimiter, 1 }, keepEmptyStrings, keepSpaceStrings); + return; } - void TokenizeVisitorReverse(AZStd::string_view in, const TokenVisitor& tokenVisitor, AZStd::string_view delimiters, - bool keepEmptyStrings, bool keepSpaceStrings) + size_t offset = 0; + for (;;) { - if (delimiters.empty() || in.empty()) + uint32_t nextMatch = 0; + size_t nextOffset = offset; + if (!FindFirstOf(input, offset, delimiters, nextMatch, nextOffset)) { - return; + // No more occurrences of a separator, consume whatever is left and exit + tokens.push_back(input.substr(offset)); + break; } - while (AZStd::optional nextToken = TokenizeLast(in, delimiters)) + // Take the substring, not including the separator, and increment our offset + AZStd::string nextSubstring = input.substr(offset, nextOffset - offset); + if (keepEmptyStrings || keepSpaceStrings || !nextSubstring.empty()) { - bool bIsEmpty = nextToken->empty(); - bool bIsSpaces = false; - if (!bIsEmpty) - { - AZStd::string_view strippedNextToken = StripEnds(*nextToken, " "); - bIsSpaces = strippedNextToken.empty(); - } - - if ((bIsEmpty && keepEmptyStrings) || - (bIsSpaces && keepSpaceStrings) || - (!bIsSpaces && !bIsEmpty)) - { - tokenVisitor(*nextToken); - } + tokens.push_back(nextSubstring); } + + offset = nextOffset + delimiters[nextMatch].size(); } + } - AZStd::optional TokenizeNext(AZStd::string_view& inout, const char delimiter) + int ToInt(const char* in) + { + if (!in) { - return TokenizeNext(inout, { &delimiter, 1 }); + return 0; } - AZStd::optional TokenizeNext(AZStd::string_view& inout, AZStd::string_view delimiters) + return atoi(in); + } + + bool LooksLikeInt(const char* in, int* pInt /*=nullptr*/) + { + if (!in) { - if (delimiters.empty() || inout.empty()) - { - return AZStd::nullopt; - } - - AZStd::string_view resultToken; - if (size_t pos = inout.find_first_of(delimiters); pos == AZStd::string_view::npos) - { - // The delimiter has not been found, a new view containing the entire - // string will be returned and the input parameter will be set to empty - resultToken.swap(inout); - } - else - { - resultToken = { inout.data(), pos }; - // Strip off all previous characters before the delimiter plus - // the delimiter itself from the input view - inout.remove_prefix(pos + 1); - } - - return resultToken; - } - - AZStd::optional TokenizeLast(AZStd::string_view& inout, const char delimiter) - { - return TokenizeLast(inout, { &delimiter, 1 }); - } - AZStd::optional TokenizeLast(AZStd::string_view& inout, AZStd::string_view delimiters) - { - if (delimiters.empty() || inout.empty()) - { - return AZStd::nullopt; - } - - AZStd::string_view resultToken; - if (size_t pos = inout.find_last_of(delimiters); pos == AZStd::string_view::npos) - { - // The delimiter has not been found, a new view containing the entire - // string will be returned and the input parameter will be set to empty - resultToken.swap(inout); - } - else - { - resultToken = inout.substr(pos + 1); - // Strip off all previous characters before the delimiter plus - // the delimiter itself from the input view - inout = inout.substr(0, pos); - } - - return resultToken; - } - - bool FindFirstOf(AZStd::string_view inString, size_t offset, const AZStd::vector& searchStrings, uint32_t& outIndex, size_t& outOffset) - { - bool found = false; - - outIndex = 0; - outOffset = AZStd::string::npos; - for (int32_t i = 0; i < searchStrings.size(); ++i) - { - const AZStd::string& search = searchStrings[i]; - - size_t entry = inString.find(search, offset); - if (entry != AZStd::string::npos) - { - if (!found || (entry < outOffset)) - { - found = true; - outIndex = i; - outOffset = entry; - } - } - } - - return found; - } - - void Tokenize(AZStd::string_view input, AZStd::vector& tokens, const AZStd::vector& delimiters, bool keepEmptyStrings /*= false*/, bool keepSpaceStrings /*= false*/) - { - if (input.empty()) - { - return; - } - - size_t offset = 0; - for (;;) - { - uint32_t nextMatch = 0; - size_t nextOffset = offset; - if (!FindFirstOf(input, offset, delimiters, nextMatch, nextOffset)) - { - // No more occurrences of a separator, consume whatever is left and exit - tokens.push_back(input.substr(offset)); - break; - } - - // Take the substring, not including the separator, and increment our offset - AZStd::string nextSubstring = input.substr(offset, nextOffset - offset); - if (keepEmptyStrings || keepSpaceStrings || !nextSubstring.empty()) - { - tokens.push_back(nextSubstring); - } - - offset = nextOffset + delimiters[nextMatch].size(); - } - } - - int ToInt(const char* in) - { - if (!in) - { - return 0; - } - return atoi(in); - } - - bool LooksLikeInt(const char* in, int* pInt /*=nullptr*/) - { - if (!in) - { - return false; - } - - //if pos is past then end of the string false - size_t len = strlen(in); - if (!len)//must at least 1 characters to work with "1" - { - return false; - } - - const char* pStr = in; - - size_t countNeg = 0; - while (*pStr != '\0' && - (isdigit(*pStr) || - *pStr == '-')) - { - if (*pStr == '-') - { - countNeg++; - } - pStr++; - } - - if (*pStr == '\0' && - countNeg < 2) - { - if (pInt) - { - *pInt = ToInt(in); - } - - return true; - } return false; } - double ToDouble(const char* in) + //if pos is past then end of the string false + size_t len = strlen(in); + if (!len)//must at least 1 characters to work with "1" { - if (!in) - { - return 0.; - } - return atof(in); - } - - bool LooksLikeDouble(const char* in, double* pDouble) - { - if (!in) - { - return false; - } - - size_t len = strlen(in); - if (len < 2)//must have at least 2 characters to work with "1." - { - return false; - } - - const char* pStr = in; - - size_t countDot = 0; - size_t countNeg = 0; - while (*pStr != '\0' && - (isdigit(*pStr) || - (*pStr == '-' || - *pStr == '.'))) - { - if (*pStr == '.') - { - countDot++; - } - if (*pStr == '-') - { - countNeg++; - } - pStr++; - } - - if (*pStr == '\0' && - countDot == 1 && - countNeg < 2) - { - if (pDouble) - { - *pDouble = ToDouble(in); - } - - return true; - } - return false; } - float ToFloat(const char* in) + const char* pStr = in; + + size_t countNeg = 0; + while (*pStr != '\0' && + (isdigit(*pStr) || + *pStr == '-')) { - if (!in) + if (*pStr == '-') { - return 0.f; + countNeg++; } - return (float)atof(in); + pStr++; } - bool LooksLikeFloat(const char* in, float* pFloat /* = nullptr */) + if (*pStr == '\0' && + countNeg < 2) { - bool result = false; - - if (pFloat) + if (pInt) { - double doubleValue = 0.0; - result = LooksLikeDouble(in, &doubleValue); - - (*pFloat) = aznumeric_cast(doubleValue); - } - else - { - result = LooksLikeDouble(in); + *pInt = ToInt(in); } - return result; + return true; } + return false; + } - bool ToBool(const char* in) + double ToDouble(const char* in) + { + if (!in) + { + return 0.; + } + return atof(in); + } + + bool LooksLikeDouble(const char* in, double* pDouble) + { + if (!in) { - bool boolValue = false; - if (LooksLikeBool(in, &boolValue)) - { - return boolValue; - } return false; } - bool LooksLikeBool(const char* in, bool* pBool /* = nullptr */) + size_t len = strlen(in); + if (len < 2)//must have at least 2 characters to work with "1." { - if (!in) - { - return false; - } - - if (!azstricmp(in, "true") || !azstricmp(in, "1")) - { - if (pBool) - { - *pBool = true; - } - return true; - } - - if (!azstricmp(in, "false") || !azstricmp(in, "0")) - { - if (pBool) - { - *pBool = false; - } - return true; - } - return false; } - template - bool LooksLikeVectorHelper(const char* in, VECTOR_TYPE* outVector) - { - AZStd::vector tokens; - Tokenize(in, tokens, ',', false, true); - if (tokens.size() == ELEMENT_COUNT) - { - float vectorValues[4] = { 0.0f, 0.0f, 0.0f, 0.0f }; + const char* pStr = in; + size_t countDot = 0; + size_t countNeg = 0; + while (*pStr != '\0' && + (isdigit(*pStr) || + (*pStr == '-' || + *pStr == '.'))) + { + if (*pStr == '.') + { + countDot++; + } + if (*pStr == '-') + { + countNeg++; + } + pStr++; + } + + if (*pStr == '\0' && + countDot == 1 && + countNeg < 2) + { + if (pDouble) + { + *pDouble = ToDouble(in); + } + + return true; + } + + return false; + } + + float ToFloat(const char* in) + { + if (!in) + { + return 0.f; + } + return (float)atof(in); + } + + bool LooksLikeFloat(const char* in, float* pFloat /* = nullptr */) + { + bool result = false; + + if (pFloat) + { + double doubleValue = 0.0; + result = LooksLikeDouble(in, &doubleValue); + + (*pFloat) = aznumeric_cast(doubleValue); + } + else + { + result = LooksLikeDouble(in); + } + + return result; + } + + bool ToBool(const char* in) + { + bool boolValue = false; + if (LooksLikeBool(in, &boolValue)) + { + return boolValue; + } + return false; + } + + bool LooksLikeBool(const char* in, bool* pBool /* = nullptr */) + { + if (!in) + { + return false; + } + + if (!azstricmp(in, "true") || !azstricmp(in, "1")) + { + if (pBool) + { + *pBool = true; + } + return true; + } + + if (!azstricmp(in, "false") || !azstricmp(in, "0")) + { + if (pBool) + { + *pBool = false; + } + return true; + } + + return false; + } + + template + bool LooksLikeVectorHelper(const char* in, VECTOR_TYPE* outVector) + { + AZStd::vector tokens; + Tokenize(in, tokens, ',', false, true); + if (tokens.size() == ELEMENT_COUNT) + { + float vectorValues[4] = { 0.0f, 0.0f, 0.0f, 0.0f }; + + for (uint32_t element = 0; element < ELEMENT_COUNT; ++element) + { + if (!LooksLikeFloat(tokens[element].c_str(), outVector ? &vectorValues[element] : nullptr)) + { + return false; + } + } + + if (outVector) + { for (uint32_t element = 0; element < ELEMENT_COUNT; ++element) { - if (!LooksLikeFloat(tokens[element].c_str(), outVector ? &vectorValues[element] : nullptr)) - { - return false; - } + outVector->SetElement(element, vectorValues[element]); } - - if (outVector) - { - for (uint32_t element = 0; element < ELEMENT_COUNT; ++element) - { - outVector->SetElement(element, vectorValues[element]); - } - } - - return true; } - return false; - } - - bool LooksLikeVector2(const char* in, AZ::Vector2* outVector) - { - return LooksLikeVectorHelper(in, outVector); - } - - AZ::Vector2 ToVector2(const char* in) - { - AZ::Vector2 vector; - LooksLikeVector2(in, &vector); - return vector; - } - - bool LooksLikeVector3(const char* in, AZ::Vector3* outVector) - { - return LooksLikeVectorHelper(in, outVector); - } - - AZ::Vector3 ToVector3(const char* in) - { - AZ::Vector3 vector; - LooksLikeVector3(in, &vector); - return vector; - } - - bool LooksLikeVector4(const char* in, AZ::Vector4* outVector) - { - return LooksLikeVectorHelper(in, outVector); - } - - AZ::Vector4 ToVector4(const char* in) - { - AZ::Vector4 vector; - LooksLikeVector4(in, &vector); - return vector; - } - - bool ToHexDump(const char* in, AZStd::string& out) - { - struct TInline - { - static void ByteToHex(char* pszHex, unsigned char bValue) - { - pszHex[0] = bValue / 16; - - if (pszHex[0] < 10) - { - pszHex[0] += '0'; - } - else - { - pszHex[0] -= 10; - pszHex[0] += 'A'; - } - - pszHex[1] = bValue % 16; - - if (pszHex[1] < 10) - { - pszHex[1] += '0'; - } - else - { - pszHex[1] -= 10; - pszHex[1] += 'A'; - } - } - }; - - size_t len = strlen(in); - if (len < 1) //must be at least 1 character to work with - { - return false; - } - - size_t nBytes = len; - - char* pszData = reinterpret_cast(azmalloc((nBytes * 2) + 1)); - - for (size_t ii = 0; ii < nBytes; ++ii) - { - TInline::ByteToHex(&pszData[ii * 2], in[ii]); - } - - pszData[nBytes * 2] = 0x00; - out = pszData; - azfree(pszData); - return true; } - bool FromHexDump(const char* in, AZStd::string& out) + return false; + } + + bool LooksLikeVector2(const char* in, AZ::Vector2* outVector) + { + return LooksLikeVectorHelper(in, outVector); + } + + AZ::Vector2 ToVector2(const char* in) + { + AZ::Vector2 vector; + LooksLikeVector2(in, &vector); + return vector; + } + + bool LooksLikeVector3(const char* in, AZ::Vector3* outVector) + { + return LooksLikeVectorHelper(in, outVector); + } + + AZ::Vector3 ToVector3(const char* in) + { + AZ::Vector3 vector; + LooksLikeVector3(in, &vector); + return vector; + } + + bool LooksLikeVector4(const char* in, AZ::Vector4* outVector) + { + return LooksLikeVectorHelper(in, outVector); + } + + AZ::Vector4 ToVector4(const char* in) + { + AZ::Vector4 vector; + LooksLikeVector4(in, &vector); + return vector; + } + + bool ToHexDump(const char* in, AZStd::string& out) + { + struct TInline { - struct TInline + static void ByteToHex(char* pszHex, unsigned char bValue) { - static unsigned char HexToByte(const char* pszHex) + pszHex[0] = bValue / 16; + + if (pszHex[0] < 10) { - unsigned char bHigh = 0; - unsigned char bLow = 0; - - if ((pszHex[0] >= '0') && (pszHex[0] <= '9')) - { - bHigh = pszHex[0] - '0'; - } - else if ((pszHex[0] >= 'A') && (pszHex[0] <= 'F')) - { - bHigh = (pszHex[0] - 'A') + 10; - } - - bHigh = bHigh << 4; - - if ((pszHex[1] >= '0') && (pszHex[1] <= '9')) - { - bLow = pszHex[1] - '0'; - } - else if ((pszHex[1] >= 'A') && (pszHex[1] <= 'F')) - { - bLow = (pszHex[1] - 'A') + 10; - } - - return bHigh | bLow; - } - }; - - size_t len = strlen(in); - if (len < 2) //must be at least 2 characters to work with - { - return false; - } - - size_t nBytes = len / 2; - char* pszData = reinterpret_cast(azmalloc(nBytes + 1)); - - for (size_t ii = 0; ii < nBytes; ++ii) - { - pszData[ii] = TInline::HexToByte(&in[ii * 2]); - } - - pszData[nBytes] = 0x00; - out = pszData; - azfree(pszData); - - return true; - } - - namespace NumberFormatting - { - int GroupDigits(char* buffer, size_t bufferSize, size_t decimalPosHint, char digitSeparator, char decimalSeparator, int groupingSize, int firstGroupingSize) - { - static const int MAX_SEPARATORS = 16; - - AZ_Assert(buffer, "Null string buffer"); - AZ_Assert(bufferSize > decimalPosHint, "Decimal position %lu cannot be located beyond bufferSize %lu", decimalPosHint, bufferSize); - AZ_Assert(groupingSize > 0, "Grouping size must be a positive integer"); - - int numberEndPos = 0; - int stringEndPos = 0; - - if (decimalPosHint > 0 && decimalPosHint < (bufferSize - 1) && buffer[decimalPosHint] == decimalSeparator) - { - // Assume the number ends at the supplied location - numberEndPos = (int)decimalPosHint; - stringEndPos = numberEndPos + (int)strnlen(buffer + numberEndPos, bufferSize - numberEndPos); + pszHex[0] += '0'; } else { - // Search for the final digit or separator while obtaining the string length - int lastDigitSeenPos = 0; - - while (stringEndPos < bufferSize) - { - char c = buffer[stringEndPos]; - - if (!c) - { - break; - } - else if (c == decimalSeparator) - { - // End the number if there's a decimal - numberEndPos = stringEndPos; - } - else if (numberEndPos <= 0 && c >= '0' && c <= '9') - { - // Otherwise keep track of where the last digit we've seen is - lastDigitSeenPos = stringEndPos; - } - - stringEndPos++; - } - - if (numberEndPos <= 0) - { - if (lastDigitSeenPos > 0) - { - // No decimal, so use the last seen digit as the end of the number - numberEndPos = lastDigitSeenPos + 1; - } - else - { - // No digits, no decimals, therefore no change in the string - return stringEndPos; - } - } + pszHex[0] -= 10; + pszHex[0] += 'A'; } - if (firstGroupingSize <= 0) + pszHex[1] = bValue % 16; + + if (pszHex[1] < 10) { - firstGroupingSize = groupingSize; + pszHex[1] += '0'; + } + else + { + pszHex[1] -= 10; + pszHex[1] += 'A'; + } + } + }; + + size_t len = strlen(in); + if (len < 1) //must be at least 1 character to work with + { + return false; + } + + size_t nBytes = len; + + char* pszData = reinterpret_cast(azmalloc((nBytes * 2) + 1)); + + for (size_t ii = 0; ii < nBytes; ++ii) + { + TInline::ByteToHex(&pszData[ii * 2], in[ii]); + } + + pszData[nBytes * 2] = 0x00; + out = pszData; + azfree(pszData); + + return true; + } + + bool FromHexDump(const char* in, AZStd::string& out) + { + struct TInline + { + static unsigned char HexToByte(const char* pszHex) + { + unsigned char bHigh = 0; + unsigned char bLow = 0; + + if ((pszHex[0] >= '0') && (pszHex[0] <= '9')) + { + bHigh = pszHex[0] - '0'; + } + else if ((pszHex[0] >= 'A') && (pszHex[0] <= 'F')) + { + bHigh = (pszHex[0] - 'A') + 10; } - // Determine where to place the separators - int groupingSizes[] = { firstGroupingSize + 1, groupingSize }; // First group gets +1 since we begin all subsequent groups at the second digit - int groupingOffsetsToNext[] = { 1, 0 }; // We will offset from the first entry to the second, then stay at the second for remaining iterations - const int* currentGroupingSize = groupingSizes; - const int* currentGroupingOffsetToNext = groupingOffsetsToNext; - AZStd::fixed_vector separatorLocations; - int groupCounter = 0; - int digitPosition = numberEndPos - 1; + bHigh = bHigh << 4; - while (digitPosition >= 0) + if ((pszHex[1] >= '0') && (pszHex[1] <= '9')) { - // Walk backwards in the string from the least significant digit to the most significant, demarcating consecutive groups of digits - char c = buffer[digitPosition]; + bLow = pszHex[1] - '0'; + } + else if ((pszHex[1] >= 'A') && (pszHex[1] <= 'F')) + { + bLow = (pszHex[1] - 'A') + 10; + } - if (c >= '0' && c <= '9') - { - if (++groupCounter == *currentGroupingSize) - { - // Demarcate a new group of digits at this location - separatorLocations.push_back(buffer + digitPosition); - currentGroupingSize += *currentGroupingOffsetToNext; - currentGroupingOffsetToNext += *currentGroupingOffsetToNext; - groupCounter = 0; - } + return bHigh | bLow; + } + }; - digitPosition--; - } - else + size_t len = strlen(in); + if (len < 2) //must be at least 2 characters to work with + { + return false; + } + + size_t nBytes = len / 2; + char* pszData = reinterpret_cast(azmalloc(nBytes + 1)); + + for (size_t ii = 0; ii < nBytes; ++ii) + { + pszData[ii] = TInline::HexToByte(&in[ii * 2]); + } + + pszData[nBytes] = 0x00; + out = pszData; + azfree(pszData); + + return true; + } + + namespace NumberFormatting + { + int GroupDigits(char* buffer, size_t bufferSize, size_t decimalPosHint, char digitSeparator, char decimalSeparator, int groupingSize, int firstGroupingSize) + { + static const int MAX_SEPARATORS = 16; + + AZ_Assert(buffer, "Null string buffer"); + AZ_Assert(bufferSize > decimalPosHint, "Decimal position %lu cannot be located beyond bufferSize %lu", decimalPosHint, bufferSize); + AZ_Assert(groupingSize > 0, "Grouping size must be a positive integer"); + + int numberEndPos = 0; + int stringEndPos = 0; + + if (decimalPosHint > 0 && decimalPosHint < (bufferSize - 1) && buffer[decimalPosHint] == decimalSeparator) + { + // Assume the number ends at the supplied location + numberEndPos = (int)decimalPosHint; + stringEndPos = numberEndPos + (int)strnlen(buffer + numberEndPos, bufferSize - numberEndPos); + } + else + { + // Search for the final digit or separator while obtaining the string length + int lastDigitSeenPos = 0; + + while (stringEndPos < bufferSize) + { + char c = buffer[stringEndPos]; + + if (!c) { break; } - } - - if (stringEndPos + separatorLocations.size() >= bufferSize) - { - // Won't fit into buffer, so return unchanged - return stringEndPos; - } - - // Insert the separators by shifting characters forward in the string, starting at the end and working backwards - const char* src = buffer + stringEndPos; - char* dest = buffer + stringEndPos + separatorLocations.size(); - auto separatorItr = separatorLocations.begin(); - - while (separatorItr != separatorLocations.end()) - { - while (src > *separatorItr) + else if (c == decimalSeparator) { - *dest-- = *src--; + // End the number if there's a decimal + numberEndPos = stringEndPos; + } + else if (numberEndPos <= 0 && c >= '0' && c <= '9') + { + // Otherwise keep track of where the last digit we've seen is + lastDigitSeenPos = stringEndPos; } - // Insert the separator and reduce the distance between our destination and source by one - *dest-- = digitSeparator; - ++separatorItr; + stringEndPos++; } - return (int)(stringEndPos + separatorLocations.size()); - } - } - - namespace AssetPath - { - namespace Internal - { - AZ::u32 CalculateBranchTokenHash(AZStd::string_view engineRootPath) + if (numberEndPos <= 0) { - // Normalize the token to prepare for CRC32 calculation - auto NormalizeEnginePath = [](const char element) -> char + if (lastDigitSeenPos > 0) { - // Substitute path separators with '_' and lower case - return element == AZ::IO::WindowsPathSeparator || element == AZ::IO::PosixPathSeparator - ? '_' : static_cast(std::tolower(element)); - }; - - // Trim off trailing path separators - engineRootPath = RStrip(engineRootPath, AZ_CORRECT_AND_WRONG_FILESYSTEM_SEPARATOR); - AZ::IO::FixedMaxPathString enginePath; - AZStd::transform(engineRootPath.begin(), engineRootPath.end(), - AZStd::back_inserter(enginePath), AZStd::move(NormalizeEnginePath)); - - // Perform the CRC32 calculation - constexpr bool forceLowercase = true; - return static_cast(AZ::Crc32(enginePath.c_str(), enginePath.size(), forceLowercase)); + // No decimal, so use the last seen digit as the end of the number + numberEndPos = lastDigitSeenPos + 1; + } + else + { + // No digits, no decimals, therefore no change in the string + return stringEndPos; + } } } - void CalculateBranchToken(AZStd::string_view engineRootPath, AZStd::string& token) + + if (firstGroupingSize <= 0) { - token = AZStd::string::format("0x%08X", Internal::CalculateBranchTokenHash(engineRootPath)); + firstGroupingSize = groupingSize; } - void CalculateBranchToken(AZStd::string_view engineRootPath, AZ::IO::FixedMaxPathString& token) + + // Determine where to place the separators + int groupingSizes[] = { firstGroupingSize + 1, groupingSize }; // First group gets +1 since we begin all subsequent groups at the second digit + int groupingOffsetsToNext[] = { 1, 0 }; // We will offset from the first entry to the second, then stay at the second for remaining iterations + const int* currentGroupingSize = groupingSizes; + const int* currentGroupingOffsetToNext = groupingOffsetsToNext; + AZStd::fixed_vector separatorLocations; + int groupCounter = 0; + int digitPosition = numberEndPos - 1; + + while (digitPosition >= 0) { - token = AZ::IO::FixedMaxPathString::format("0x%08X", Internal::CalculateBranchTokenHash(engineRootPath)); + // Walk backwards in the string from the least significant digit to the most significant, demarcating consecutive groups of digits + char c = buffer[digitPosition]; + + if (c >= '0' && c <= '9') + { + if (++groupCounter == *currentGroupingSize) + { + // Demarcate a new group of digits at this location + separatorLocations.push_back(buffer + digitPosition); + currentGroupingSize += *currentGroupingOffsetToNext; + currentGroupingOffsetToNext += *currentGroupingOffsetToNext; + groupCounter = 0; + } + + digitPosition--; + } + else + { + break; + } } + + if (stringEndPos + separatorLocations.size() >= bufferSize) + { + // Won't fit into buffer, so return unchanged + return stringEndPos; + } + + // Insert the separators by shifting characters forward in the string, starting at the end and working backwards + const char* src = buffer + stringEndPos; + char* dest = buffer + stringEndPos + separatorLocations.size(); + auto separatorItr = separatorLocations.begin(); + + while (separatorItr != separatorLocations.end()) + { + while (src > *separatorItr) + { + *dest-- = *src--; + } + + // Insert the separator and reduce the distance between our destination and source by one + *dest-- = digitSeparator; + ++separatorItr; + } + + return (int)(stringEndPos + separatorLocations.size()); + } + } + + namespace AssetPath + { + namespace Internal + { + AZ::u32 CalculateBranchTokenHash(AZStd::string_view engineRootPath) + { + // Normalize the token to prepare for CRC32 calculation + auto NormalizeEnginePath = [](const char element) -> char + { + // Substitute path separators with '_' and lower case + return element == AZ::IO::WindowsPathSeparator || element == AZ::IO::PosixPathSeparator + ? '_' + : static_cast(std::tolower(element)); + }; + + // Trim off trailing path separators + engineRootPath = RStrip(engineRootPath, AZ_CORRECT_AND_WRONG_FILESYSTEM_SEPARATOR); + AZ::IO::FixedMaxPathString enginePath; + AZStd::transform( + engineRootPath.begin(), engineRootPath.end(), AZStd::back_inserter(enginePath), AZStd::move(NormalizeEnginePath)); + + // Perform the CRC32 calculation + constexpr bool forceLowercase = true; + return static_cast(AZ::Crc32(enginePath.c_str(), enginePath.size(), forceLowercase)); + } + } // namespace Internal + void CalculateBranchToken(AZStd::string_view engineRootPath, AZStd::string& token) + { + token = AZStd::string::format("0x%08X", Internal::CalculateBranchTokenHash(engineRootPath)); + } + void CalculateBranchToken(AZStd::string_view engineRootPath, AZ::IO::FixedMaxPathString& token) + { + token = AZ::IO::FixedMaxPathString::format("0x%08X", Internal::CalculateBranchTokenHash(engineRootPath)); + } + } // namespace AssetPath + + namespace AssetDatabasePath + { + bool Normalize(AZStd::string& inout) + { + // Asset Paths uses the forward slash for the database separator + AZ::IO::Path path(AZStd::move(inout), AZ_CORRECT_DATABASE_SEPARATOR); + bool appendTrailingSlash = (path.Native().ends_with(AZ_CORRECT_DATABASE_SEPARATOR) || path.Native().ends_with(AZ_WRONG_DATABASE_SEPARATOR)) + && path.HasRelativePath(); + inout = AZStd::move(path.LexicallyNormal().Native()); + if (appendTrailingSlash) + { + inout.push_back(AZ_CORRECT_DATABASE_SEPARATOR); + } + return IsValid(inout.c_str()); } - namespace AssetDatabasePath + bool IsValid(const char* in) { - bool Normalize(AZStd::string& inout) + if (!in) { - // Asset Paths uses the forward slash for the database separator - AZ::IO::Path path(AZStd::move(inout), AZ_CORRECT_DATABASE_SEPARATOR); - bool appendTrailingSlash = (path.Native().ends_with(AZ_CORRECT_DATABASE_SEPARATOR) || path.Native().ends_with(AZ_WRONG_DATABASE_SEPARATOR)) - && path.HasRelativePath(); - inout = AZStd::move(path.LexicallyNormal().Native()); - if (appendTrailingSlash) - { - inout.push_back(AZ_CORRECT_DATABASE_SEPARATOR); - } - return IsValid(inout.c_str()); + return false; } - bool IsValid(const char* in) + if (!strlen(in)) { - if (!in) - { - return false; - } + return false; + } - if (!strlen(in)) - { - return false; - } + if (Find(in, AZ_DATABASE_INVALID_CHARACTERS) != AZStd::string::npos) + { + return false; + } - if (Find(in, AZ_DATABASE_INVALID_CHARACTERS) != AZStd::string::npos) - { - return false; - } - - if (Find(in, AZ_WRONG_DATABASE_SEPARATOR) != AZStd::string::npos) - { - return false; - } + if (Find(in, AZ_WRONG_DATABASE_SEPARATOR) != AZStd::string::npos) + { + return false; + } #ifndef AZ_FILENAME_ALLOW_SPACES - if (Find(in, AZ_SPACE_CHARACTERS) != AZStd::string::npos) - { - return false; - } + if (Find(in, AZ_SPACE_CHARACTERS) != AZStd::string::npos) + { + return false; + } #endif // AZ_FILENAME_ALLOW_SPACES - if (LastCharacter(in) == AZ_CORRECT_DATABASE_SEPARATOR) - { - return false; - } - - return true; - } - - bool Split(const char* in, [[maybe_unused]] AZStd::string* pDstProjectRootOut, AZStd::string* pDstDatabaseRootOut, - AZStd::string* pDstDatabasePathOut , AZStd::string* pDstFileOut, AZStd::string* pDstFileExtensionOut) + if (LastCharacter(in) == AZ_CORRECT_DATABASE_SEPARATOR) { - AZStd::string_view path{ in }; - if (path.empty()) - { - return false; - } - - AZ::IO::PathView pathView(path, AZ_CORRECT_DATABASE_SEPARATOR); - if (pDstDatabaseRootOut) - { - AZStd::string_view rootNameView = pathView.RootName().Native(); - if (rootNameView.size() > pDstDatabaseRootOut->max_size()) - { - return false; - } - *pDstDatabaseRootOut = rootNameView; - } - if (pDstDatabasePathOut) - { - AZStd::string_view rootPathView = pathView.RootPath().Native(); - AZStd::string_view relPathParentView = pathView.ParentPath().RelativePath().Native(); - if (rootPathView.size() + relPathParentView.size() > pDstDatabasePathOut->max_size()) - { - return false; - } - // Append the root directory if there is one - *pDstDatabasePathOut = rootPathView; - // Append the relative path portion of the split path excluding the filename - *pDstDatabasePathOut += relPathParentView; - } - if (pDstFileOut) - { - AZStd::string_view stemView = pathView.Stem().Native(); - if (stemView.size() > pDstFileOut->max_size()) - { - return false; - } - *pDstFileOut = stemView; - } - if (pDstFileExtensionOut) - { - AZStd::string_view extensionView = pathView.Extension().Native(); - if (extensionView.size() > pDstFileExtensionOut->max_size()) - { - return false; - } - *pDstFileExtensionOut = extensionView; - } - - return true; + return false; } - bool Join(const char* pFirstPart, const char* pSecondPart, AZStd::string& out, [[maybe_unused]] bool bCaseInsensitive /*= true*/, bool bNormalize /*= true*/) - { - // both paths cannot be empty - if (!pFirstPart || !pSecondPart) - { - return false; - } + return true; + } - AZ::IO::PathView secondPath(pSecondPart); - AZ_Warning("StringFunc", secondPath.IsRelative(), "The second join parameter %s is an absolute path," - " this will replace the first part of the path resulting in an output of just the second part", - pSecondPart); - - AZ::IO::Path resultPath(pFirstPart, AZ_CORRECT_DATABASE_SEPARATOR); - resultPath /= secondPath; - out = bNormalize ? AZStd::move(resultPath.LexicallyNormal().Native()) : AZStd::move(resultPath.Native()); - return true; - } - } //namespace AssetDatabasePath - - namespace Root + bool Split(const char* in, [[maybe_unused]] AZStd::string* pDstProjectRootOut, AZStd::string* pDstDatabaseRootOut, + AZStd::string* pDstDatabasePathOut , AZStd::string* pDstFileOut, AZStd::string* pDstFileExtensionOut) { - bool Normalize(AZStd::string& inout) + AZStd::string_view path{ in }; + if (path.empty()) { - AZ::IO::Path path(AZStd::move(inout)); - path = path.LexicallyNormal(); - // After normalization check if the path contains a relative path - bool appendTrailingSlash = path.HasRelativePath(); - inout = AZStd::move(path.Native()); - if (appendTrailingSlash) - { - inout.push_back(AZ_CORRECT_FILESYSTEM_SEPARATOR); // Append a trailing separator for Root path normalization - } - return IsValid(inout.c_str()); + return false; } - bool IsValid(const char* in) + AZ::IO::PathView pathView(path, AZ_CORRECT_DATABASE_SEPARATOR); + if (pDstDatabaseRootOut) { - if (!in) + AZStd::string_view rootNameView = pathView.RootName().Native(); + if (rootNameView.size() > pDstDatabaseRootOut->max_size()) { return false; } - - if (!strlen(in)) + *pDstDatabaseRootOut = rootNameView; + } + if (pDstDatabasePathOut) + { + AZStd::string_view rootPathView = pathView.RootPath().Native(); + AZStd::string_view relPathParentView = pathView.ParentPath().RelativePath().Native(); + if (rootPathView.size() + relPathParentView.size() > pDstDatabasePathOut->max_size()) { return false; } - - if (Find(in, AZ_FILESYSTEM_INVALID_CHARACTERS) != AZStd::string::npos) + // Append the root directory if there is one + *pDstDatabasePathOut = rootPathView; + // Append the relative path portion of the split path excluding the filename + *pDstDatabasePathOut += relPathParentView; + } + if (pDstFileOut) + { + AZStd::string_view stemView = pathView.Stem().Native(); + if (stemView.size() > pDstFileOut->max_size()) { return false; } - - if (Find(in, AZ_WRONG_FILESYSTEM_SEPARATOR) != AZStd::string::npos) + *pDstFileOut = stemView; + } + if (pDstFileExtensionOut) + { + AZStd::string_view extensionView = pathView.Extension().Native(); + if (extensionView.size() > pDstFileExtensionOut->max_size()) { return false; } + *pDstFileExtensionOut = extensionView; + } - #ifndef AZ_FILENAME_ALLOW_SPACES - if (Find(in, AZ_SPACE_CHARACTERS) != AZStd::string::npos) - { - return false; - } - #endif // AZ_FILENAME_ALLOW_SPACES + return true; + } - AZ::IO::PathView pathView(in); - if (!pathView.HasRootPath()) - { - return false; - } + bool Join(const char* pFirstPart, const char* pSecondPart, AZStd::string& out, [[maybe_unused]] bool bCaseInsensitive /*= true*/, bool bNormalize /*= true*/) + { + // both paths cannot be empty + if (!pFirstPart || !pSecondPart) + { + return false; + } - if (LastCharacter(in) != AZ_CORRECT_FILESYSTEM_SEPARATOR) - { - return false; - } + AZ::IO::PathView secondPath(pSecondPart); + AZ_Warning("StringFunc", secondPath.IsRelative(), "The second join parameter %s is an absolute path," + " this will replace the first part of the path resulting in an output of just the second part", + pSecondPart); + AZ::IO::Path resultPath(pFirstPart, AZ_CORRECT_DATABASE_SEPARATOR); + resultPath /= secondPath; + out = bNormalize ? AZStd::move(resultPath.LexicallyNormal().Native()) : AZStd::move(resultPath.Native()); + return true; + } + } //namespace AssetDatabasePath + + namespace Root + { + bool Normalize(AZStd::string& inout) + { + AZ::IO::Path path(AZStd::move(inout)); + path = path.LexicallyNormal(); + // After normalization check if the path contains a relative path + bool appendTrailingSlash = path.HasRelativePath(); + inout = AZStd::move(path.Native()); + if (appendTrailingSlash) + { + inout.push_back(AZ_CORRECT_FILESYSTEM_SEPARATOR); // Append a trailing separator for Root path normalization + } + return IsValid(inout.c_str()); + } + + bool IsValid(const char* in) + { + if (!in) + { + return false; + } + + if (!strlen(in)) + { + return false; + } + + if (Find(in, AZ_FILESYSTEM_INVALID_CHARACTERS) != AZStd::string::npos) + { + return false; + } + + if (Find(in, AZ_WRONG_FILESYSTEM_SEPARATOR) != AZStd::string::npos) + { + return false; + } + +#ifndef AZ_FILENAME_ALLOW_SPACES + if (Find(in, AZ_SPACE_CHARACTERS) != AZStd::string::npos) + { + return false; + } +#endif // AZ_FILENAME_ALLOW_SPACES + + AZ::IO::PathView pathView(in); + if (!pathView.HasRootPath()) + { + return false; + } + + if (LastCharacter(in) != AZ_CORRECT_FILESYSTEM_SEPARATOR) + { + return false; + } + + return true; + } + }//namespace Root + + namespace RelativePath + { + bool Normalize(AZStd::string& inout) + { + AZ::IO::Path path(AZStd::move(inout)); + path = path.LexicallyNormal(); + // After normalization check if the path contains a relative path + bool appendTrailingSlash = path.HasRelativePath(); + inout = AZStd::move(path.Native()); + if (appendTrailingSlash) + { + inout.push_back(AZ_CORRECT_FILESYSTEM_SEPARATOR); // Append trailing separator for Relative path normalization if it it is not empty + } + return IsValid(inout.c_str()); + } + + bool IsValid(const char* in) + { + if (!in) + { + return false; + } + + if (!strlen(in)) + { return true; } - }//namespace Root - namespace RelativePath + if (Find(in, AZ_FILESYSTEM_INVALID_CHARACTERS) != AZStd::string::npos) + { + return false; + } + + if (Find(in, AZ_WRONG_FILESYSTEM_SEPARATOR) != AZStd::string::npos) + { + return false; + } + +#ifndef AZ_FILENAME_ALLOW_SPACES + if (Find(in, AZ_SPACE_CHARACTERS) != AZStd::string::npos) + { + return false; + } +#endif // AZ_FILENAME_ALLOW_SPACES + + if (Path::HasDrive(in)) + { + return false; + } + + if (FirstCharacter(in) == AZ_CORRECT_FILESYSTEM_SEPARATOR) + { + return false; + } + + if (LastCharacter(in) != AZ_CORRECT_FILESYSTEM_SEPARATOR) + { + return false; + } + + return true; + } + }//namespace RelativePath + + namespace Path + { + bool Normalize(AZStd::string& inout) { - bool Normalize(AZStd::string& inout) + AZ::IO::Path path(AZStd::move(inout)); + bool appendTrailingSlash = (path.Native().ends_with(AZ_CORRECT_FILESYSTEM_SEPARATOR) || path.Native().ends_with(AZ_WRONG_FILESYSTEM_SEPARATOR)); + path = path.LexicallyNormal(); + // After normalization check if the path contains a relative path and addition to ending with a path separator before + appendTrailingSlash = appendTrailingSlash && path.HasRelativePath(); + inout = AZStd::move(path.Native()); + if (appendTrailingSlash) { - AZ::IO::Path path(AZStd::move(inout)); - path = path.LexicallyNormal(); - // After normalization check if the path contains a relative path - bool appendTrailingSlash = path.HasRelativePath(); - inout = AZStd::move(path.Native()); - if (appendTrailingSlash) - { - inout.push_back(AZ_CORRECT_FILESYSTEM_SEPARATOR); // Append trailing separator for Relative path normalization if it it is not empty - } - return IsValid(inout.c_str()); + inout.push_back(AZ_CORRECT_FILESYSTEM_SEPARATOR); } + return IsValid(inout.c_str()); + } - bool IsValid(const char* in) - { - if (!in) - { - return false; - } - - if (!strlen(in)) - { - return true; - } - - if (Find(in, AZ_FILESYSTEM_INVALID_CHARACTERS) != AZStd::string::npos) - { - return false; - } - - if (Find(in, AZ_WRONG_FILESYSTEM_SEPARATOR) != AZStd::string::npos) - { - return false; - } - - #ifndef AZ_FILENAME_ALLOW_SPACES - if (Find(in, AZ_SPACE_CHARACTERS) != AZStd::string::npos) - { - return false; - } - #endif // AZ_FILENAME_ALLOW_SPACES - - if (Path::HasDrive(in)) - { - return false; - } - - if (FirstCharacter(in) == AZ_CORRECT_FILESYSTEM_SEPARATOR) - { - return false; - } - - if (LastCharacter(in) != AZ_CORRECT_FILESYSTEM_SEPARATOR) - { - return false; - } - - return true; - } - }//namespace RelativePath - - namespace Path + bool Normalize(FixedString& inout) { - bool Normalize(AZStd::string& inout) + AZ::IO::FixedMaxPath path(AZStd::move(inout)); + bool appendTrailingSlash = (path.Native().ends_with(AZ_CORRECT_FILESYSTEM_SEPARATOR) || path.Native().ends_with(AZ_WRONG_FILESYSTEM_SEPARATOR)); + path = path.LexicallyNormal(); + // After normalization check if the path contains a relative path and addition to ending with a path separator before + appendTrailingSlash = appendTrailingSlash && path.HasRelativePath(); + inout = AZStd::move(path.Native()); + if (appendTrailingSlash) { - AZ::IO::Path path(AZStd::move(inout)); - bool appendTrailingSlash = (path.Native().ends_with(AZ_CORRECT_FILESYSTEM_SEPARATOR) || path.Native().ends_with(AZ_WRONG_FILESYSTEM_SEPARATOR)); - path = path.LexicallyNormal(); - // After normalization check if the path contains a relative path and addition to ending with a path separator before - appendTrailingSlash = appendTrailingSlash && path.HasRelativePath(); - inout = AZStd::move(path.Native()); - if (appendTrailingSlash) - { - inout.push_back(AZ_CORRECT_FILESYSTEM_SEPARATOR); - } - return IsValid(inout.c_str()); + inout.push_back(AZ_CORRECT_FILESYSTEM_SEPARATOR); + } + return IsValid(inout.c_str()); + } + + bool IsValid(const char* in, bool bHasDrive /*= false*/, bool bHasExtension /*= false*/, AZStd::string* errors /*= nullptr*/) + { + //if they gave us a error reporting string empty it. + if (errors) + { + errors->clear(); } - bool Normalize(FixedString& inout) + //empty is not a valid path + if (!in) { - AZ::IO::FixedMaxPath path(AZStd::move(inout)); - bool appendTrailingSlash = (path.Native().ends_with(AZ_CORRECT_FILESYSTEM_SEPARATOR) || path.Native().ends_with(AZ_WRONG_FILESYSTEM_SEPARATOR)); - path = path.LexicallyNormal(); - // After normalization check if the path contains a relative path and addition to ending with a path separator before - appendTrailingSlash = appendTrailingSlash && path.HasRelativePath(); - inout = AZStd::move(path.Native()); - if (appendTrailingSlash) - { - inout.push_back(AZ_CORRECT_FILESYSTEM_SEPARATOR); - } - return IsValid(inout.c_str()); - } - - bool IsValid(const char* in, bool bHasDrive /*= false*/, bool bHasExtension /*= false*/, AZStd::string* errors /*= nullptr*/) - { - //if they gave us a error reporting string empty it. if (errors) { - errors->clear(); + *errors += "The path is Empty."; } + return false; + } - //empty is not a valid path - if (!in) + //empty is not a valid path + size_t length = strlen(in); + if (!length) + { + if (errors) { - if (errors) - { - *errors += "The path is Empty."; - } - return false; + *errors += "The path is Empty."; } + return false; + } - //empty is not a valid path - size_t length = strlen(in); - if (!length) + //invalid characters + const char* inEnd = in + length; + const char* invalidCharactersBegin = AZ_FILESYSTEM_INVALID_CHARACTERS; + const char* invalidCharactersEnd = invalidCharactersBegin + AZ_ARRAY_SIZE(AZ_FILESYSTEM_INVALID_CHARACTERS); + if (AZStd::find_first_of(in, inEnd, invalidCharactersBegin, invalidCharactersEnd) != inEnd) + { + if (errors) { - if (errors) - { - *errors += "The path is Empty."; - } - return false; + *errors += "The path has invalid characters."; } + return false; + } - //invalid characters - const char* inEnd = in + length; - const char* invalidCharactersBegin = AZ_FILESYSTEM_INVALID_CHARACTERS; - const char* invalidCharactersEnd = invalidCharactersBegin + AZ_ARRAY_SIZE(AZ_FILESYSTEM_INVALID_CHARACTERS); - if (AZStd::find_first_of(in, inEnd, invalidCharactersBegin, invalidCharactersEnd) != inEnd) + //invalid characters + if (Find(in, AZ_WRONG_FILESYSTEM_SEPARATOR) != AZStd::string::npos) + { + if (errors) { - if (errors) - { - *errors += "The path has invalid characters."; - } - return false; + *errors += "The path has wrong separator."; } + return false; + } - //invalid characters - if (Find(in, AZ_WRONG_FILESYSTEM_SEPARATOR) != AZStd::string::npos) +#ifndef AZ_FILENAME_ALLOW_SPACES + const char* spaceCharactersBegin = AZ_SPACE_CHARACTERS; + const char* spaceCharactersEnd = spaceCharactersBegin + AZ_ARRAY_SIZE(AZ_SPACE_CHARACTERS); + if (AZStd::find_first_of(in, inEnd, spaceCharactersBegin, spaceCharactersEnd) != inEnd) + { + if (errors) { - if (errors) - { - *errors += "The path has wrong separator."; - } - return false; + *errors += "The path has space characters."; } + return false; + } +#endif // AZ_FILENAME_ALLOW_SPACES - #ifndef AZ_FILENAME_ALLOW_SPACES - const char* spaceCharactersBegin = AZ_SPACE_CHARACTERS; - const char* spaceCharactersEnd = spaceCharactersBegin + AZ_ARRAY_SIZE(AZ_SPACE_CHARACTERS); - if (AZStd::find_first_of(in, inEnd, spaceCharactersBegin, spaceCharactersEnd) != inEnd) + //does it have a drive if specified + if (bHasDrive && !HasDrive(in)) + { + if (errors) { - if (errors) - { - *errors += "The path has space characters."; - } - return false; + *errors += "The path should have a drive. The path ["; + *errors += in; + *errors += "] is invalid."; } - #endif // AZ_FILENAME_ALLOW_SPACES + return false; + } - //does it have a drive if specified - if (bHasDrive && !HasDrive(in)) + //does it have and extension if specified + if (bHasExtension && !HasExtension(in)) + { + if (errors) { - if (errors) - { - *errors += "The path should have a drive. The path ["; - *errors += in; - *errors += "] is invalid."; - } - return false; + *errors += "The path should have the a file extension. The path ["; + *errors += in; + *errors += "] is invalid."; } + return false; + } - //does it have and extension if specified - if (bHasExtension && !HasExtension(in)) + //start at the beginning and walk down the characters of the path + const char* elementStart = in; + const char* walk = elementStart; + while (*walk) + { + if (*walk == AZ_CORRECT_FILESYSTEM_SEPARATOR) //is this the correct separator { - if (errors) - { - *errors += "The path should have the a file extension. The path ["; - *errors += in; - *errors += "] is invalid."; - } - return false; + elementStart = walk; } - - //start at the beginning and walk down the characters of the path - const char* elementStart = in; - const char* walk = elementStart; - while (*walk) +#if AZ_TRAIT_OS_USE_WINDOWS_FILE_PATHS + else if (*walk == AZ_FILESYSTEM_DRIVE_SEPARATOR) //is this the drive separator { - if (*walk == AZ_CORRECT_FILESYSTEM_SEPARATOR) //is this the correct separator - { - elementStart = walk; - } - #if AZ_TRAIT_OS_USE_WINDOWS_FILE_PATHS - else if (*walk == AZ_FILESYSTEM_DRIVE_SEPARATOR) //is this the drive separator - { - //A AZ_FILESYSTEM_DRIVE_SEPARATOR character con only occur in the first - //component of a valid path. If the elementStart is not GetBufferPtr() - //then we have past the first component - if (elementStart != in) - { - if (errors) - { - *errors += "There is a stray AZ_FILESYSTEM_DRIVE_SEPARATOR = "; - *errors += AZ_FILESYSTEM_DRIVE_SEPARATOR; - *errors += " found after the first component. The path ["; - *errors += in; - *errors += "] is invalid."; - } - return false; - } - } - #endif - #ifndef AZ_FILENAME_ALLOW_SPACES - else if (*walk == ' ') //is this a space + //A AZ_FILESYSTEM_DRIVE_SEPARATOR character con only occur in the first + //component of a valid path. If the elementStart is not GetBufferPtr() + //then we have past the first component + if (elementStart != in) { if (errors) { - *errors += "The component ["; - for (const char* c = elementStart + 1; c != walk; ++c) - { - *errors += *c; - } - *errors += "] has a SPACE character. The path ["; + *errors += "There is a stray AZ_FILESYSTEM_DRIVE_SEPARATOR = "; + *errors += AZ_FILESYSTEM_DRIVE_SEPARATOR; + *errors += " found after the first component. The path ["; *errors += in; *errors += "] is invalid."; } return false; } - #endif - - ++walk; } - - #if !AZ_TRAIT_OS_ALLOW_UNLIMITED_PATH_COMPONENT_LENGTH - //is this full path longer than AZ::IO::MaxPathLength (The longest a path with all components can possibly be)? - if (walk - in > AZ::IO::MaxPathLength) +#endif +#ifndef AZ_FILENAME_ALLOW_SPACES + else if (*walk == ' ') //is this a space { - if (errors != 0) + if (errors) { - *errors += "The path ["; + *errors += "The component ["; + for (const char* c = elementStart + 1; c != walk; ++c) + { + *errors += *c; + } + *errors += "] has a SPACE character. The path ["; *errors += in; - *errors += "] is over the AZ::IO::MaxPathLength = "; - char buf[64]; - _itoa_s(AZ::IO::MaxPathLength, buf, 10); - *errors += buf; - *errors += " characters total length limit."; + *errors += "] is invalid."; } return false; } - #endif +#endif - return true; + ++walk; } - bool ConstructFull(const char* pRootPath, const char* pFileName, AZStd::string& out, bool bNormalize /* = false*/) +#if !AZ_TRAIT_OS_ALLOW_UNLIMITED_PATH_COMPONENT_LENGTH + //is this full path longer than AZ::IO::MaxPathLength (The longest a path with all components can possibly be)? + if (walk - in > AZ::IO::MaxPathLength) { - if (!pRootPath || AZ::IO::PathView(pRootPath).IsRelative() - || !pFileName || AZ::IO::PathView(pFileName).IsAbsolute()) + if (errors != 0) { - return false; - } - AZ::IO::Path path(pRootPath); - path /= pFileName; - if (bNormalize) - { - out = AZStd::move(path.LexicallyNormal().Native()); - } - else - { - out = AZStd::move(path.Native()); - } - return IsValid(out.c_str()); - } - - bool ConstructFull(const char* pRootPath, const char* pFileName, const char* pFileExtension, AZStd::string& out, bool bNormalize /* = false*/) - { - if (!pRootPath || AZ::IO::PathView(pRootPath).IsRelative() - || !pFileName || AZ::IO::PathView(pFileName).IsAbsolute()) - { - return false; - } - AZ::IO::Path path(pRootPath); - path /= pFileName; - if (pFileExtension) - { - path.ReplaceExtension(pFileExtension); - } - if (bNormalize) - { - out = AZStd::move(path.LexicallyNormal().Native()); - } - else - { - out = AZStd::move(path.Native()); - } - return IsValid(out.c_str()); - } - - bool ConstructFull(const char* pRoot, const char* pRelativePath, const char* pFileName, const char* pFileExtension, AZStd::string& out, bool bNormalize /* = false*/) - { - if (!pRoot || AZ::IO::PathView(pRoot).IsRelative() - || !pRelativePath || AZ::IO::PathView(pRelativePath).IsAbsolute() - || !pFileName || AZ::IO::PathView(pFileName).IsAbsolute()) - { - return false; - } - AZ::IO::Path path(pRoot); - path /= pRelativePath; - path /= pFileName; - if (pFileExtension) - { - path.ReplaceExtension(pFileExtension); - } - if (bNormalize) - { - out = AZStd::move(path.LexicallyNormal().Native()); - } - else - { - out = AZStd::move(path.Native()); - } - return IsValid(out.c_str()); - } - - bool Split(const char* in, AZStd::string* pDstDrive, AZStd::string* pDstPath, AZStd::string* pDstName, AZStd::string* pDstExtension) - { - AZStd::string_view path{ in }; - if (path.empty()) - { - return false; - } - - AZ::IO::PathView pathView(path); - if (pDstDrive) - { - AZStd::string_view rootNameView = pathView.RootName().Native(); - if (rootNameView.size() > pDstDrive->max_size()) - { - return false; - } - *pDstDrive = rootNameView; - } - if (pDstPath) - { - AZStd::string_view rootDirectoryView = pathView.RootDirectory().Native(); - AZStd::string_view relPathParentView = pathView.ParentPath().RelativePath().Native(); - if (rootDirectoryView.size() + relPathParentView.size() > pDstPath->max_size()) - { - return false; - } - // Append the root directory if there is one - *pDstPath = rootDirectoryView; - // Append the relative path portion of the split path excluding the filename - *pDstPath += relPathParentView; - } - if (pDstName) - { - AZStd::string_view stemView = pathView.Stem().Native(); - if (stemView.size() > pDstName->max_size()) - { - return false; - } - *pDstName = stemView; - } - if (pDstExtension) - { - AZStd::string_view extensionView = pathView.Extension().Native(); - if (extensionView.size() > pDstExtension->max_size()) - { - return false; - } - *pDstExtension = extensionView; - } - - return true; - } - - bool Join(const char* pFirstPart, const char* pSecondPart, AZStd::string& out, [[maybe_unused]] bool bCaseInsensitive, bool bNormalize) - { - if (!pFirstPart || !pSecondPart) - { - return false; - } - - AZ::IO::PathView secondPath(pSecondPart); - AZ_Warning("StringFunc", secondPath.IsRelative(), "The second join parameter %s is an absolute path," - " this will replace the first part of the path resulting in an output of just the second part", - pSecondPart); - - AZ::IO::Path resultPath(pFirstPart); - resultPath /= secondPath; - out = bNormalize ? AZStd::move(resultPath.LexicallyNormal().Native()) : AZStd::move(resultPath.Native()); - return true; - } - - bool Join(const char* pFirstPart, const char* pSecondPart, FixedString& out, [[maybe_unused]] bool bCaseInsensitive, bool bNormalize) - { - if (!pFirstPart || !pSecondPart) - { - return false; - } - - AZ::IO::PathView secondPath(pSecondPart); - AZ_Warning("StringFunc", secondPath.IsRelative(), "The second join parameter %s is an absolute path," - " this will replace the first part of the path resulting in an output of just the second part", - pSecondPart); - - AZ::IO::FixedMaxPath resultPath(pFirstPart); - resultPath /= secondPath; - out = bNormalize ? AZStd::move(resultPath.LexicallyNormal().Native()) : AZStd::move(resultPath.Native()); - return true; - } - - bool HasDrive(const char* in, bool bCheckAllFileSystemFormats /*= false*/) - { - // no drive if empty - if (!in || in[0] == '\0') - { - return false; - } - AZ::IO::PathView pathView(in); - return pathView.HasRootName() || (bCheckAllFileSystemFormats && pathView.HasRootDirectory()); - } - - bool HasExtension(const char* in) - { - //it doesn't have an extension if it's empty - if (!in || in[0] == '\0') - { - return false; - } - - return AZ::IO::PathView(in).HasExtension(); - } - - bool IsExtension(const char* in, const char* pExtension, bool bCaseInsenitive /*= false*/) - { - //it doesn't have an extension if it's empty - if (!in || in[0] == '\0' || !pExtension || pExtension[0] == '\0') - { - return false; - } - - AZStd::string_view pathExtension = AZ::IO::PathView(in).Extension().Native(); - if (pathExtension.starts_with(AZ_FILESYSTEM_EXTENSION_SEPARATOR)) - { - pathExtension.remove_prefix(1); - } - AZStd::string_view extensionView(pExtension); - if (extensionView.starts_with(AZ_FILESYSTEM_EXTENSION_SEPARATOR)) - { - extensionView.remove_prefix(1); - } - - return AZStd::equal(pathExtension.begin(), pathExtension.end(), extensionView.begin(), extensionView.end(), - [bCaseInsenitive](const char lhs, const char rhs) - { - return !bCaseInsenitive ? lhs == rhs : tolower(lhs) == tolower(rhs); - }); - } - - bool IsRelative(const char* in) - { - //not relative if empty - if (!in || in[0] == '\0') - { - return false; - } - - return AZ::IO::PathView(in).IsRelative(); - } - - bool StripDrive(AZStd::string& inout) - { - AZ::IO::PathView pathView(inout); - AZ::IO::PathView rootNameView(pathView.RootName()); - if (!rootNameView.empty()) - { - inout.replace(0, rootNameView.Native().size(), ""); - return true; + *errors += "The path ["; + *errors += in; + *errors += "] is over the AZ::IO::MaxPathLength = "; + char buf[64]; + _itoa_s(AZ::IO::MaxPathLength, buf, 10); + *errors += buf; + *errors += " characters total length limit."; } return false; } +#endif - void StripPath(AZStd::string& inout) + return true; + } + + bool ConstructFull(const char* pRootPath, const char* pFileName, AZStd::string& out, bool bNormalize /* = false*/) + { + if (!pRootPath || AZ::IO::PathView(pRootPath).IsRelative() + || !pFileName || AZ::IO::PathView(pFileName).IsAbsolute()) { - inout = AZ::IO::PathView(inout).Filename().Native(); + return false; + } + AZ::IO::Path path(pRootPath); + path /= pFileName; + if (bNormalize) + { + out = AZStd::move(path.LexicallyNormal().Native()); + } + else + { + out = AZStd::move(path.Native()); + } + return IsValid(out.c_str()); + } + + bool ConstructFull(const char* pRootPath, const char* pFileName, const char* pFileExtension, AZStd::string& out, bool bNormalize /* = false*/) + { + if (!pRootPath || AZ::IO::PathView(pRootPath).IsRelative() + || !pFileName || AZ::IO::PathView(pFileName).IsAbsolute()) + { + return false; + } + AZ::IO::Path path(pRootPath); + path /= pFileName; + if (pFileExtension) + { + path.ReplaceExtension(pFileExtension); + } + if (bNormalize) + { + out = AZStd::move(path.LexicallyNormal().Native()); + } + else + { + out = AZStd::move(path.Native()); + } + return IsValid(out.c_str()); + } + + bool ConstructFull(const char* pRoot, const char* pRelativePath, const char* pFileName, const char* pFileExtension, AZStd::string& out, bool bNormalize /* = false*/) + { + if (!pRoot || AZ::IO::PathView(pRoot).IsRelative() + || !pRelativePath || AZ::IO::PathView(pRelativePath).IsAbsolute() + || !pFileName || AZ::IO::PathView(pFileName).IsAbsolute()) + { + return false; + } + AZ::IO::Path path(pRoot); + path /= pRelativePath; + path /= pFileName; + if (pFileExtension) + { + path.ReplaceExtension(pFileExtension); + } + if (bNormalize) + { + out = AZStd::move(path.LexicallyNormal().Native()); + } + else + { + out = AZStd::move(path.Native()); + } + return IsValid(out.c_str()); + } + + bool Split(const char* in, AZStd::string* pDstDrive, AZStd::string* pDstPath, AZStd::string* pDstName, AZStd::string* pDstExtension) + { + AZStd::string_view path{ in }; + if (path.empty()) + { + return false; } - void StripFullName(AZStd::string& inout) + AZ::IO::PathView pathView(path); + if (pDstDrive) { - inout = AZ::IO::Path(AZStd::move(inout)).RemoveFilename().Native(); - } - - void StripExtension(AZStd::string& inout) - { - AZ::IO::Path path(AZStd::move(inout)); - path.ReplaceExtension(); - inout = AZStd::move(path.Native()); - } - - bool StripComponent(AZStd::string& inout, bool bLastComponent /* = false*/) - { - AZ::IO::PathView pathView(inout); - auto pathBeginIter = pathView.begin(); - auto pathEndIter = pathView.end(); - if (pathBeginIter == pathEndIter) + AZStd::string_view rootNameView = pathView.RootName().Native(); + if (rootNameView.size() > pDstDrive->max_size()) { return false; } - AZ::IO::Path resultPath; - if (!bLastComponent) - { - // Removing leading path component - AZStd::advance(pathBeginIter, 1); - } - else - { - // Remove trailing path component - AZStd::advance(pathEndIter, -1); - } - for (; pathBeginIter != pathEndIter; ++pathBeginIter) - { - resultPath /= *pathBeginIter; - } - if (resultPath.empty()) + *pDstDrive = rootNameView; + } + if (pDstPath) + { + AZStd::string_view rootDirectoryView = pathView.RootDirectory().Native(); + AZStd::string_view relPathParentView = pathView.ParentPath().RelativePath().Native(); + if (rootDirectoryView.size() + relPathParentView.size() > pDstPath->max_size()) { return false; } - inout = AZStd::move(resultPath.Native()); + // Append the root directory if there is one + *pDstPath = rootDirectoryView; + // Append the relative path portion of the split path excluding the filename + *pDstPath += relPathParentView; + } + if (pDstName) + { + AZStd::string_view stemView = pathView.Stem().Native(); + if (stemView.size() > pDstName->max_size()) + { + return false; + } + *pDstName = stemView; + } + if (pDstExtension) + { + AZStd::string_view extensionView = pathView.Extension().Native(); + if (extensionView.size() > pDstExtension->max_size()) + { + return false; + } + *pDstExtension = extensionView; + } + + return true; + } + + bool Join(const char* pFirstPart, const char* pSecondPart, AZStd::string& out, [[maybe_unused]] bool bCaseInsensitive, bool bNormalize) + { + if (!pFirstPart || !pSecondPart) + { + return false; + } + + AZ::IO::PathView secondPath(pSecondPart); + AZ_Warning("StringFunc", secondPath.IsRelative(), "The second join parameter %s is an absolute path," + " this will replace the first part of the path resulting in an output of just the second part", + pSecondPart); + + AZ::IO::Path resultPath(pFirstPart); + resultPath /= secondPath; + out = bNormalize ? AZStd::move(resultPath.LexicallyNormal().Native()) : AZStd::move(resultPath.Native()); + return true; + } + + bool Join(const char* pFirstPart, const char* pSecondPart, FixedString& out, [[maybe_unused]] bool bCaseInsensitive, bool bNormalize) + { + if (!pFirstPart || !pSecondPart) + { + return false; + } + + AZ::IO::PathView secondPath(pSecondPart); + AZ_Warning("StringFunc", secondPath.IsRelative(), "The second join parameter %s is an absolute path," + " this will replace the first part of the path resulting in an output of just the second part", + pSecondPart); + + AZ::IO::FixedMaxPath resultPath(pFirstPart); + resultPath /= secondPath; + out = bNormalize ? AZStd::move(resultPath.LexicallyNormal().Native()) : AZStd::move(resultPath.Native()); + return true; + } + + bool HasDrive(const char* in, bool bCheckAllFileSystemFormats /*= false*/) + { + // no drive if empty + if (!in || in[0] == '\0') + { + return false; + } + AZ::IO::PathView pathView(in); + return pathView.HasRootName() || (bCheckAllFileSystemFormats && pathView.HasRootDirectory()); + } + + bool HasExtension(const char* in) + { + //it doesn't have an extension if it's empty + if (!in || in[0] == '\0') + { + return false; + } + + return AZ::IO::PathView(in).HasExtension(); + } + + bool IsExtension(const char* in, const char* pExtension, bool bCaseInsenitive /*= false*/) + { + //it doesn't have an extension if it's empty + if (!in || in[0] == '\0' || !pExtension || pExtension[0] == '\0') + { + return false; + } + + AZStd::string_view pathExtension = AZ::IO::PathView(in).Extension().Native(); + if (pathExtension.starts_with(AZ_FILESYSTEM_EXTENSION_SEPARATOR)) + { + pathExtension.remove_prefix(1); + } + AZStd::string_view extensionView(pExtension); + if (extensionView.starts_with(AZ_FILESYSTEM_EXTENSION_SEPARATOR)) + { + extensionView.remove_prefix(1); + } + + return AZStd::equal(pathExtension.begin(), pathExtension.end(), extensionView.begin(), extensionView.end(), + [bCaseInsenitive](const char lhs, const char rhs) + { + return !bCaseInsenitive ? lhs == rhs : tolower(lhs) == tolower(rhs); + }); + } + + bool IsRelative(const char* in) + { + //not relative if empty + if (!in || in[0] == '\0') + { + return false; + } + + return AZ::IO::PathView(in).IsRelative(); + } + + bool StripDrive(AZStd::string& inout) + { + AZ::IO::PathView pathView(inout); + AZ::IO::PathView rootNameView(pathView.RootName()); + if (!rootNameView.empty()) + { + inout.replace(0, rootNameView.Native().size(), ""); return true; } + return false; + } - bool GetDrive(const char* in, AZStd::string& out) + void StripPath(AZStd::string& inout) + { + inout = AZ::IO::PathView(inout).Filename().Native(); + } + + void StripFullName(AZStd::string& inout) + { + inout = AZ::IO::Path(AZStd::move(inout)).RemoveFilename().Native(); + } + + void StripExtension(AZStd::string& inout) + { + AZ::IO::Path path(AZStd::move(inout)); + path.ReplaceExtension(); + inout = AZStd::move(path.Native()); + } + + bool StripComponent(AZStd::string& inout, bool bLastComponent /* = false*/) + { + AZ::IO::PathView pathView(inout); + auto pathBeginIter = pathView.begin(); + auto pathEndIter = pathView.end(); + if (pathBeginIter == pathEndIter) { - if (!in || in[0] == '\0') - { - return false; - } + return false; + } + AZ::IO::Path resultPath; + if (!bLastComponent) + { + // Removing leading path component + AZStd::advance(pathBeginIter, 1); + } + else + { + // Remove trailing path component + AZStd::advance(pathEndIter, -1); + } + for (; pathBeginIter != pathEndIter; ++pathBeginIter) + { + resultPath /= *pathBeginIter; + } + if (resultPath.empty()) + { + return false; + } + inout = AZStd::move(resultPath.Native()); + return true; + } - out = AZ::IO::PathView(in).RootName().Native(); + bool GetDrive(const char* in, AZStd::string& out) + { + if (!in || in[0] == '\0') + { + return false; + } + + out = AZ::IO::PathView(in).RootName().Native(); + return !out.empty(); + } + + AZStd::optional GetParentDir(AZStd::string_view path) + { + if (path.empty()) + { + return {}; + } + + AZStd::string_view parentDir = AZ::IO::PathView(path).ParentPath().Native(); + return !parentDir.empty() ? AZStd::make_optional(parentDir) : AZStd::nullopt; + } + + bool GetFullPath(const char* in, AZStd::string& out) + { + if (!in || in[0] == '\0') + { + return false; + } + + out = AZ::IO::PathView(in).ParentPath().Native(); + return !out.empty(); + } + + bool GetFolderPath(const char* in, AZStd::string& out) + { + return GetFullPath(in, out); + } + + bool GetFolder(const char* in, AZStd::string& out, bool bFirst /* = false*/) + { + if (!in || in[0] == '\0') + { + return false; + } + + if (!bFirst) + { + out = AZ::IO::PathView(in).ParentPath().Filename().Native(); return !out.empty(); } - - AZStd::optional GetParentDir(AZStd::string_view path) + else { - if (path.empty()) - { - return {}; - } - - AZStd::string_view parentDir = AZ::IO::PathView(path).ParentPath().Native(); - return !parentDir.empty() ? AZStd::make_optional(parentDir) : AZStd::nullopt; - } - - bool GetFullPath(const char* in, AZStd::string& out) - { - if (!in || in[0] == '\0') - { - return false; - } - - out = AZ::IO::PathView(in).ParentPath().Native(); + AZStd::string_view relativePath = AZ::IO::PathView(in).RelativePath().Native(); + size_t nextSeparator = relativePath.find_first_of(AZ_CORRECT_FILESYSTEM_SEPARATOR); + out = nextSeparator != AZStd::string_view::npos ? relativePath.substr(0, nextSeparator) : relativePath; return !out.empty(); } + } - bool GetFolderPath(const char* in, AZStd::string& out) + bool GetFullFileName(const char* in, AZStd::string& out) + { + if (!in || in[0] == '\0') { - return GetFullPath(in, out); + return false; } - bool GetFolder(const char* in, AZStd::string& out, bool bFirst /* = false*/) - { - if (!in || in[0] == '\0') - { - return false; - } + out = AZ::IO::PathView(in).Filename().Native(); + return !out.empty(); + } - if (!bFirst) - { - out = AZ::IO::PathView(in).ParentPath().Filename().Native(); - return !out.empty(); - } - else - { - AZStd::string_view relativePath = AZ::IO::PathView(in).RelativePath().Native(); - size_t nextSeparator = relativePath.find_first_of(AZ_CORRECT_FILESYSTEM_SEPARATOR); - out = nextSeparator != AZStd::string_view::npos ? relativePath.substr(0, nextSeparator) : relativePath; - return !out.empty(); - } + bool GetFileName(const char* in, AZStd::string& out) + { + if (!in || in[0] == '\0') + { + return false; } - bool GetFullFileName(const char* in, AZStd::string& out) - { - if (!in || in[0] == '\0') - { - return false; - } + out = AZ::IO::PathView(in).Stem().Native(); + return !out.empty(); + } - out = AZ::IO::PathView(in).Filename().Native(); - return !out.empty(); + bool GetExtension(const char* in, AZStd::string& out, bool includeDot) + { + if (!in || in[0] == '\0') + { + return false; } - bool GetFileName(const char* in, AZStd::string& out) + AZStd::string_view extensionView = AZ::IO::PathView(in).Extension().Native(); + // PathView returns extensions with the character, so remove the + // if it is not included + if (!includeDot && extensionView.starts_with(AZ_FILESYSTEM_EXTENSION_SEPARATOR)) { - if (!in || in[0] == '\0') + extensionView.remove_prefix(1); + } + out = extensionView; + return !out.empty(); + } + + void ReplaceFullName(AZStd::string& inout, const char* pFileName /* = nullptr*/, const char* pFileExtension /* = nullptr*/) + { + //strip the full file name if it has one + AZ::IO::Path path(AZStd::move(inout)); + path.RemoveFilename(); + if (pFileName) + { + path /= pFileName; + } + if (pFileExtension) + { + path.ReplaceExtension(pFileExtension); + } + inout = AZStd::move(path.Native()); + } + + void ReplaceExtension(AZStd::string& inout, const char* newExtension /* = nullptr*/) + { + //treat this as a strip + if (!newExtension || newExtension[0] == '\0') + { + return; + } + AZ::IO::Path path(AZStd::move(inout)); + path.ReplaceExtension(newExtension); + inout = AZStd::move(path.Native()); + } + + AZStd::string& AppendSeparator(AZStd::string& inout) + { + if (inout.ends_with(AZ_WRONG_FILESYSTEM_SEPARATOR)) + { + inout.replace(inout.end() - 1, inout.end(), 1, AZ_CORRECT_FILESYSTEM_SEPARATOR); + } + else if (!inout.ends_with(AZ_CORRECT_FILESYSTEM_SEPARATOR)) + { + inout.append(1, AZ_CORRECT_FILESYSTEM_SEPARATOR); + } + return inout; + } + } // namespace Path + + namespace Json + { + /* + According to http://www.ecma-international.org/publications/files/ECMA-ST/ECMA-404.pdf: + A string is a sequence of Unicode code points wrapped with quotation marks (U+0022). All characters may be + placed within the quotation marks except for the characters that must be escaped: quotation mark (U+0022), + reverse solidus (U+005C), and the control characters U+0000 to U+001F. + */ + AZStd::string& ToEscapedString(AZStd::string& inout) + { + size_t strSize = inout.size(); + + for (size_t i = 0; i < strSize; ++i) + { + char character = inout[i]; + + // defaults to 1 if it hits any cases except default + size_t jumpChar = 1; + switch (character) { - return false; + case '"': + inout.insert(i, "\\"); + break; + + case '\\': + inout.insert(i, "\\"); + break; + + case '/': + inout.insert(i, "\\"); + break; + + case '\b': + inout.replace(i, i + 1, "\\b"); + break; + + case '\f': + inout.replace(i, i + 1, "\\f"); + break; + + case '\n': + inout.replace(i, i + 1, "\\n"); + break; + + case '\r': + inout.replace(i, i + 1, "\\r"); + break; + + case '\t': + inout.replace(i, i + 1, "\\t"); + break; + + default: + /* + Control characters U+0000 to U+001F may be represented as a six - character sequence : a reverse solidus, + followed by the lowercase letter u, followed by four hexadecimal digits that encode the code point. + */ + if (character >= '\x0000' && character <= '\x001f') + { + // jumping "\uXXXX" characters + jumpChar = 6; + + AZStd::string hexStr = AZStd::string::format("\\u%04x", static_cast(character)); + inout.replace(i, i + 1, hexStr); + } + else + { + jumpChar = 0; + } } - out = AZ::IO::PathView(in).Stem().Native(); - return !out.empty(); + i += jumpChar; + strSize += jumpChar; } - bool GetExtension(const char* in, AZStd::string& out, bool includeDot) - { - if (!in || in[0] == '\0') - { - return false; - } + return inout; + } + } // namespace Json - AZStd::string_view extensionView = AZ::IO::PathView(in).Extension().Native(); - // PathView returns extensions with the character, so remove the - // if it is not included - if (!includeDot && extensionView.starts_with(AZ_FILESYSTEM_EXTENSION_SEPARATOR)) - { - extensionView.remove_prefix(1); - } - out = extensionView; - return !out.empty(); - } + namespace Base64 + { + static const char base64pad = '='; - void ReplaceFullName(AZStd::string& inout, const char* pFileName /* = nullptr*/, const char* pFileExtension /* = nullptr*/) - { - //strip the full file name if it has one - AZ::IO::Path path(AZStd::move(inout)); - path.RemoveFilename(); - if (pFileName) - { - path /= pFileName; - } - if (pFileExtension) - { - path.ReplaceExtension(pFileExtension); - } - inout = AZStd::move(path.Native()); - } + static const char c_base64Table[] = + { + "ABCDEFGHIJKLMNOPQRSTUVWXYZ" + "abcdefghijklmnopqrstuvwxyz" + "0123456789+/" + }; - void ReplaceExtension(AZStd::string& inout, const char* newExtension /* = nullptr*/) - { - //treat this as a strip - if (!newExtension || newExtension[0] == '\0') - { - return; - } - AZ::IO::Path path(AZStd::move(inout)); - path.ReplaceExtension(newExtension); - inout = AZStd::move(path.Native()); - } + static const AZ::u8 c_inverseBase64Table[] = + { + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0x3e, 0xff, 0xff, 0xff, 0x3f, + 0x34, 0x35, 0x36, 0x37, 0x38, 0x39, 0x3a, 0x3b, 0x3c, 0x3d, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, 0x08, 0x09, 0x0a, 0x0b, 0x0c, 0x0d, 0x0e, + 0x0f, 0x10, 0x11, 0x12, 0x13, 0x14, 0x15, 0x16, 0x17, 0x18, 0x19, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0x1a, 0x1b, 0x1c, 0x1d, 0x1e, 0x1f, 0x20, 0x21, 0x22, 0x23, 0x24, 0x25, 0x26, 0x27, 0x28, + 0x29, 0x2a, 0x2b, 0x2c, 0x2d, 0x2e, 0x2f, 0x30, 0x31, 0x32, 0x33, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff + }; - AZStd::string& AppendSeparator(AZStd::string& inout) - { - if (inout.ends_with(AZ_WRONG_FILESYSTEM_SEPARATOR)) - { - inout.replace(inout.end() - 1, inout.end(), 1, AZ_CORRECT_FILESYSTEM_SEPARATOR); - } - else if (!inout.ends_with(AZ_CORRECT_FILESYSTEM_SEPARATOR)) - { - inout.append(1, AZ_CORRECT_FILESYSTEM_SEPARATOR); - } - return inout; - } - } // namespace Path + bool IsValidEncodedChar(const char encodedChar) + { + return c_inverseBase64Table[static_cast(encodedChar)] != 0xff; + } - namespace Json + AZStd::string Encode(const AZ::u8* in, const size_t size) { /* - According to http://www.ecma-international.org/publications/files/ECMA-ST/ECMA-404.pdf: - A string is a sequence of Unicode code points wrapped with quotation marks (U+0022). All characters may be - placed within the quotation marks except for the characters that must be escaped: quotation mark (U+0022), - reverse solidus (U+005C), and the control characters U+0000 to U+001F. + figure retrieved from the Base encoding rfc https://tools.ietf.org/html/rfc4648 + +--first octet--+-second octet--+--third octet--+ + |7 6 5 4 3 2 1 0|7 6 5 4 3 2 1 0|7 6 5 4 3 2 1 0| + +-----------+---+-------+-------+---+-----------+ + |5 4 3 2 1 0|5 4 3 2 1 0|5 4 3 2 1 0|5 4 3 2 1 0| + +--1.index--+--2.index--+--3.index--+--4.index--+ */ - AZStd::string& ToEscapedString(AZStd::string& inout) + AZStd::string result; + + const size_t remainder = size % 3; + const size_t alignEndSize = size - remainder; + const AZ::u8* encodeBuf = in; + size_t encodeIndex = 0; + for (; encodeIndex < alignEndSize; encodeIndex += 3) { - size_t strSize = inout.size(); - - for (size_t i = 0; i < strSize; ++i) - { - char character = inout[i]; - - // defaults to 1 if it hits any cases except default - size_t jumpChar = 1; - switch (character) - { - case '"': - inout.insert(i, "\\"); - break; - - case '\\': - inout.insert(i, "\\"); - break; - - case '/': - inout.insert(i, "\\"); - break; - - case '\b': - inout.replace(i, i + 1, "\\b"); - break; - - case '\f': - inout.replace(i, i + 1, "\\f"); - break; - - case '\n': - inout.replace(i, i + 1, "\\n"); - break; - - case '\r': - inout.replace(i, i + 1, "\\r"); - break; - - case '\t': - inout.replace(i, i + 1, "\\t"); - break; - - default: - /* - Control characters U+0000 to U+001F may be represented as a six - character sequence : a reverse solidus, - followed by the lowercase letter u, followed by four hexadecimal digits that encode the code point. - */ - if (character >= '\x0000' && character <= '\x001f') - { - // jumping "\uXXXX" characters - jumpChar = 6; - - AZStd::string hexStr = AZStd::string::format("\\u%04x", static_cast(character)); - inout.replace(i, i + 1, hexStr); - } - else - { - jumpChar = 0; - } - } - - i += jumpChar; - strSize += jumpChar; - } - - return inout; - } - } // namespace Json - - namespace Base64 - { - static const char base64pad = '='; - - static const char c_base64Table[] = - { - "ABCDEFGHIJKLMNOPQRSTUVWXYZ" - "abcdefghijklmnopqrstuvwxyz" - "0123456789+/" - }; - - static const AZ::u8 c_inverseBase64Table[] = - { - 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, - 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, - 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0x3e, 0xff, 0xff, 0xff, 0x3f, - 0x34, 0x35, 0x36, 0x37, 0x38, 0x39, 0x3a, 0x3b, 0x3c, 0x3d, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, - 0xff, 0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, 0x08, 0x09, 0x0a, 0x0b, 0x0c, 0x0d, 0x0e, - 0x0f, 0x10, 0x11, 0x12, 0x13, 0x14, 0x15, 0x16, 0x17, 0x18, 0x19, 0xff, 0xff, 0xff, 0xff, 0xff, - 0xff, 0x1a, 0x1b, 0x1c, 0x1d, 0x1e, 0x1f, 0x20, 0x21, 0x22, 0x23, 0x24, 0x25, 0x26, 0x27, 0x28, - 0x29, 0x2a, 0x2b, 0x2c, 0x2d, 0x2e, 0x2f, 0x30, 0x31, 0x32, 0x33, 0xff, 0xff, 0xff, 0xff, 0xff, - 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, - 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, - 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, - 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, - 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, - 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, - 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, - 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff - }; - - bool IsValidEncodedChar(const char encodedChar) - { - return c_inverseBase64Table[static_cast(encodedChar)] != 0xff; - } - - AZStd::string Encode(const AZ::u8* in, const size_t size) - { - /* - figure retrieved from the Base encoding rfc https://tools.ietf.org/html/rfc4648 - +--first octet--+-second octet--+--third octet--+ - |7 6 5 4 3 2 1 0|7 6 5 4 3 2 1 0|7 6 5 4 3 2 1 0| - +-----------+---+-------+-------+---+-----------+ - |5 4 3 2 1 0|5 4 3 2 1 0|5 4 3 2 1 0|5 4 3 2 1 0| - +--1.index--+--2.index--+--3.index--+--4.index--+ - */ - AZStd::string result; - - const size_t remainder = size % 3; - const size_t alignEndSize = size - remainder; - const AZ::u8* encodeBuf = in; - size_t encodeIndex = 0; - for (; encodeIndex < alignEndSize; encodeIndex += 3) - { - encodeBuf = &in[encodeIndex]; - - result.push_back(c_base64Table[(encodeBuf[0] & 0xfc) >> 2]); - result.push_back(c_base64Table[((encodeBuf[0] & 0x03) << 4) | ((encodeBuf[1] & 0xf0) >> 4)]); - result.push_back(c_base64Table[((encodeBuf[1] & 0x0f) << 2) | ((encodeBuf[2] & 0xc0) >> 6)]); - result.push_back(c_base64Table[encodeBuf[2] & 0x3f]); - } - encodeBuf = &in[encodeIndex]; - if (remainder == 2) - { - result.push_back(c_base64Table[(encodeBuf[0] & 0xfc) >> 2]); - result.push_back(c_base64Table[((encodeBuf[0] & 0x03) << 4) | ((encodeBuf[1] & 0xf0) >> 4)]); - result.push_back(c_base64Table[((encodeBuf[1] & 0x0f) << 2)]); - result.push_back(base64pad); - } - else if (remainder == 1) - { - result.push_back(c_base64Table[(encodeBuf[0] & 0xfc) >> 2]); - result.push_back(c_base64Table[(encodeBuf[0] & 0x03) << 4]); - result.push_back(base64pad); - result.push_back(base64pad); - } - return result; + result.push_back(c_base64Table[(encodeBuf[0] & 0xfc) >> 2]); + result.push_back(c_base64Table[((encodeBuf[0] & 0x03) << 4) | ((encodeBuf[1] & 0xf0) >> 4)]); + result.push_back(c_base64Table[((encodeBuf[1] & 0x0f) << 2) | ((encodeBuf[2] & 0xc0) >> 6)]); + result.push_back(c_base64Table[encodeBuf[2] & 0x3f]); } - bool Decode(AZStd::vector& out, const char* in, const size_t size) + encodeBuf = &in[encodeIndex]; + if (remainder == 2) { - if (size % 4 != 0) - { - AZ_Warning("StringFunc", size % 4 == 0, "Base 64 encoded data length must be multiple of 4"); - return false; - } - - AZStd::vector result; - result.reserve(size * 3 / 4); - const char* decodeBuf = in; - size_t decodeIndex = 0; - for (; decodeIndex < size; decodeIndex += 4) - { - decodeBuf = &in[decodeIndex]; - //Check if each character is a valid Base64 encoded character - { - // First Octet - if (!IsValidEncodedChar(decodeBuf[0])) - { - AZ_Warning("StringFunc", false, "Invalid Base64 encoded text at offset %tu", AZStd::distance(in, &decodeBuf[0])); - return false; - } - if (!IsValidEncodedChar(decodeBuf[1])) - { - AZ_Warning("StringFunc", false, "Invalid Base64 encoded text at offset %tu", AZStd::distance(in, &decodeBuf[1])); - return false; - } - - result.push_back((c_inverseBase64Table[static_cast(decodeBuf[0])] << 2) | ((c_inverseBase64Table[static_cast(decodeBuf[1])] & 0x30) >> 4)); - } - - { - // Second Octet - if (decodeBuf[2] == base64pad) - { - break; - } - - if (!IsValidEncodedChar(decodeBuf[2])) - { - AZ_Warning("StringFunc", false, "Invalid Base64 encoded text at offset %tu", AZStd::distance(in, &decodeBuf[2])); - return false; - } - - result.push_back(((c_inverseBase64Table[static_cast(decodeBuf[1])] & 0x0f) << 4) | ((c_inverseBase64Table[static_cast(decodeBuf[2])] & 0x3c) >> 2)); - } - - { - // Third Octet - if (decodeBuf[3] == base64pad) - { - break; - } - - if (!IsValidEncodedChar(decodeBuf[3])) - { - AZ_Warning("StringFunc", false, "Invalid Base64 encoded text at offset %tu", AZStd::distance(in, &decodeBuf[3])); - return false; - } - - result.push_back(((c_inverseBase64Table[static_cast(decodeBuf[2])] & 0x03) << 6) | (c_inverseBase64Table[static_cast(decodeBuf[3])] & 0x3f)); - } - } - - out = AZStd::move(result); - return true; + result.push_back(c_base64Table[(encodeBuf[0] & 0xfc) >> 2]); + result.push_back(c_base64Table[((encodeBuf[0] & 0x03) << 4) | ((encodeBuf[1] & 0xf0) >> 4)]); + result.push_back(c_base64Table[((encodeBuf[1] & 0x0f) << 2)]); + result.push_back(base64pad); } + else if (remainder == 1) + { + result.push_back(c_base64Table[(encodeBuf[0] & 0xfc) >> 2]); + result.push_back(c_base64Table[(encodeBuf[0] & 0x03) << 4]); + result.push_back(base64pad); + result.push_back(base64pad); + } + + return result; } - namespace Utf8 + bool Decode(AZStd::vector& out, const char* in, const size_t size) { - bool CheckNonAsciiChar(const AZStd::string& in) + if (size % 4 != 0) { - for (int i = 0; i < in.length(); ++i) - { - char byte = in[i]; - if (byte & 0x80) - { - return true; - } - } + AZ_Warning("StringFunc", size % 4 == 0, "Base 64 encoded data length must be multiple of 4"); return false; } + + AZStd::vector result; + result.reserve(size * 3 / 4); + const char* decodeBuf = in; + size_t decodeIndex = 0; + for (; decodeIndex < size; decodeIndex += 4) + { + decodeBuf = &in[decodeIndex]; + //Check if each character is a valid Base64 encoded character + { + // First Octet + if (!IsValidEncodedChar(decodeBuf[0])) + { + AZ_Warning("StringFunc", false, "Invalid Base64 encoded text at offset %tu", AZStd::distance(in, &decodeBuf[0])); + return false; + } + if (!IsValidEncodedChar(decodeBuf[1])) + { + AZ_Warning("StringFunc", false, "Invalid Base64 encoded text at offset %tu", AZStd::distance(in, &decodeBuf[1])); + return false; + } + + result.push_back((c_inverseBase64Table[static_cast(decodeBuf[0])] << 2) | ((c_inverseBase64Table[static_cast(decodeBuf[1])] & 0x30) >> 4)); + } + + { + // Second Octet + if (decodeBuf[2] == base64pad) + { + break; + } + + if (!IsValidEncodedChar(decodeBuf[2])) + { + AZ_Warning("StringFunc", false, "Invalid Base64 encoded text at offset %tu", AZStd::distance(in, &decodeBuf[2])); + return false; + } + + result.push_back(((c_inverseBase64Table[static_cast(decodeBuf[1])] & 0x0f) << 4) | ((c_inverseBase64Table[static_cast(decodeBuf[2])] & 0x3c) >> 2)); + } + + { + // Third Octet + if (decodeBuf[3] == base64pad) + { + break; + } + + if (!IsValidEncodedChar(decodeBuf[3])) + { + AZ_Warning("StringFunc", false, "Invalid Base64 encoded text at offset %tu", AZStd::distance(in, &decodeBuf[3])); + return false; + } + + result.push_back(((c_inverseBase64Table[static_cast(decodeBuf[2])] & 0x03) << 6) | (c_inverseBase64Table[static_cast(decodeBuf[3])] & 0x3f)); + } + } + + out = AZStd::move(result); + return true; } - } // namespace StringFunc -} // namespace AZ + } + + namespace Utf8 + { + bool CheckNonAsciiChar(const AZStd::string& in) + { + for (int i = 0; i < in.length(); ++i) + { + char byte = in[i]; + if (byte & 0x80) + { + return true; + } + } + return false; + } + } +} // namespace AZ::StringFunc diff --git a/Code/Framework/AzCore/AzCore/std/string/memorytoascii.cpp b/Code/Framework/AzCore/AzCore/std/string/memorytoascii.cpp index 8f2d5ddf25..21f3a1c5f4 100644 --- a/Code/Framework/AzCore/AzCore/std/string/memorytoascii.cpp +++ b/Code/Framework/AzCore/AzCore/std/string/memorytoascii.cpp @@ -8,179 +8,176 @@ #include -namespace AZStd +namespace AZStd::MemoryToASCII { - namespace MemoryToASCII + AZStd::string ToString(const void* memoryAddrs, AZStd::size_t dataSize, AZStd::size_t maxShowSize, AZStd::size_t dataWidth/*=16*/, Options format/*=Options::Default*/) { - AZStd::string ToString(const void* memoryAddrs, AZStd::size_t dataSize, AZStd::size_t maxShowSize, AZStd::size_t dataWidth/*=16*/, Options format/*=Options::Default*/) + AZStd::string output; + + if ((memoryAddrs != nullptr) && (dataSize > 0)) { - AZStd::string output; + const AZ::u8 *data = reinterpret_cast(memoryAddrs); - if ((memoryAddrs != nullptr) && (dataSize > 0)) + if (static_cast(format) != 0) { - const AZ::u8 *data = reinterpret_cast(memoryAddrs); + output.reserve(8162); - if (static_cast(format) != 0) + bool showHeader = static_cast(format) & static_cast(Options::Header) ? true : false; + bool showOffset = static_cast(format) & static_cast(Options::Offset) ? true : false; + bool showBinary = static_cast(format) & static_cast(Options::Binary) ? true : false; + bool showASCII = static_cast(format) & static_cast(Options::ASCII) ? true : false; + bool showInfo = static_cast(format) & static_cast(Options::Info) ? true : false; + + // Because of the auto formatting for the headers, the min width is 3 + if (dataWidth < 3) { - output.reserve(8162); + dataWidth = 3; + } - bool showHeader = static_cast(format) & static_cast(Options::Header) ? true : false; - bool showOffset = static_cast(format) & static_cast(Options::Offset) ? true : false; - bool showBinary = static_cast(format) & static_cast(Options::Binary) ? true : false; - bool showASCII = static_cast(format) & static_cast(Options::ASCII) ? true : false; - bool showInfo = static_cast(format) & static_cast(Options::Info) ? true : false; + if (showHeader) + { + AZStd::string line1; + AZStd::string line2; + line1.reserve(1024); + line2.reserve(1024); - // Because of the auto formatting for the headers, the min width is 3 - if (dataWidth < 3) + if (showOffset) { - dataWidth = 3; + line1 += "Offset"; + line2 += "------"; + + if (showBinary || showASCII) + { + line1 += " "; + line2 += " "; + } } - if (showHeader) + if (showBinary) { - AZStd::string line1; - AZStd::string line2; - line1.reserve(1024); - line2.reserve(1024); + static const char *kHeaderName = "Data"; + static AZStd::size_t kHeaderNameSize = 4; - if (showOffset) + AZStd::size_t lineLength = (dataWidth * 3) - 1; + AZStd::size_t numPreSpaces = (lineLength - kHeaderNameSize) / 2; + AZStd::size_t numPostSpaces = lineLength - numPreSpaces - kHeaderNameSize; + + line1 += AZStd::string(numPreSpaces, ' ') + kHeaderName + AZStd::string(numPostSpaces, ' '); + //line2 += AZStd::string(lineLength, '-'); + for(size_t i=0; i 0) { - line1 += " "; - line2 += " "; - } - } - - if (showBinary) - { - static const char *kHeaderName = "Data"; - static AZStd::size_t kHeaderNameSize = 4; - - AZStd::size_t lineLength = (dataWidth * 3) - 1; - AZStd::size_t numPreSpaces = (lineLength - kHeaderNameSize) / 2; - AZStd::size_t numPostSpaces = lineLength - numPreSpaces - kHeaderNameSize; - - line1 += AZStd::string(numPreSpaces, ' ') + kHeaderName + AZStd::string(numPostSpaces, ' '); - //line2 += AZStd::string(lineLength, '-'); - for(size_t i=0; i 0) - { - line2 += "-"; - } - - line2 += AZStd::string::format("%02zx", i); + line2 += "-"; } - if (showASCII) - { - line1 += " "; - line2 += " "; - } + line2 += AZStd::string::format("%02zx", i); } if (showASCII) { - static const char *kHeaderName = "ASCII"; - static AZStd::size_t kHeaderNameSize = 5; - - AZStd::size_t numPreSpaces = (dataWidth - kHeaderNameSize) / 2; - AZStd::size_t numPostSpaces = dataWidth - numPreSpaces - kHeaderNameSize; - - line1 += AZStd::string(numPreSpaces, ' ') + kHeaderName + AZStd::string(numPostSpaces, ' '); - line2 += AZStd::string(dataWidth, '-'); + line1 += " "; + line2 += " "; } - - if (showInfo) - { - output += AZStd::string::format("Address: 0x%p Data Size:%zu Max Size:%zu\n", data, dataSize, maxShowSize); - } - - output += line1 + "\n"; - output += line2 + "\n"; } - AZStd::size_t offset = 0; - AZStd::size_t maxSize = dataSize > maxShowSize ? maxShowSize : dataSize; - - while (offset < maxSize) + if (showASCII) { - if (showOffset) - { - output += AZStd::string::format("%06zx", offset); + static const char *kHeaderName = "ASCII"; + static AZStd::size_t kHeaderNameSize = 5; - if (showBinary || showASCII) + AZStd::size_t numPreSpaces = (dataWidth - kHeaderNameSize) / 2; + AZStd::size_t numPostSpaces = dataWidth - numPreSpaces - kHeaderNameSize; + + line1 += AZStd::string(numPreSpaces, ' ') + kHeaderName + AZStd::string(numPostSpaces, ' '); + line2 += AZStd::string(dataWidth, '-'); + } + + if (showInfo) + { + output += AZStd::string::format("Address: 0x%p Data Size:%zu Max Size:%zu\n", data, dataSize, maxShowSize); + } + + output += line1 + "\n"; + output += line2 + "\n"; + } + + AZStd::size_t offset = 0; + AZStd::size_t maxSize = dataSize > maxShowSize ? maxShowSize : dataSize; + + while (offset < maxSize) + { + if (showOffset) + { + output += AZStd::string::format("%06zx", offset); + + if (showBinary || showASCII) + { + output += " "; + } + } + + if (showBinary) + { + AZStd::string binLine; + binLine.reserve((dataWidth * 3) * 2); + + for (AZStd::size_t index = 0; index < dataWidth; index++) + { + if (!binLine.empty()) { - output += " "; + binLine += " "; + } + + if ((offset + index) < maxSize) + { + binLine += AZStd::string::format("%02x", data[offset + index]); + } + else + { + binLine += " "; } } - if (showBinary) - { - AZStd::string binLine; - binLine.reserve((dataWidth * 3) * 2); - - for (AZStd::size_t index = 0; index < dataWidth; index++) - { - if (!binLine.empty()) - { - binLine += " "; - } - - if ((offset + index) < maxSize) - { - binLine += AZStd::string::format("%02x", data[offset + index]); - } - else - { - binLine += " "; - } - } - - output += binLine; - - if (showASCII) - { - output += " "; - } - } + output += binLine; if (showASCII) { - AZStd::string asciiLine; - asciiLine.reserve(dataWidth * 2); + output += " "; + } + } - for (AZStd::size_t index = 0; index < dataWidth; index++) + if (showASCII) + { + AZStd::string asciiLine; + asciiLine.reserve(dataWidth * 2); + + for (AZStd::size_t index = 0; index < dataWidth; index++) + { + if ((offset + index) > maxSize) { - if ((offset + index) > maxSize) - { - break; - } - else - { - char value = static_cast(data[offset + index]); - - if ((value < 32) || (value > 127)) - value = ' '; - - asciiLine += value; - } + break; } + else + { + char value = static_cast(data[offset + index]); - output += asciiLine; + if ((value < 32) || (value > 127)) + value = ' '; + + asciiLine += value; + } } - output += "\n"; - offset += dataWidth; + output += asciiLine; } + + output += "\n"; + offset += dataWidth; } } - - return output; } - } // namespace MemoryToASCII -} // namespace AZStd + + return output; + } +} // namespace AZStd::MemoryToASCII diff --git a/Code/Framework/AzCore/Platform/Common/UnixLike/AzCore/Debug/Trace_UnixLike.cpp b/Code/Framework/AzCore/Platform/Common/UnixLike/AzCore/Debug/Trace_UnixLike.cpp index f4d7db802b..92a80b0d9a 100644 --- a/Code/Framework/AzCore/Platform/Common/UnixLike/AzCore/Debug/Trace_UnixLike.cpp +++ b/Code/Framework/AzCore/Platform/Common/UnixLike/AzCore/Debug/Trace_UnixLike.cpp @@ -13,73 +13,67 @@ #include #include -namespace AZ +namespace AZ::Debug::Platform { - namespace Debug - { - namespace Platform - { #if defined(AZ_ENABLE_DEBUG_TOOLS) - bool performDebuggerDetection() + bool performDebuggerDetection() + { + AZ::IO::SystemFile processStatusFile; + if (!processStatusFile.Open("/proc/self/status", AZ::IO::SystemFile::SF_OPEN_READ_ONLY)) + { + return false; + } + + char buffer[4096]; + AZ::IO::SystemFile::SizeType numRead = processStatusFile.Read(sizeof(buffer), buffer); + + const AZStd::string_view processStatusView(buffer, buffer + numRead); + constexpr AZStd::string_view tracerPidString = "TracerPid:"; + const size_t tracerPidOffset = processStatusView.find(tracerPidString); + if (tracerPidOffset == AZStd::string_view::npos) + { + return false; + } + for (size_t i = tracerPidOffset + tracerPidString.length(); i < numRead; ++i) + { + if (!::isspace(processStatusView[i])) { - AZ::IO::SystemFile processStatusFile; - if (!processStatusFile.Open("/proc/self/status", AZ::IO::SystemFile::SF_OPEN_READ_ONLY)) - { - return false; - } - - char buffer[4096]; - AZ::IO::SystemFile::SizeType numRead = processStatusFile.Read(sizeof(buffer), buffer); - - const AZStd::string_view processStatusView(buffer, buffer + numRead); - constexpr AZStd::string_view tracerPidString = "TracerPid:"; - const size_t tracerPidOffset = processStatusView.find(tracerPidString); - if (tracerPidOffset == AZStd::string_view::npos) - { - return false; - } - for (size_t i = tracerPidOffset + tracerPidString.length(); i < numRead; ++i) - { - if (!::isspace(processStatusView[i])) - { - return processStatusView[i] != '0'; - } - } - return false; - } - - bool IsDebuggerPresent() - { - static bool s_detectionPerformed = false; - static bool s_debuggerDetected = false; - if (!s_detectionPerformed) - { - s_debuggerDetected = performDebuggerDetection(); - s_detectionPerformed = true; - } - return s_debuggerDetected; - } - - bool AttachDebugger() - { - // Not supported yet - AZ_Assert(false, "AttachDebugger() is not supported for Unix platform yet"); - return false; - } - - void HandleExceptions(bool) - {} - - void DebugBreak() - { - raise(SIGINT); - } -#endif // AZ_ENABLE_DEBUG_TOOLS - - void Terminate(int exitCode) - { - _exit(exitCode); + return processStatusView[i] != '0'; } } + return false; } -} + + bool IsDebuggerPresent() + { + static bool s_detectionPerformed = false; + static bool s_debuggerDetected = false; + if (!s_detectionPerformed) + { + s_debuggerDetected = performDebuggerDetection(); + s_detectionPerformed = true; + } + return s_debuggerDetected; + } + + bool AttachDebugger() + { + // Not supported yet + AZ_Assert(false, "AttachDebugger() is not supported for Unix platform yet"); + return false; + } + + void HandleExceptions(bool) + {} + + void DebugBreak() + { + raise(SIGINT); + } +#endif // AZ_ENABLE_DEBUG_TOOLS + + void Terminate(int exitCode) + { + _exit(exitCode); + } +} // namespace AZ::Debug::Platform diff --git a/Code/Framework/AzCore/Platform/Common/UnixLike/AzCore/IO/Internal/SystemFileUtils_UnixLike.cpp b/Code/Framework/AzCore/Platform/Common/UnixLike/AzCore/IO/Internal/SystemFileUtils_UnixLike.cpp index 07614ad56b..4c605f6154 100644 --- a/Code/Framework/AzCore/Platform/Common/UnixLike/AzCore/IO/Internal/SystemFileUtils_UnixLike.cpp +++ b/Code/Framework/AzCore/Platform/Common/UnixLike/AzCore/IO/Internal/SystemFileUtils_UnixLike.cpp @@ -8,82 +8,76 @@ #include "SystemFileUtils_UnixLike.h" -namespace AZ +namespace AZ::IO::Internal { - namespace IO + bool FormatAndPeelOffWildCardExtension(const char* sourcePath, char* filePath, size_t filePathSize, char* extensionPath, size_t extensionSize, bool keepWildcard) { - namespace Internal + if (sourcePath == nullptr || filePath == nullptr || extensionPath == nullptr || filePathSize == 0 || extensionSize == 0) { - bool FormatAndPeelOffWildCardExtension(const char* sourcePath, char* filePath, size_t filePathSize, char* extensionPath, size_t extensionSize, bool keepWildcard) + AZ_Error("AZ::IO::Internal", false, "FormatAndPeelOffWildCardExtension: One or more parameters was invalid."); + return false; + } + const char* pSrcPath = sourcePath; + char* pDestPath = filePath; + size_t destinationSize = filePathSize; + unsigned numFileChars = 0; + unsigned numExtensionChars = 0; + unsigned* pNumDestChars = &numFileChars; + bool bIsWildcardExtension = false; + while (*pSrcPath) + { + char srcChar = *pSrcPath++; + + // Skip '*' and '.' + if ((!bIsWildcardExtension && srcChar != '*') || (bIsWildcardExtension && srcChar != '.' && (keepWildcard || srcChar != '*'))) { - if (sourcePath == nullptr || filePath == nullptr || extensionPath == nullptr || filePathSize == 0 || extensionSize == 0) + unsigned numChars = *pNumDestChars; + pDestPath[numChars++] = srcChar; + *pNumDestChars = numChars; + + --destinationSize; + if (destinationSize == 0) { - AZ_Error("AZ::IO::Internal", false, "FormatAndPeelOffWildCardExtension: One or more parameters was invalid."); + AZ_Error( + "AZ::IO::Internal", + false, + "Error splitting sourcePath '%s' into filePath and extension, %s length is larger than storage size %d.", + sourcePath, + bIsWildcardExtension ? "extensionPath" : "filePath", + bIsWildcardExtension ? extensionSize : filePathSize); return false; } - const char* pSrcPath = sourcePath; - char* pDestPath = filePath; - size_t destinationSize = filePathSize; - unsigned numFileChars = 0; - unsigned numExtensionChars = 0; - unsigned* pNumDestChars = &numFileChars; - bool bIsWildcardExtension = false; - while (*pSrcPath) + } + // Wild-card extension is separate + if (srcChar == '*') + { + bIsWildcardExtension = true; + pDestPath = extensionPath; + destinationSize = extensionSize; + pNumDestChars = &numExtensionChars; + if (keepWildcard) { - char srcChar = *pSrcPath++; + unsigned numChars = *pNumDestChars; + pDestPath[numChars++] = srcChar; + *pNumDestChars = numChars; - // Skip '*' and '.' - if ((!bIsWildcardExtension && srcChar != '*') || (bIsWildcardExtension && srcChar != '.' && (keepWildcard || srcChar != '*'))) + --destinationSize; + if (destinationSize == 0) { - unsigned numChars = *pNumDestChars; - pDestPath[numChars++] = srcChar; - *pNumDestChars = numChars; - - --destinationSize; - if (destinationSize == 0) - { - AZ_Error( - "AZ::IO::Internal", - false, - "Error splitting sourcePath '%s' into filePath and extension, %s length is larger than storage size %d.", - sourcePath, - bIsWildcardExtension ? "extensionPath" : "filePath", - bIsWildcardExtension ? extensionSize : filePathSize); - return false; - } - } - // Wild-card extension is separate - if (srcChar == '*') - { - bIsWildcardExtension = true; - pDestPath = extensionPath; - destinationSize = extensionSize; - pNumDestChars = &numExtensionChars; - if (keepWildcard) - { - unsigned numChars = *pNumDestChars; - pDestPath[numChars++] = srcChar; - *pNumDestChars = numChars; - - --destinationSize; - if (destinationSize == 0) - { - AZ_Error( - "AZ::IO::Internal", - false, - "Error splitting sourcePath '%s' into filePath and extension, extensionPath length is larger than storage size %d.", - sourcePath, - extensionSize); - return false; - } - } + AZ_Error( + "AZ::IO::Internal", + false, + "Error splitting sourcePath '%s' into filePath and extension, extensionPath length is larger than storage size %d.", + sourcePath, + extensionSize); + return false; } } - // Close strings - filePath[numFileChars] = 0; - extensionPath[numExtensionChars] = 0; - return true; } } + // Close strings + filePath[numFileChars] = 0; + extensionPath[numExtensionChars] = 0; + return true; } -} +} // namespace AZ::IO::Internal diff --git a/Code/Framework/AzCore/Platform/Common/UnixLike/AzCore/Platform_UnixLike.cpp b/Code/Framework/AzCore/Platform/Common/UnixLike/AzCore/Platform_UnixLike.cpp index bdc2e753be..4a316bcde6 100644 --- a/Code/Framework/AzCore/Platform/Common/UnixLike/AzCore/Platform_UnixLike.cpp +++ b/Code/Framework/AzCore/Platform/Common/UnixLike/AzCore/Platform_UnixLike.cpp @@ -14,31 +14,28 @@ #include #include -namespace AZ +namespace AZ::Platform { - namespace Platform + ProcessId GetCurrentProcessId() { - ProcessId GetCurrentProcessId() - { - return static_cast(::getpid()); - } + return static_cast(::getpid()); + } - MachineId GetLocalMachineId() + MachineId GetLocalMachineId() + { + if (s_machineId == 0) { + // In specialized server situations, SetLocalMachineId() should be used, with whatever criteria works best in that environment + // A proper implementation for each supported system will be needed instead of this temporary measure to avoid collision in the small scale. + // On a larger scale, the odds of two people getting in here at the same millisecond will go up drastically, and we'll have the same issue again, + // though far less reproducible, for duplicated EntityId's across a network. + s_machineId = static_cast(AZStd::GetTimeUTCMilliSecond() & 0xffffffff); if (s_machineId == 0) { - // In specialized server situations, SetLocalMachineId() should be used, with whatever criteria works best in that environment - // A proper implementation for each supported system will be needed instead of this temporary measure to avoid collision in the small scale. - // On a larger scale, the odds of two people getting in here at the same millisecond will go up drastically, and we'll have the same issue again, - // though far less reproducible, for duplicated EntityId's across a network. - s_machineId = static_cast(AZStd::GetTimeUTCMilliSecond() & 0xffffffff); - if (s_machineId == 0) - { - s_machineId = 1; - AZ_Warning("System", false, "0 machine ID is reserved!"); - } + s_machineId = 1; + AZ_Warning("System", false, "0 machine ID is reserved!"); } - return s_machineId; } - } // namespace Platform -} + return s_machineId; + } +} // namespace AZ::Platform diff --git a/Code/Framework/AzCore/Platform/Common/UnixLike/AzCore/Socket/AzSocket_UnixLike.cpp b/Code/Framework/AzCore/Platform/Common/UnixLike/AzCore/Socket/AzSocket_UnixLike.cpp index 5b35b13d8b..555d70f1e6 100644 --- a/Code/Framework/AzCore/Platform/Common/UnixLike/AzCore/Socket/AzSocket_UnixLike.cpp +++ b/Code/Framework/AzCore/Platform/Common/UnixLike/AzCore/Socket/AzSocket_UnixLike.cpp @@ -18,365 +18,362 @@ #define INVALID_SOCKET (-1) #define closesocket(_s) close(_s) #define GetInternalSocketError errno -typedef int SOCKET; -typedef AZ::u32 AZSOCKLEN; +using SOCKET = int; +using AZSOCKLEN = AZ::u32; -namespace AZ +namespace AZ::AzSock { - namespace AzSock + AZ::s32 TranslateOSError(AZ::s32 oserror) { - AZ::s32 TranslateOSError(AZ::s32 oserror) - { - AZ::s32 error; + AZ::s32 error; #define TRANSLATE(_from, _to) case (_from): error = static_cast(_to); break; - switch (oserror) - { - TRANSLATE(0, AzSockError::eASE_NO_ERROR); - TRANSLATE(EACCES, AzSockError::eASE_EACCES); - TRANSLATE(EADDRINUSE, AzSockError::eASE_EADDRINUSE); - TRANSLATE(EADDRNOTAVAIL, AzSockError::eASE_EADDRNOTAVAIL); - TRANSLATE(EAFNOSUPPORT, AzSockError::eASE_EAFNOSUPPORT); - TRANSLATE(EALREADY, AzSockError::eASE_EALREADY); - TRANSLATE(EBADF, AzSockError::eASE_EBADF); - TRANSLATE(ECONNABORTED, AzSockError::eASE_ECONNABORTED); - TRANSLATE(ECONNREFUSED, AzSockError::eASE_ECONNREFUSED); - TRANSLATE(ECONNRESET, AzSockError::eASE_ECONNRESET); - TRANSLATE(EFAULT, AzSockError::eASE_EFAULT); - TRANSLATE(EHOSTDOWN, AzSockError::eASE_EHOSTDOWN); - TRANSLATE(EINPROGRESS, AzSockError::eASE_EINPROGRESS); - TRANSLATE(EINTR, AzSockError::eASE_EINTR); - TRANSLATE(EINVAL, AzSockError::eASE_EINVAL); - TRANSLATE(EISCONN, AzSockError::eASE_EISCONN); - TRANSLATE(EMFILE, AzSockError::eASE_EMFILE); - TRANSLATE(EMSGSIZE, AzSockError::eASE_EMSGSIZE); - TRANSLATE(ENETUNREACH, AzSockError::eASE_ENETUNREACH); - TRANSLATE(ENOBUFS, AzSockError::eASE_ENOBUFS); - TRANSLATE(ENOPROTOOPT, AzSockError::eASE_ENOPROTOOPT); - TRANSLATE(ENOTCONN, AzSockError::eASE_ENOTCONN); - TRANSLATE(EOPNOTSUPP, AzSockError::eASE_EOPNOTSUPP); - TRANSLATE(EPROTONOSUPPORT, AzSockError::eASE_EAFNOSUPPORT); - TRANSLATE(ETIMEDOUT, AzSockError::eASE_ETIMEDOUT); - TRANSLATE(ETOOMANYREFS, AzSockError::eASE_ETOOMANYREFS); - TRANSLATE(EWOULDBLOCK, AzSockError::eASE_EWOULDBLOCK); + switch (oserror) + { + TRANSLATE(0, AzSockError::eASE_NO_ERROR); + TRANSLATE(EACCES, AzSockError::eASE_EACCES); + TRANSLATE(EADDRINUSE, AzSockError::eASE_EADDRINUSE); + TRANSLATE(EADDRNOTAVAIL, AzSockError::eASE_EADDRNOTAVAIL); + TRANSLATE(EAFNOSUPPORT, AzSockError::eASE_EAFNOSUPPORT); + TRANSLATE(EALREADY, AzSockError::eASE_EALREADY); + TRANSLATE(EBADF, AzSockError::eASE_EBADF); + TRANSLATE(ECONNABORTED, AzSockError::eASE_ECONNABORTED); + TRANSLATE(ECONNREFUSED, AzSockError::eASE_ECONNREFUSED); + TRANSLATE(ECONNRESET, AzSockError::eASE_ECONNRESET); + TRANSLATE(EFAULT, AzSockError::eASE_EFAULT); + TRANSLATE(EHOSTDOWN, AzSockError::eASE_EHOSTDOWN); + TRANSLATE(EINPROGRESS, AzSockError::eASE_EINPROGRESS); + TRANSLATE(EINTR, AzSockError::eASE_EINTR); + TRANSLATE(EINVAL, AzSockError::eASE_EINVAL); + TRANSLATE(EISCONN, AzSockError::eASE_EISCONN); + TRANSLATE(EMFILE, AzSockError::eASE_EMFILE); + TRANSLATE(EMSGSIZE, AzSockError::eASE_EMSGSIZE); + TRANSLATE(ENETUNREACH, AzSockError::eASE_ENETUNREACH); + TRANSLATE(ENOBUFS, AzSockError::eASE_ENOBUFS); + TRANSLATE(ENOPROTOOPT, AzSockError::eASE_ENOPROTOOPT); + TRANSLATE(ENOTCONN, AzSockError::eASE_ENOTCONN); + TRANSLATE(EOPNOTSUPP, AzSockError::eASE_EOPNOTSUPP); + TRANSLATE(EPROTONOSUPPORT, AzSockError::eASE_EAFNOSUPPORT); + TRANSLATE(ETIMEDOUT, AzSockError::eASE_ETIMEDOUT); + TRANSLATE(ETOOMANYREFS, AzSockError::eASE_ETOOMANYREFS); + TRANSLATE(EWOULDBLOCK, AzSockError::eASE_EWOULDBLOCK); - default: - AZ_TracePrintf("AzSock", "AzSocket could not translate OS error code %x, treating as miscellaneous.\n", oserror); - error = static_cast(AzSockError::eASE_MISC_ERROR); - break; - } + default: + AZ_TracePrintf("AzSock", "AzSocket could not translate OS error code %x, treating as miscellaneous.\n", oserror); + error = static_cast(AzSockError::eASE_MISC_ERROR); + break; + } #undef TRANSLATE - return error; - } + return error; + } - AZ::s32 TranslateSocketOption(AzSocketOption opt) - { - AZ::s32 value; + AZ::s32 TranslateSocketOption(AzSocketOption opt) + { + AZ::s32 value; #define TRANSLATE(_from, _to) case (_from): value = (_to); break; - switch (opt) - { - TRANSLATE(AzSocketOption::REUSEADDR, SO_REUSEADDR); - TRANSLATE(AzSocketOption::KEEPALIVE, SO_KEEPALIVE); - TRANSLATE(AzSocketOption::LINGER, SO_LINGER); + switch (opt) + { + TRANSLATE(AzSocketOption::REUSEADDR, SO_REUSEADDR); + TRANSLATE(AzSocketOption::KEEPALIVE, SO_KEEPALIVE); + TRANSLATE(AzSocketOption::LINGER, SO_LINGER); - default: - AZ_TracePrintf("AzSock", "AzSocket option %x not yet supported", opt); - value = 0; - break; - } + default: + AZ_TracePrintf("AzSock", "AzSocket option %x not yet supported", opt); + value = 0; + break; + } #undef TRANSLATE - return value; - } + return value; + } - AZSOCKET HandleInvalidSocket(SOCKET sock) + AZSOCKET HandleInvalidSocket(SOCKET sock) + { + AZSOCKET azsock = static_cast(sock); + if (sock == INVALID_SOCKET) { - AZSOCKET azsock = static_cast(sock); - if (sock == INVALID_SOCKET) - { - azsock = TranslateOSError(GetInternalSocketError); - } - return azsock; + azsock = TranslateOSError(GetInternalSocketError); } + return azsock; + } - AZ::s32 HandleSocketError(AZ::s32 socketError) + AZ::s32 HandleSocketError(AZ::s32 socketError) + { + if (socketError == SOCKET_ERROR) { - if (socketError == SOCKET_ERROR) - { - socketError = TranslateOSError(GetInternalSocketError); - } - return socketError; + socketError = TranslateOSError(GetInternalSocketError); } + return socketError; + } - const char* GetStringForError(AZ::s32 errorNumber) - { - AzSockError errorCode = AzSockError(errorNumber); + const char* GetStringForError(AZ::s32 errorNumber) + { + AzSockError errorCode = AzSockError(errorNumber); #define CASE_RETSTRING(errorEnum) case errorEnum: { return #errorEnum; } - switch (errorCode) - { - CASE_RETSTRING(AzSockError::eASE_NO_ERROR); - CASE_RETSTRING(AzSockError::eASE_SOCKET_INVALID); - CASE_RETSTRING(AzSockError::eASE_EACCES); - CASE_RETSTRING(AzSockError::eASE_EADDRINUSE); - CASE_RETSTRING(AzSockError::eASE_EADDRNOTAVAIL); - CASE_RETSTRING(AzSockError::eASE_EAFNOSUPPORT); - CASE_RETSTRING(AzSockError::eASE_EALREADY); - CASE_RETSTRING(AzSockError::eASE_EBADF); - CASE_RETSTRING(AzSockError::eASE_ECONNABORTED); - CASE_RETSTRING(AzSockError::eASE_ECONNREFUSED); - CASE_RETSTRING(AzSockError::eASE_ECONNRESET); - CASE_RETSTRING(AzSockError::eASE_EFAULT); - CASE_RETSTRING(AzSockError::eASE_EHOSTDOWN); - CASE_RETSTRING(AzSockError::eASE_EINPROGRESS); - CASE_RETSTRING(AzSockError::eASE_EINTR); - CASE_RETSTRING(AzSockError::eASE_EINVAL); - CASE_RETSTRING(AzSockError::eASE_EISCONN); - CASE_RETSTRING(AzSockError::eASE_EMFILE); - CASE_RETSTRING(AzSockError::eASE_EMSGSIZE); - CASE_RETSTRING(AzSockError::eASE_ENETUNREACH); - CASE_RETSTRING(AzSockError::eASE_ENOBUFS); - CASE_RETSTRING(AzSockError::eASE_ENOPROTOOPT); - CASE_RETSTRING(AzSockError::eASE_ENOTCONN); - CASE_RETSTRING(AzSockError::eASE_ENOTINITIALISED); - CASE_RETSTRING(AzSockError::eASE_EOPNOTSUPP); - CASE_RETSTRING(AzSockError::eASE_EPIPE); - CASE_RETSTRING(AzSockError::eASE_EPROTONOSUPPORT); - CASE_RETSTRING(AzSockError::eASE_ETIMEDOUT); - CASE_RETSTRING(AzSockError::eASE_ETOOMANYREFS); - CASE_RETSTRING(AzSockError::eASE_EWOULDBLOCK); - CASE_RETSTRING(AzSockError::eASE_EWOULDBLOCK_CONN); - CASE_RETSTRING(AzSockError::eASE_MISC_ERROR); - } + switch (errorCode) + { + CASE_RETSTRING(AzSockError::eASE_NO_ERROR); + CASE_RETSTRING(AzSockError::eASE_SOCKET_INVALID); + CASE_RETSTRING(AzSockError::eASE_EACCES); + CASE_RETSTRING(AzSockError::eASE_EADDRINUSE); + CASE_RETSTRING(AzSockError::eASE_EADDRNOTAVAIL); + CASE_RETSTRING(AzSockError::eASE_EAFNOSUPPORT); + CASE_RETSTRING(AzSockError::eASE_EALREADY); + CASE_RETSTRING(AzSockError::eASE_EBADF); + CASE_RETSTRING(AzSockError::eASE_ECONNABORTED); + CASE_RETSTRING(AzSockError::eASE_ECONNREFUSED); + CASE_RETSTRING(AzSockError::eASE_ECONNRESET); + CASE_RETSTRING(AzSockError::eASE_EFAULT); + CASE_RETSTRING(AzSockError::eASE_EHOSTDOWN); + CASE_RETSTRING(AzSockError::eASE_EINPROGRESS); + CASE_RETSTRING(AzSockError::eASE_EINTR); + CASE_RETSTRING(AzSockError::eASE_EINVAL); + CASE_RETSTRING(AzSockError::eASE_EISCONN); + CASE_RETSTRING(AzSockError::eASE_EMFILE); + CASE_RETSTRING(AzSockError::eASE_EMSGSIZE); + CASE_RETSTRING(AzSockError::eASE_ENETUNREACH); + CASE_RETSTRING(AzSockError::eASE_ENOBUFS); + CASE_RETSTRING(AzSockError::eASE_ENOPROTOOPT); + CASE_RETSTRING(AzSockError::eASE_ENOTCONN); + CASE_RETSTRING(AzSockError::eASE_ENOTINITIALISED); + CASE_RETSTRING(AzSockError::eASE_EOPNOTSUPP); + CASE_RETSTRING(AzSockError::eASE_EPIPE); + CASE_RETSTRING(AzSockError::eASE_EPROTONOSUPPORT); + CASE_RETSTRING(AzSockError::eASE_ETIMEDOUT); + CASE_RETSTRING(AzSockError::eASE_ETOOMANYREFS); + CASE_RETSTRING(AzSockError::eASE_EWOULDBLOCK); + CASE_RETSTRING(AzSockError::eASE_EWOULDBLOCK_CONN); + CASE_RETSTRING(AzSockError::eASE_MISC_ERROR); + } #undef CASE_RETSTRING - return "(invalid)"; - } - - AZ::u32 HostToNetLong(AZ::u32 hstLong) - { - return htonl(hstLong); - } - - AZ::u32 NetToHostLong(AZ::u32 netLong) - { - return ntohl(netLong); - } - - AZ::u16 HostToNetShort(AZ::u16 hstShort) - { - return htons(hstShort); - } - - AZ::u16 NetToHostShort(AZ::u16 netShort) - { - return ntohs(netShort); - } - - AZ::s32 GetHostName(AZStd::string& hostname) - { - AZ::s32 result = 0; - hostname.clear(); - char name[256]; - result = HandleSocketError(gethostname(name, AZ_ARRAY_SIZE(name))); - if (result == static_cast(AzSockError::eASE_NO_ERROR)) - { - hostname = name; - } - return result; - } - - AZSOCKET Socket() - { - return Socket(AF_INET, SOCK_STREAM, IPPROTO_TCP); - } - - AZSOCKET Socket(AZ::s32 af, AZ::s32 type, AZ::s32 protocol) - { - return HandleInvalidSocket(socket(af, type, protocol)); - } - - AZ::s32 SetSockOpt(AZSOCKET sock, AZ::s32 level, AZ::s32 optname, const char* optval, AZ::s32 optlen) - { - AZSOCKLEN length(optlen); - return HandleSocketError(setsockopt(sock, level, optname, optval, length)); - } - - AZ::s32 SetSocketOption(AZSOCKET sock, AzSocketOption opt, bool enable) - { - AZ::u32 val = enable ? 1 : 0; - return SetSockOpt(sock, SOL_SOCKET, TranslateSocketOption(opt), reinterpret_cast(&val), sizeof(val)); - } - - AZ::s32 EnableTCPNoDelay(AZSOCKET sock, bool enable) - { - AZ::u32 val = enable ? 1 : 0; - return SetSockOpt(sock, IPPROTO_TCP, TCP_NODELAY, reinterpret_cast(&val), sizeof(val)); - } - - AZ::s32 SetSocketBlockingMode(AZSOCKET sock, bool blocking) - { - AZ::s32 flags = ::fcntl(sock, F_GETFL); - flags &= ~O_NONBLOCK; - flags |= (blocking ? 0 : O_NONBLOCK); - return ::fcntl(sock, F_SETFL, flags); - } - - AZ::s32 CloseSocket(AZSOCKET sock) - { - return HandleSocketError(closesocket(sock)); - } - - AZ::s32 Shutdown(AZSOCKET sock, AZ::s32 how) - { - return HandleSocketError(shutdown(sock, how)); - } - - AZ::s32 GetSockName(AZSOCKET sock, AzSocketAddress& addr) - { - AZSOCKADDR sAddr; - AZSOCKLEN sAddrLen = sizeof(AZSOCKADDR); - memset(&sAddr, 0, sAddrLen); - AZ::s32 result = HandleSocketError(getsockname(sock, &sAddr, &sAddrLen)); - addr = sAddr; - return result; - } - - AZ::s32 Connect(AZSOCKET sock, const AzSocketAddress& addr) - { - AZ::s32 err = HandleSocketError(connect(sock, addr.GetTargetAddress(), sizeof(AZSOCKADDR_IN))); - if (err == static_cast(AzSockError::eASE_EINPROGRESS)) - { - err = static_cast(AzSockError::eASE_EWOULDBLOCK_CONN); - } - return err; - } - - AZ::s32 Listen(AZSOCKET sock, AZ::s32 backlog) - { - return HandleSocketError(listen(sock, backlog)); - } - - AZSOCKET Accept(AZSOCKET sock, AzSocketAddress& addr) - { - AZSOCKADDR sAddr; - AZSOCKLEN sAddrLen = sizeof(AZSOCKADDR); - memset(&sAddr, 0, sAddrLen); - AZSOCKET outSock = HandleInvalidSocket(accept(sock, &sAddr, &sAddrLen)); - addr = sAddr; - return outSock; - } - - AZ::s32 Send(AZSOCKET sock, const char* buf, AZ::s32 len, AZ::s32 flags) - { - AZ::s32 msgNoSignal = MSG_NOSIGNAL; - return HandleSocketError(send(sock, buf, len, flags | msgNoSignal)); - } - - AZ::s32 Recv(AZSOCKET sock, char* buf, AZ::s32 len, AZ::s32 flags) - { - return HandleSocketError(recv(sock, buf, len, flags)); - } - - AZ::s32 Bind(AZSOCKET sock, const AzSocketAddress& addr) - { - return HandleSocketError(bind(sock, addr.GetTargetAddress(), sizeof(AZSOCKADDR_IN))); - } - - AZ::s32 Select(AZSOCKET sock, AZFD_SET* readfdsock, AZFD_SET* writefdsock, AZFD_SET* exceptfdsock, AZTIMEVAL* timeout) - { - return HandleSocketError(::select(sock + 1, readfdsock, writefdsock, exceptfdsock, timeout)); - } - - AZ::s32 IsRecvPending(AZSOCKET sock, AZTIMEVAL* timeout) - { - AZFD_SET readSet; - FD_ZERO(&readSet); - FD_SET(sock, &readSet); - - AZ::s32 ret = Select(sock, &readSet, nullptr, nullptr, timeout); - if (ret >= 0) - { - ret = FD_ISSET(sock, &readSet); - if (ret != 0) - { - ret = 1; - } - } - - return ret; - } - - AZ::s32 WaitForWritableSocket(AZSOCKET sock, AZTIMEVAL* timeout) - { - AZFD_SET writeSet; - FD_ZERO(&writeSet); - FD_SET(sock, &writeSet); - - AZ::s32 ret = Select(sock, nullptr, &writeSet, nullptr, timeout); - if (ret >= 0) - { - ret = FD_ISSET(sock, &writeSet); - if (ret != 0) - { - ret = 1; - } - } - - return ret; - } - - AZ::s32 Startup() - { - return static_cast(AzSockError::eASE_NO_ERROR); - } - - AZ::s32 Cleanup() - { - return static_cast(AzSockError::eASE_NO_ERROR); - } - - bool ResolveAddress(const AZStd::string& ip, AZ::u16 port, AZSOCKADDR_IN& socketAddress) - { - bool foundAddr = false; - addrinfo hints; - memset(&hints, 0, sizeof(addrinfo)); - addrinfo* addrInfo; - hints.ai_family = AF_INET; - hints.ai_flags = AI_CANONNAME; - char strPort[8]; - azsnprintf(strPort, AZ_ARRAY_SIZE(strPort), "%d", port); - - const char* address = ip.c_str(); - if (address && strlen(address) == 0) // getaddrinfo doesn't accept empty string - { - address = nullptr; - } - - AZ::s32 err = HandleSocketError(getaddrinfo(address, strPort, &hints, &addrInfo)); - if (err == 0) // eASE_NO_ERROR - { - if (addrInfo->ai_family == AF_INET) - { - socketAddress = *reinterpret_cast(addrInfo->ai_addr); - foundAddr = true; - } - - freeaddrinfo(addrInfo); - } - else - { - AZ_Assert(false, "AzSocketAddress could not resolve address %s with port %d. (reason - %s)", ip.c_str(), port, GetStringForError(err)); - } - return foundAddr; - } + return "(invalid)"; } -} + + AZ::u32 HostToNetLong(AZ::u32 hstLong) + { + return htonl(hstLong); + } + + AZ::u32 NetToHostLong(AZ::u32 netLong) + { + return ntohl(netLong); + } + + AZ::u16 HostToNetShort(AZ::u16 hstShort) + { + return htons(hstShort); + } + + AZ::u16 NetToHostShort(AZ::u16 netShort) + { + return ntohs(netShort); + } + + AZ::s32 GetHostName(AZStd::string& hostname) + { + AZ::s32 result = 0; + hostname.clear(); + char name[256]; + result = HandleSocketError(gethostname(name, AZ_ARRAY_SIZE(name))); + if (result == static_cast(AzSockError::eASE_NO_ERROR)) + { + hostname = name; + } + return result; + } + + AZSOCKET Socket() + { + return Socket(AF_INET, SOCK_STREAM, IPPROTO_TCP); + } + + AZSOCKET Socket(AZ::s32 af, AZ::s32 type, AZ::s32 protocol) + { + return HandleInvalidSocket(socket(af, type, protocol)); + } + + AZ::s32 SetSockOpt(AZSOCKET sock, AZ::s32 level, AZ::s32 optname, const char* optval, AZ::s32 optlen) + { + AZSOCKLEN length(optlen); + return HandleSocketError(setsockopt(sock, level, optname, optval, length)); + } + + AZ::s32 SetSocketOption(AZSOCKET sock, AzSocketOption opt, bool enable) + { + AZ::u32 val = enable ? 1 : 0; + return SetSockOpt(sock, SOL_SOCKET, TranslateSocketOption(opt), reinterpret_cast(&val), sizeof(val)); + } + + AZ::s32 EnableTCPNoDelay(AZSOCKET sock, bool enable) + { + AZ::u32 val = enable ? 1 : 0; + return SetSockOpt(sock, IPPROTO_TCP, TCP_NODELAY, reinterpret_cast(&val), sizeof(val)); + } + + AZ::s32 SetSocketBlockingMode(AZSOCKET sock, bool blocking) + { + AZ::s32 flags = ::fcntl(sock, F_GETFL); + flags &= ~O_NONBLOCK; + flags |= (blocking ? 0 : O_NONBLOCK); + return ::fcntl(sock, F_SETFL, flags); + } + + AZ::s32 CloseSocket(AZSOCKET sock) + { + return HandleSocketError(closesocket(sock)); + } + + AZ::s32 Shutdown(AZSOCKET sock, AZ::s32 how) + { + return HandleSocketError(shutdown(sock, how)); + } + + AZ::s32 GetSockName(AZSOCKET sock, AzSocketAddress& addr) + { + AZSOCKADDR sAddr; + AZSOCKLEN sAddrLen = sizeof(AZSOCKADDR); + memset(&sAddr, 0, sAddrLen); + AZ::s32 result = HandleSocketError(getsockname(sock, &sAddr, &sAddrLen)); + addr = sAddr; + return result; + } + + AZ::s32 Connect(AZSOCKET sock, const AzSocketAddress& addr) + { + AZ::s32 err = HandleSocketError(connect(sock, addr.GetTargetAddress(), sizeof(AZSOCKADDR_IN))); + if (err == static_cast(AzSockError::eASE_EINPROGRESS)) + { + err = static_cast(AzSockError::eASE_EWOULDBLOCK_CONN); + } + return err; + } + + AZ::s32 Listen(AZSOCKET sock, AZ::s32 backlog) + { + return HandleSocketError(listen(sock, backlog)); + } + + AZSOCKET Accept(AZSOCKET sock, AzSocketAddress& addr) + { + AZSOCKADDR sAddr; + AZSOCKLEN sAddrLen = sizeof(AZSOCKADDR); + memset(&sAddr, 0, sAddrLen); + AZSOCKET outSock = HandleInvalidSocket(accept(sock, &sAddr, &sAddrLen)); + addr = sAddr; + return outSock; + } + + AZ::s32 Send(AZSOCKET sock, const char* buf, AZ::s32 len, AZ::s32 flags) + { + AZ::s32 msgNoSignal = MSG_NOSIGNAL; + return HandleSocketError(send(sock, buf, len, flags | msgNoSignal)); + } + + AZ::s32 Recv(AZSOCKET sock, char* buf, AZ::s32 len, AZ::s32 flags) + { + return HandleSocketError(recv(sock, buf, len, flags)); + } + + AZ::s32 Bind(AZSOCKET sock, const AzSocketAddress& addr) + { + return HandleSocketError(bind(sock, addr.GetTargetAddress(), sizeof(AZSOCKADDR_IN))); + } + + AZ::s32 Select(AZSOCKET sock, AZFD_SET* readfdsock, AZFD_SET* writefdsock, AZFD_SET* exceptfdsock, AZTIMEVAL* timeout) + { + return HandleSocketError(::select(sock + 1, readfdsock, writefdsock, exceptfdsock, timeout)); + } + + AZ::s32 IsRecvPending(AZSOCKET sock, AZTIMEVAL* timeout) + { + AZFD_SET readSet; + FD_ZERO(&readSet); + FD_SET(sock, &readSet); + + AZ::s32 ret = Select(sock, &readSet, nullptr, nullptr, timeout); + if (ret >= 0) + { + ret = FD_ISSET(sock, &readSet); + if (ret != 0) + { + ret = 1; + } + } + + return ret; + } + + AZ::s32 WaitForWritableSocket(AZSOCKET sock, AZTIMEVAL* timeout) + { + AZFD_SET writeSet; + FD_ZERO(&writeSet); + FD_SET(sock, &writeSet); + + AZ::s32 ret = Select(sock, nullptr, &writeSet, nullptr, timeout); + if (ret >= 0) + { + ret = FD_ISSET(sock, &writeSet); + if (ret != 0) + { + ret = 1; + } + } + + return ret; + } + + AZ::s32 Startup() + { + return static_cast(AzSockError::eASE_NO_ERROR); + } + + AZ::s32 Cleanup() + { + return static_cast(AzSockError::eASE_NO_ERROR); + } + + bool ResolveAddress(const AZStd::string& ip, AZ::u16 port, AZSOCKADDR_IN& socketAddress) + { + bool foundAddr = false; + addrinfo hints; + memset(&hints, 0, sizeof(addrinfo)); + addrinfo* addrInfo; + hints.ai_family = AF_INET; + hints.ai_flags = AI_CANONNAME; + char strPort[8]; + azsnprintf(strPort, AZ_ARRAY_SIZE(strPort), "%d", port); + + const char* address = ip.c_str(); + if (address && strlen(address) == 0) // getaddrinfo doesn't accept empty string + { + address = nullptr; + } + + AZ::s32 err = HandleSocketError(getaddrinfo(address, strPort, &hints, &addrInfo)); + if (err == 0) // eASE_NO_ERROR + { + if (addrInfo->ai_family == AF_INET) + { + socketAddress = *reinterpret_cast(addrInfo->ai_addr); + foundAddr = true; + } + + freeaddrinfo(addrInfo); + } + else + { + AZ_Assert(false, "AzSocketAddress could not resolve address %s with port %d. (reason - %s)", ip.c_str(), port, GetStringForError(err)); + } + return foundAddr; + } +} // namespace AZ::AzSock diff --git a/Code/Framework/AzCore/Platform/Common/UnixLike/AzCore/Utils/Utils_UnixLike.cpp b/Code/Framework/AzCore/Platform/Common/UnixLike/AzCore/Utils/Utils_UnixLike.cpp index 7327c8f152..8a93f88ac2 100644 --- a/Code/Framework/AzCore/Platform/Common/UnixLike/AzCore/Utils/Utils_UnixLike.cpp +++ b/Code/Framework/AzCore/Platform/Common/UnixLike/AzCore/Utils/Utils_UnixLike.cpp @@ -11,68 +11,65 @@ #include #include -namespace AZ +namespace AZ::Utils { - namespace Utils + void RequestAbnormalTermination() { - void RequestAbnormalTermination() + abort(); + } + + void NativeErrorMessageBox(const char*, const char*) {} + + AZ::IO::FixedMaxPathString GetHomeDirectory() + { + constexpr AZStd::string_view overrideHomeDirKey = "/Amazon/Settings/override_home_dir"; + AZ::IO::FixedMaxPathString overrideHomeDir; + if (auto settingsRegistry = AZ::SettingsRegistry::Get(); settingsRegistry != nullptr) { - abort(); - } - - void NativeErrorMessageBox(const char*, const char*) {} - - AZ::IO::FixedMaxPathString GetHomeDirectory() - { - constexpr AZStd::string_view overrideHomeDirKey = "/Amazon/Settings/override_home_dir"; - AZ::IO::FixedMaxPathString overrideHomeDir; - if (auto settingsRegistry = AZ::SettingsRegistry::Get(); settingsRegistry != nullptr) + if (settingsRegistry->Get(overrideHomeDir, overrideHomeDirKey)) { - if (settingsRegistry->Get(overrideHomeDir, overrideHomeDirKey)) - { - AZ::IO::FixedMaxPath path{overrideHomeDir}; - return path.Native(); - } - } - - if (const char* homePath = std::getenv("HOME"); homePath != nullptr) - { - AZ::IO::FixedMaxPath path{homePath}; + AZ::IO::FixedMaxPath path{overrideHomeDir}; return path.Native(); } - - struct passwd* pass = getpwuid(getuid()); - if (pass) - { - AZ::IO::FixedMaxPath path{pass->pw_dir}; - return path.Native(); - } - - return {}; } - bool ConvertToAbsolutePath(const char* path, char* absolutePath, AZ::u64 maxLength) + if (const char* homePath = std::getenv("HOME"); homePath != nullptr) { + AZ::IO::FixedMaxPath path{homePath}; + return path.Native(); + } + + struct passwd* pass = getpwuid(getuid()); + if (pass) + { + AZ::IO::FixedMaxPath path{pass->pw_dir}; + return path.Native(); + } + + return {}; + } + + bool ConvertToAbsolutePath(const char* path, char* absolutePath, AZ::u64 maxLength) + { #ifdef PATH_MAX - static constexpr size_t UnixMaxPathLength = PATH_MAX; + static constexpr size_t UnixMaxPathLength = PATH_MAX; #else - // Fallback to 4096 if the PATH_MAX macro isn't defined on the Unix System - static constexpr size_t UnixMaxPathLength = 4096; + // Fallback to 4096 if the PATH_MAX macro isn't defined on the Unix System + static constexpr size_t UnixMaxPathLength = 4096; #endif - if (!AZ::IO::PathView(path).IsAbsolute()) + if (!AZ::IO::PathView(path).IsAbsolute()) + { + // note that realpath fails if the path does not exist and actually changes the return value + // to be the actual place that FAILED, which we don't want. + // if we fail, we'd prefer to fall through and at least use the original path. + char absolutePathBuffer[UnixMaxPathLength]; + if (const char* result = realpath(path, absolutePathBuffer); result != nullptr) { - // note that realpath fails if the path does not exist and actually changes the return value - // to be the actual place that FAILED, which we don't want. - // if we fail, we'd prefer to fall through and at least use the original path. - char absolutePathBuffer[UnixMaxPathLength]; - if (const char* result = realpath(path, absolutePathBuffer); result != nullptr) - { - azstrcpy(absolutePath, maxLength, absolutePathBuffer); - return true; - } + azstrcpy(absolutePath, maxLength, absolutePathBuffer); + return true; } - azstrcpy(absolutePath, maxLength, path); - return AZ::IO::PathView(absolutePath).IsAbsolute(); } - } // namespace Utils -} // namespace AZ + azstrcpy(absolutePath, maxLength, path); + return AZ::IO::PathView(absolutePath).IsAbsolute(); + } +} // namespace AZ::Utils diff --git a/Code/Framework/AzCore/Platform/Linux/AzCore/Debug/Trace_Linux.cpp b/Code/Framework/AzCore/Platform/Linux/AzCore/Debug/Trace_Linux.cpp index 1ca06fcf7f..e4cd0d0db4 100644 --- a/Code/Framework/AzCore/Platform/Linux/AzCore/Debug/Trace_Linux.cpp +++ b/Code/Framework/AzCore/Platform/Linux/AzCore/Debug/Trace_Linux.cpp @@ -9,16 +9,10 @@ #include #include -namespace AZ +namespace AZ::Debug::Platform { - namespace Debug + void OutputToDebugger([[maybe_unused]] const char* title, [[maybe_unused]] const char* message) { - namespace Platform - { - void OutputToDebugger([[maybe_unused]] const char* title, [[maybe_unused]] const char* message) - { - // std::cout << title << ": " << message; - } - } + // std::cout << title << ": " << message; } -} +} // namespace AZ::Debug::Platform diff --git a/Code/Framework/AzCore/Platform/Linux/AzCore/Memory/HeapSchema_Linux.cpp b/Code/Framework/AzCore/Platform/Linux/AzCore/Memory/HeapSchema_Linux.cpp index e28832cf61..d9defd5acc 100644 --- a/Code/Framework/AzCore/Platform/Linux/AzCore/Memory/HeapSchema_Linux.cpp +++ b/Code/Framework/AzCore/Platform/Linux/AzCore/Memory/HeapSchema_Linux.cpp @@ -8,13 +8,10 @@ #include -namespace AZ +namespace AZ::Platform { - namespace Platform + size_t GetHeapCapacity() { - size_t GetHeapCapacity() - { - return 0; - } + return 0; } -} +} // namespace AZ::Platform diff --git a/Code/Framework/AzCore/Platform/Linux/AzCore/Module/DynamicModuleHandle_Linux.cpp b/Code/Framework/AzCore/Platform/Linux/AzCore/Module/DynamicModuleHandle_Linux.cpp index 62718c2d46..aee2bdb622 100644 --- a/Code/Framework/AzCore/Platform/Linux/AzCore/Module/DynamicModuleHandle_Linux.cpp +++ b/Code/Framework/AzCore/Platform/Linux/AzCore/Module/DynamicModuleHandle_Linux.cpp @@ -10,28 +10,25 @@ #include #include -namespace AZ +namespace AZ::Platform { - namespace Platform + AZ::IO::FixedMaxPath GetModulePath() { - AZ::IO::FixedMaxPath GetModulePath() - { - return AZ::Utils::GetExecutableDirectory(); - } - - void* OpenModule(const AZ::OSString& fileName, bool& alreadyOpen) - { - void* handle = dlopen(fileName.c_str(), RTLD_NOLOAD); - alreadyOpen = (handle != nullptr); - if (!alreadyOpen) - { - handle = dlopen(fileName.c_str(), RTLD_NOW); - } - return handle; - } - - void ConstructModuleFullFileName(AZ::IO::FixedMaxPath&) - { - } + return AZ::Utils::GetExecutableDirectory(); } -} + + void* OpenModule(const AZ::OSString& fileName, bool& alreadyOpen) + { + void* handle = dlopen(fileName.c_str(), RTLD_NOLOAD); + alreadyOpen = (handle != nullptr); + if (!alreadyOpen) + { + handle = dlopen(fileName.c_str(), RTLD_NOW); + } + return handle; + } + + void ConstructModuleFullFileName(AZ::IO::FixedMaxPath&) + { + } +} // namespace AZ::Platform diff --git a/Code/Framework/AzCore/Platform/Linux/AzCore/Module/Internal/ModuleManagerSearchPathTool_Linux.cpp b/Code/Framework/AzCore/Platform/Linux/AzCore/Module/Internal/ModuleManagerSearchPathTool_Linux.cpp index 39b4cbd4ea..9dfa08e554 100644 --- a/Code/Framework/AzCore/Platform/Linux/AzCore/Module/Internal/ModuleManagerSearchPathTool_Linux.cpp +++ b/Code/Framework/AzCore/Platform/Linux/AzCore/Module/Internal/ModuleManagerSearchPathTool_Linux.cpp @@ -8,20 +8,17 @@ #include -namespace AZ +namespace AZ::Internal { - namespace Internal + ModuleManagerSearchPathTool::ModuleManagerSearchPathTool() { - ModuleManagerSearchPathTool::ModuleManagerSearchPathTool() - { - } + } - ModuleManagerSearchPathTool::~ModuleManagerSearchPathTool() - { - } + ModuleManagerSearchPathTool::~ModuleManagerSearchPathTool() + { + } - void ModuleManagerSearchPathTool::SetModuleSearchPath(const AZ::DynamicModuleDescriptor&) - { - } - } // namespace Internal -} // namespace AZ + void ModuleManagerSearchPathTool::SetModuleSearchPath(const AZ::DynamicModuleDescriptor&) + { + } +} // namespace AZ::Internal diff --git a/Code/Framework/AzCore/Platform/Linux/AzCore/Utils/Utils_Linux.cpp b/Code/Framework/AzCore/Platform/Linux/AzCore/Utils/Utils_Linux.cpp index d75eab0139..ed4e2674c5 100644 --- a/Code/Framework/AzCore/Platform/Linux/AzCore/Utils/Utils_Linux.cpp +++ b/Code/Framework/AzCore/Platform/Linux/AzCore/Utils/Utils_Linux.cpp @@ -13,42 +13,39 @@ #include -namespace AZ +namespace AZ::Utils { - namespace Utils + GetExecutablePathReturnType GetExecutablePath(char* exeStorageBuffer, size_t exeStorageSize) { - GetExecutablePathReturnType GetExecutablePath(char* exeStorageBuffer, size_t exeStorageSize) + GetExecutablePathReturnType result; + result.m_pathIncludesFilename = true; + + // http://man7.org/linux/man-pages/man5/proc.5.html + const ssize_t bytesWritten = readlink("/proc/self/exe", exeStorageBuffer, exeStorageSize); + if (bytesWritten == -1) { - GetExecutablePathReturnType result; - result.m_pathIncludesFilename = true; - - // http://man7.org/linux/man-pages/man5/proc.5.html - const ssize_t bytesWritten = readlink("/proc/self/exe", exeStorageBuffer, exeStorageSize); - if (bytesWritten == -1) - { - result.m_pathStored = ExecutablePathResult::GeneralError; - } - else if (bytesWritten == exeStorageSize) - { - result.m_pathStored = ExecutablePathResult::BufferSizeNotLargeEnough; - } - else - { - // readlink doesn't null terminate - exeStorageBuffer[bytesWritten] = '\0'; - } - - return result; + result.m_pathStored = ExecutablePathResult::GeneralError; + } + else if (bytesWritten == exeStorageSize) + { + result.m_pathStored = ExecutablePathResult::BufferSizeNotLargeEnough; + } + else + { + // readlink doesn't null terminate + exeStorageBuffer[bytesWritten] = '\0'; } - AZStd::optional GetDefaultAppRootPath() - { - return AZStd::nullopt; - } - - AZStd::optional GetDevWriteStoragePath() - { - return AZStd::nullopt; - } + return result; } -} + + AZStd::optional GetDefaultAppRootPath() + { + return AZStd::nullopt; + } + + AZStd::optional GetDevWriteStoragePath() + { + return AZStd::nullopt; + } +} // namespace AZ::Utils diff --git a/Code/Legacy/CryCommon/Cry_Vector2.h b/Code/Legacy/CryCommon/Cry_Vector2.h index 81a8c10e49..5bf47d0642 100644 --- a/Code/Legacy/CryCommon/Cry_Vector2.h +++ b/Code/Legacy/CryCommon/Cry_Vector2.h @@ -8,10 +8,6 @@ // Description : Common matrix class - - -#ifndef CRYINCLUDE_CRYCOMMON_CRY_VECTOR2_H -#define CRYINCLUDE_CRYCOMMON_CRY_VECTOR2_H #pragma once #include @@ -68,9 +64,7 @@ struct Vec2_tpl : x((F)v.x) , y((F)v.y) { assert(this->IsValid()); } - ILINE Vec2_tpl& operator=(const Vec2_tpl& src) { x = src.x; y = src.y; return *this; } - //template Vec2_tpl& operator=(const Vec2_tpl& src) { x=F(src.x); y=F(src.y); return *this; } - //template Vec2_tpl& operator=(const Vec3_tpl& src) { x=F(src.x); y=F(src.y); return *this; } + Vec2_tpl& operator=(const Vec2_tpl& src) = default; ILINE int operator!() const { return x == 0 && y == 0; } @@ -372,4 +366,3 @@ namespace AZ { AZ_TYPE_INFO_SPECIALIZE(Vec2, "{844131BA-9565-42F3-8482-6F65A6D5FC59}"); } -#endif // CRYINCLUDE_CRYCOMMON_CRY_VECTOR2_H diff --git a/Gems/Atom/RPI/Code/Source/RPI.Builders/Model/ModelExporterComponent.h b/Gems/Atom/RPI/Code/Source/RPI.Builders/Model/ModelExporterComponent.h index ee4bce634d..de4e1a0fe5 100644 --- a/Gems/Atom/RPI/Code/Source/RPI.Builders/Model/ModelExporterComponent.h +++ b/Gems/Atom/RPI/Code/Source/RPI.Builders/Model/ModelExporterComponent.h @@ -25,7 +25,7 @@ namespace AZ namespace RPI { /** - * This is the central component that drive the process of exporting a scene to Model + * This is the central component that drive the process of exporting a scene to Model * and Material assets. It delegates asset-build duties to other components like * ModelAssetBuilderComponent and MaterialAssetBuilderComponent via export events. */ @@ -55,7 +55,7 @@ namespace AZ AZStd::string_view m_relativeFileName; AZStd::string_view m_extension; - const Uuid m_sourceUuid; + const Uuid m_sourceUuid = Uuid::CreateNull(); const DataStream::StreamType m_dataStreamType = DataStream::ST_BINARY; };