birkeh
/
o3de
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Maximize read file size limit and set it to 1MiB for Atom use cases

Browse Source 
Signed-off-by: srikappa-amzn <srikappa@amazon.com>
monroegm-disable-blank-issue-2
srikappa-amzn 4 years ago
parent daf901e809
commit 5a6daf4352
12 changed files with 34 additions and 31 deletions
Show Stats Download Patch File Download Diff File

9
Code/Framework/AzCore/AzCore/Serialization/Json/JsonUtils.cpp
Unescape Escape View File

@ -240,11 +240,6 @@ namespace AZ
{
IO::SizeType length = stream.GetLength();
if (length > AZ::Utils::DefaultMaxFileSize)
{
return AZ::Failure(AZStd::string{ "Data is too large." });
}
AZStd::vector<char> memoryBuffer;
memoryBuffer.resize_no_construct(static_cast<AZStd::vector<char>::size_type>(static_cast<AZStd::vector<char>::size_type>(length) + 1));
@ -259,12 +254,12 @@ namespace AZ
return ReadJsonString(AZStd::string_view{memoryBuffer.data(), memoryBuffer.size()});
}
AZ::Outcome<rapidjson::Document, AZStd::string> ReadJsonFile(AZStd::string_view filePath)
AZ::Outcome<rapidjson::Document, AZStd::string> ReadJsonFile(AZStd::string_view filePath, size_t maxFileSize)
{
// Read into memory first and then parse the json, rather than passing a file stream to rapidjson.
// This should avoid creating a large number of micro-reads from the file.
auto readResult = AZ::Utils::ReadFile<AZStd::string>(filePath);
auto readResult = AZ::Utils::ReadFile<AZStd::string>(filePath, maxFileSize);
if(!readResult.IsSuccess())
{
return AZ::Failure(readResult.GetError());

3
Code/Framework/AzCore/AzCore/Serialization/Json/JsonUtils.h
Unescape Escape View File

@ -71,7 +71,8 @@ namespace AZ
AZ::Outcome<rapidjson::Document, AZStd::string> ReadJsonString(AZStd::string_view jsonText);
//! Parse a json file. Returns a failure with error message if the content is not valid JSON.
AZ::Outcome<rapidjson::Document, AZStd::string> ReadJsonFile(AZStd::string_view filePath);
AZ::Outcome<rapidjson::Document, AZStd::string> ReadJsonFile(
AZStd::string_view filePath, size_t maxFileSize = AZStd::numeric_limits<size_t>::max());
//! Parse a json stream. Returns a failure with error message if the content is not valid JSON.
AZ::Outcome<rapidjson::Document, AZStd::string> ReadJsonStream(IO::GenericStream& stream);

7
Code/Framework/AzCore/AzCore/Utils/Utils.h
Unescape Escape View File

@ -22,10 +22,6 @@ namespace AZ
{
namespace Utils
{
//! Protects from allocating too much memory. The choice of a 1MB threshold is arbitrary.
//! If you need to work with larger files, please use AZ::IO directly instead of these utility functions.
inline constexpr size_t DefaultMaxFileSize = 1024 * 1024;
//! Terminates the application without going through the shutdown procedure.
//! This is used when due to abnormal circumstances the application can no
//! longer continue. On most platforms and in most configurations this will
@ -115,6 +111,7 @@ namespace AZ
//! Read a file into a string. Returns a failure with error message if the content could not be loaded or if
//! the file size is larger than the max file size provided.
template<typename Container = AZStd::string>
AZ::Outcome<Container, AZStd::string> ReadFile(AZStd::string_view filePath, size_t maxFileSize = DefaultMaxFileSize);
AZ::Outcome<Container, AZStd::string> ReadFile(
AZStd::string_view filePath, size_t maxFileSize);
}
}

4
Gems/Atom/Asset/Shader/Code/Source/Editor/AzslCompiler.cpp
Unescape Escape View File

@ -1149,7 +1149,7 @@ namespace AZ
return BuildResult::CompilationFailed;
}
auto readJsonResult = JsonSerializationUtils::ReadJsonFile(outputFile);
auto readJsonResult = JsonSerializationUtils::ReadJsonFile(outputFile, AZ::RPI::JsonUtils::AtomMaxFileSize);
if (readJsonResult.IsSuccess())
{
@ -1170,7 +1170,7 @@ namespace AZ
AZStd::string outputFile = m_inputFilePath;
AzFramework::StringFunc::Path::ReplaceExtension(outputFile, outputExtension);
auto readJsonResult = JsonSerializationUtils::ReadJsonFile(outputFile);
auto readJsonResult = JsonSerializationUtils::ReadJsonFile(outputFile, AZ::RPI::JsonUtils::AtomMaxFileSize);
if (readJsonResult.IsSuccess())
{

3
Gems/Atom/Asset/Shader/Code/Source/Editor/ShaderAssetBuilder.cpp
Unescape Escape View File

@ -20,6 +20,7 @@
#include <Atom/RHI.Edit/ShaderPlatformInterface.h>
#include <Atom/RPI.Edit/Common/JsonReportingHelper.h>
#include <Atom/RPI.Edit/Common/AssetUtils.h>
#include <Atom/RPI.Edit/Common/JsonUtils.h>
#include <Atom/RHI.Reflect/ConstantsLayout.h>
#include <Atom/RHI.Reflect/PipelineLayoutDescriptor.h>
#include <Atom/RHI.Reflect/ShaderStageFunction.h>
@ -555,7 +556,7 @@ namespace AZ
shaderAssetCreator.SetRenderStates(renderStates);
}
Outcome<AZStd::string, AZStd::string> hlslSourceCodeOutcome = Utils::ReadFile(hlslFullPath);
Outcome<AZStd::string, AZStd::string> hlslSourceCodeOutcome = Utils::ReadFile(hlslFullPath, AZ::RPI::JsonUtils::AtomMaxFileSize);
if (!hlslSourceCodeOutcome.IsSuccess())
{
AZ_Error(

9
Gems/Atom/Asset/Shader/Code/Source/Editor/ShaderBuilderUtility.cpp
Unescape Escape View File

@ -25,6 +25,7 @@
#include <Atom/RPI.Edit/Common/JsonReportingHelper.h>
#include <Atom/RPI.Edit/Common/AssetUtils.h>
#include <Atom/RPI.Edit/Common/JsonUtils.h>
#include <Atom/RPI.Reflect/Shader/ShaderAsset.h>
#include <Atom/RPI.Reflect/Shader/ShaderOptionGroup.h>
@ -51,7 +52,7 @@ namespace AZ
{
RPI::ShaderSourceData shaderSourceData;
auto document = JsonSerializationUtils::ReadJsonFile(fullPathToJsonFile);
auto document = JsonSerializationUtils::ReadJsonFile(fullPathToJsonFile, AZ::RPI::JsonUtils::AtomMaxFileSize);
if (!document.IsSuccess())
{
@ -127,7 +128,7 @@ namespace AZ
AZStd::unordered_map<int, Outcome<rapidjson::Document, AZStd::string>> outcomes;
for (int i : indicesOfInterest)
{
outcomes[i] = JsonSerializationUtils::ReadJsonFile(pathOfJsonFiles[i]);
outcomes[i] = JsonSerializationUtils::ReadJsonFile(pathOfJsonFiles[i], AZ::RPI::JsonUtils::AtomMaxFileSize);
if (!outcomes[i].IsSuccess())
{
AZ_Error(builderName, false, "%s", outcomes[i].GetError().c_str());
@ -622,7 +623,7 @@ namespace AZ
StructData inputStruct;
inputStruct.m_id = "";
auto jsonOutcome = JsonSerializationUtils::ReadJsonFile(pathToIaJson);
auto jsonOutcome = JsonSerializationUtils::ReadJsonFile(pathToIaJson, AZ::RPI::JsonUtils::AtomMaxFileSize);
if (!jsonOutcome.IsSuccess())
{
AZ_Error(ShaderBuilderUtilityName, false, "%s", jsonOutcome.GetError().c_str());
@ -715,7 +716,7 @@ namespace AZ
StructData outputStruct;
outputStruct.m_id = "";
auto jsonOutcome = JsonSerializationUtils::ReadJsonFile(pathToOmJson);
auto jsonOutcome = JsonSerializationUtils::ReadJsonFile(pathToOmJson, AZ::RPI::JsonUtils::AtomMaxFileSize);
if (!jsonOutcome.IsSuccess())
{
AZ_Error(ShaderBuilderUtilityName, false, "%s", jsonOutcome.GetError().c_str());

10
Gems/Atom/Asset/Shader/Code/Source/Editor/ShaderVariantAssetBuilder.cpp
Unescape Escape View File

@ -478,7 +478,7 @@ namespace AZ
RPI::Ptr<RPI::ShaderOptionGroupLayout> shaderOptionGroupLayout = RPI::ShaderOptionGroupLayout::Create();
// The shader options define what options are available, what are the allowed values/range
// for each option and what is its default value.
auto jsonOutcome = JsonSerializationUtils::ReadJsonFile(optionsGroupJsonPath);
auto jsonOutcome = JsonSerializationUtils::ReadJsonFile(optionsGroupJsonPath, AZ::RPI::JsonUtils::AtomMaxFileSize);
if (!jsonOutcome.IsSuccess())
{
AZ_Error(ShaderVariantAssetBuilderName, false, "%s", jsonOutcome.GetError().c_str());
@ -509,7 +509,7 @@ namespace AZ
}
auto functionsJsonPath = functionsJsonPathOutcome.TakeValue();
auto jsonOutcome = JsonSerializationUtils::ReadJsonFile(functionsJsonPath);
auto jsonOutcome = JsonSerializationUtils::ReadJsonFile(functionsJsonPath, AZ::RPI::JsonUtils::AtomMaxFileSize);
if (!jsonOutcome.IsSuccess())
{
AZ_Error(ShaderVariantAssetBuilderName, false, "%s", jsonOutcome.GetError().c_str());
@ -541,7 +541,7 @@ namespace AZ
}
auto srgJsonPath = srgJsonPathOutcome.TakeValue();
auto jsonOutcome = JsonSerializationUtils::ReadJsonFile(srgJsonPath);
auto jsonOutcome = JsonSerializationUtils::ReadJsonFile(srgJsonPath, AZ::RPI::JsonUtils::AtomMaxFileSize);
if (!jsonOutcome.IsSuccess())
{
AZ_Error(ShaderVariantAssetBuilderName, false, "%s", jsonOutcome.GetError().c_str());
@ -598,7 +598,7 @@ namespace AZ
}
auto bindingsJsonPath = bindingsJsonPathOutcome.TakeValue();
auto jsonOutcome = JsonSerializationUtils::ReadJsonFile(bindingsJsonPath);
auto jsonOutcome = JsonSerializationUtils::ReadJsonFile(bindingsJsonPath, AZ::RPI::JsonUtils::AtomMaxFileSize);
if (!jsonOutcome.IsSuccess())
{
AZ_Error(ShaderVariantAssetBuilderName, false, "%s", jsonOutcome.GetError().c_str());
@ -630,7 +630,7 @@ namespace AZ
}
hlslSourcePath = hlslSourcePathOutcome.TakeValue();
Outcome<AZStd::string, AZStd::string> hlslSourceOutcome = Utils::ReadFile(hlslSourcePath);
Outcome<AZStd::string, AZStd::string> hlslSourceOutcome = Utils::ReadFile(hlslSourcePath, AZ::RPI::JsonUtils::AtomMaxFileSize);
if (!hlslSourceOutcome.IsSuccess())
{
AZ_Error(

6
Gems/Atom/RPI/Code/Include/Atom/RPI.Edit/Common/JsonUtils.h
Unescape Escape View File

@ -20,6 +20,10 @@ namespace AZ
{
namespace JsonUtils
{
//! Protects from allocating too much memory. The choice of a 1MB threshold is arbitrary.
//! If you need to work with larger files, please use AZ::IO directly instead of these utility functions.
inline constexpr size_t AtomMaxFileSize = 1024 * 1024;
// Declarations...
//! Loads serialized object data from a json file at the specified path
@ -39,7 +43,7 @@ namespace AZ
{
objectData = ObjectType();
auto loadOutcome = AZ::JsonSerializationUtils::ReadJsonFile(path);
auto loadOutcome = AZ::JsonSerializationUtils::ReadJsonFile(path, AtomMaxFileSize);
if (!loadOutcome.IsSuccess())
{
AZ_Error("AZ::RPI::JsonUtils", false, "%s", loadOutcome.GetError().c_str());

5
Gems/Atom/RPI/Code/Source/RPI.Builders/Material/MaterialBuilder.cpp
Unescape Escape View File

@ -13,6 +13,7 @@
#include <Atom/RPI.Edit/Common/AssetUtils.h>
#include <Atom/RPI.Edit/Common/JsonFileLoadContext.h>
#include <Atom/RPI.Edit/Common/JsonReportingHelper.h>
#include <Atom/RPI.Edit/Common/JsonUtils.h>
#include <AzCore/Serialization/Json/JsonUtils.h>
#include <Atom/RPI.Reflect/Image/StreamingImageAsset.h>
@ -151,7 +152,7 @@ namespace AZ
AZStd::string fullSourcePath;
AzFramework::StringFunc::Path::ConstructFull(request.m_watchFolder.data(), request.m_sourceFile.data(), fullSourcePath, true);
auto loadOutcome = JsonSerializationUtils::ReadJsonFile(fullSourcePath);
auto loadOutcome = JsonSerializationUtils::ReadJsonFile(fullSourcePath, AZ::RPI::JsonUtils::AtomMaxFileSize);
if (!loadOutcome.IsSuccess())
{
AZ_Error(MaterialBuilderName, false, "%s", loadOutcome.GetError().c_str());
@ -298,7 +299,7 @@ namespace AZ
AZStd::string fullSourcePath;
AzFramework::StringFunc::Path::ConstructFull(request.m_watchFolder.data(), request.m_sourceFile.data(), fullSourcePath, true);
auto loadOutcome = JsonSerializationUtils::ReadJsonFile(fullSourcePath);
auto loadOutcome = JsonSerializationUtils::ReadJsonFile(fullSourcePath, AZ::RPI::JsonUtils::AtomMaxFileSize);
if (!loadOutcome.IsSuccess())
{
AZ_Error(MaterialBuilderName, false, "Failed to load material file: %s", loadOutcome.GetError().c_str());

3
Gems/Atom/RPI/Code/Source/RPI.Edit/Material/MaterialSourceDataSerializer.cpp
Unescape Escape View File

@ -10,6 +10,7 @@
#include <Atom/RPI.Edit/Material/MaterialTypeSourceData.h>
#include <Atom/RPI.Edit/Common/AssetUtils.h>
#include <Atom/RPI.Edit/Common/JsonFileLoadContext.h>
#include <Atom/RPI.Edit/Common/JsonUtils.h>
#include <AzCore/Serialization/Json/JsonUtils.h>
#include <AzCore/Serialization/Json/BaseJsonSerializer.h>
@ -67,7 +68,7 @@ namespace AZ
{
AZStd::string materialTypePath = AssetUtils::ResolvePathReference(jsonFileLoadContext->GetFilePath(), materialSourceData->m_materialType);
auto materialTypeJson = JsonSerializationUtils::ReadJsonFile(materialTypePath);
auto materialTypeJson = JsonSerializationUtils::ReadJsonFile(materialTypePath, AZ::RPI::JsonUtils::AtomMaxFileSize);
if (!materialTypeJson.IsSuccess())
{
AZStd::string failureMessage;

3
Gems/Atom/RPI/Code/Source/RPI.Edit/Material/MaterialUtils.cpp
Unescape Escape View File

@ -15,6 +15,7 @@
#include <Atom/RPI.Edit/Material/MaterialTypeSourceData.h>
#include <Atom/RPI.Edit/Common/JsonReportingHelper.h>
#include <Atom/RPI.Edit/Common/JsonFileLoadContext.h>
#include <Atom/RPI.Edit/Common/JsonUtils.h>
#include <AzCore/Serialization/Json/JsonUtils.h>
#include <AzCore/std/string/string.h>
@ -64,7 +65,7 @@ namespace AZ
AZ::Outcome<rapidjson::Document, AZStd::string> loadOutcome;
if (document == nullptr)
{
loadOutcome = AZ::JsonSerializationUtils::ReadJsonFile(filePath);
loadOutcome = AZ::JsonSerializationUtils::ReadJsonFile(filePath, AZ::RPI::JsonUtils::AtomMaxFileSize);
if (!loadOutcome.IsSuccess())
{
AZ_Error("AZ::RPI::JsonUtils", false, "%s", loadOutcome.GetError().c_str());

3
Gems/AtomLyIntegration/AtomBridge/Code/Source/Editor/AssetCollectionAsyncLoaderTestComponent.cpp
Unescape Escape View File

@ -16,6 +16,7 @@
#include <AzCore/Serialization/Json/JsonUtils.h>
// Included so we can deduce the asset type from asset paths.
#include <Atom/RPI.Edit/Common/JsonUtils.h>
#include <Atom/RPI.Reflect/Shader/ShaderAsset.h>
#include <Atom/RPI.Reflect/Model/ModelAsset.h>
#include <Atom/RPI.Reflect/Image/StreamingImageAsset.h>
@ -114,7 +115,7 @@ namespace AZ
{
rapidjson::Document jsonDoc;
auto readJsonResult = JsonSerializationUtils::ReadJsonFile(pathToAssetListJson);
auto readJsonResult = JsonSerializationUtils::ReadJsonFile(pathToAssetListJson, AZ::RPI::JsonUtils::AtomMaxFileSize);
if (!readJsonResult.IsSuccess())
{

Write Preview
Loading…
Cancel
Save