From e9a57380bb82045e5ebe66ed11b65009ae53b4b0 Mon Sep 17 00:00:00 2001 From: Nicholas Van Sickle Date: Wed, 10 Nov 2021 12:14:07 -0800 Subject: [PATCH 01/18] Generic DOM: Add DomBackend abstraction and JSON support This change adds a `DomBackend` interface and a `DomBackendRegistry` for backend discovery (currently not hooked up to anything) alongside a JSON backend implementation. The JSON backend comes with a small suite of unit and performance tests. The unit tests validate generic DOM conversion to and from serialized JSON and rapidjson::Document objects (the support for which lives in `JsonSerializationUtils.h`). The performance tests show a throughput decrease compared to directly using the rapidjson serializer when mirroring our current pattern of copying strings from the serialized JSON representation, but with the coming in-memory store we have the opportunity to keep the buffer in memory and deserialize in-situ using rapidjson's API, which is consistently at least 100MiB/s faster on my machine (Ryzen Threadripper 3970X). The first parameter is the nested object complexity (N*N, objects with N keys comprised of arrays with N values) and the second parameter is the base size of the strings within each entry of this object. ``` ---------------------------------------------------------------------------------------------------------------------- Benchmark Time CPU Iterations UserCounters... ---------------------------------------------------------------------------------------------------------------------- DomJsonBenchmark/DomDeserializeToDocumentInPlace/10/5 0.050 ms 0.050 ms 10000 bytes_per_second=355.816M/s DomJsonBenchmark/DomDeserializeToDocumentInPlace/10/50 0.057 ms 0.057 ms 11200 bytes_per_second=386.064M/s DomJsonBenchmark/DomDeserializeToDocumentInPlace/100/5 4.77 ms 4.88 ms 112 bytes_per_second=364.046M/s DomJsonBenchmark/DomDeserializeToDocumentInPlace/100/500 11.6 ms 11.7 ms 64 bytes_per_second=554.518M/s DomJsonBenchmark/DomDeserializeToDocumentWithCopies/10/5 0.084 ms 0.084 ms 7467 bytes_per_second=212.55M/s DomJsonBenchmark/DomDeserializeToDocumentWithCopies/10/50 0.099 ms 0.100 ms 6400 bytes_per_second=220.608M/s DomJsonBenchmark/DomDeserializeToDocumentWithCopies/100/5 8.22 ms 8.16 ms 90 bytes_per_second=217.847M/s DomJsonBenchmark/DomDeserializeToDocumentWithCopies/100/500 23.2 ms 22.9 ms 30 bytes_per_second=283.56M/s DomJsonBenchmark/JsonUtilsDeserializeToDocument/10/5 0.070 ms 0.070 ms 11200 bytes_per_second=255.049M/s DomJsonBenchmark/JsonUtilsDeserializeToDocument/10/50 0.086 ms 0.087 ms 8960 bytes_per_second=253.258M/s DomJsonBenchmark/JsonUtilsDeserializeToDocument/100/5 6.86 ms 6.84 ms 112 bytes_per_second=260.033M/s DomJsonBenchmark/JsonUtilsDeserializeToDocument/100/500 22.8 ms 22.9 ms 32 bytes_per_second=283.158M/s ``` For `AZ::DOM::Document`, the current plan is to offer helper methods that can load from a file path or string using a given backend that can take advantage of in-place parsing by internally storing the serialized buffer. Signed-off-by: Nicholas Van Sickle --- .../AzCore/DOM/Backends/JSON/JsonBackend.cpp | 37 + .../AzCore/DOM/Backends/JSON/JsonBackend.h | 27 + .../Backends/JSON/JsonSerializationUtils.cpp | 675 ++++++++++++++++++ .../Backends/JSON/JsonSerializationUtils.h | 60 ++ .../AzCore/AzCore/DOM/DomBackend.cpp | 77 ++ Code/Framework/AzCore/AzCore/DOM/DomBackend.h | 54 ++ .../AzCore/AzCore/DOM/DomBackendRegistry.cpp | 68 ++ .../AzCore/AzCore/DOM/DomBackendRegistry.h | 44 ++ .../AzCore/DOM/DomBackendRegistryInterface.h | 54 ++ Code/Framework/AzCore/AzCore/DOM/DomVisitor.h | 29 +- .../AzCore/AzCore/azcore_files.cmake | 10 + .../AzCore/Tests/DOM/DomJsonBenchmarks.cpp | 184 +++++ .../AzCore/Tests/DOM/DomJsonTests.cpp | 288 ++++++++ .../AzCore/Tests/azcoretests_files.cmake | 2 + 14 files changed, 1599 insertions(+), 10 deletions(-) create mode 100644 Code/Framework/AzCore/AzCore/DOM/Backends/JSON/JsonBackend.cpp create mode 100644 Code/Framework/AzCore/AzCore/DOM/Backends/JSON/JsonBackend.h create mode 100644 Code/Framework/AzCore/AzCore/DOM/Backends/JSON/JsonSerializationUtils.cpp create mode 100644 Code/Framework/AzCore/AzCore/DOM/Backends/JSON/JsonSerializationUtils.h create mode 100644 Code/Framework/AzCore/AzCore/DOM/DomBackend.cpp create mode 100644 Code/Framework/AzCore/AzCore/DOM/DomBackend.h create mode 100644 Code/Framework/AzCore/AzCore/DOM/DomBackendRegistry.cpp create mode 100644 Code/Framework/AzCore/AzCore/DOM/DomBackendRegistry.h create mode 100644 Code/Framework/AzCore/AzCore/DOM/DomBackendRegistryInterface.h create mode 100644 Code/Framework/AzCore/Tests/DOM/DomJsonBenchmarks.cpp create mode 100644 Code/Framework/AzCore/Tests/DOM/DomJsonTests.cpp diff --git a/Code/Framework/AzCore/AzCore/DOM/Backends/JSON/JsonBackend.cpp b/Code/Framework/AzCore/AzCore/DOM/Backends/JSON/JsonBackend.cpp new file mode 100644 index 0000000000..4abeeaf07e --- /dev/null +++ b/Code/Framework/AzCore/AzCore/DOM/Backends/JSON/JsonBackend.cpp @@ -0,0 +1,37 @@ +/* + * Copyright (c) Contributors to the Open 3D Engine Project. + * For complete copyright and license terms please see the LICENSE at the root of this distribution. + * + * SPDX-License-Identifier: Apache-2.0 OR MIT + * + */ + +#include + +#include + +namespace AZ::DOM +{ + Visitor::Result JsonBackend::ReadFromStringInPlace(AZStd::string& buffer, Visitor* visitor) + { + return Json::VisitSerializedJsonInPlace(buffer, visitor); + } + + Visitor::Result JsonBackend::ReadFromString(AZStd::string_view buffer, Lifetime lifetime, Visitor* visitor) + { + return Json::VisitSerializedJson(buffer, lifetime, visitor); + } + + AZStd::unique_ptr JsonBackend::CreateStreamWriter(AZ::IO::GenericStream* stream) + { + return Json::GetJsonStreamWriter(stream, Json::OutputFormatting::PrettyPrintedJson); + } + + void JsonBackend::Register() + { + if (auto backendRegistry = BackendRegistry::Get()) + { + backendRegistry->RegisterBackend(kName, {kExtension}); + } + } +} diff --git a/Code/Framework/AzCore/AzCore/DOM/Backends/JSON/JsonBackend.h b/Code/Framework/AzCore/AzCore/DOM/Backends/JSON/JsonBackend.h new file mode 100644 index 0000000000..5e93ece1de --- /dev/null +++ b/Code/Framework/AzCore/AzCore/DOM/Backends/JSON/JsonBackend.h @@ -0,0 +1,27 @@ +/* + * Copyright (c) Contributors to the Open 3D Engine Project. + * For complete copyright and license terms please see the LICENSE at the root of this distribution. + * + * SPDX-License-Identifier: Apache-2.0 OR MIT + * + */ + +#pragma once + +#include +#include + +namespace AZ::DOM +{ + class JsonBackend final : public Backend + { + public: + Visitor::Result ReadFromStringInPlace(AZStd::string& buffer, Visitor* visitor) override; + Visitor::Result ReadFromString(AZStd::string_view buffer, Lifetime lifetime, Visitor* visitor) override; + AZStd::unique_ptr CreateStreamWriter(AZ::IO::GenericStream* stream) override; + + static constexpr const char* kName = "JSON"; + static constexpr const char* kExtension = ".json"; + static void Register(); + }; +} // namespace AZ::DOM diff --git a/Code/Framework/AzCore/AzCore/DOM/Backends/JSON/JsonSerializationUtils.cpp b/Code/Framework/AzCore/AzCore/DOM/Backends/JSON/JsonSerializationUtils.cpp new file mode 100644 index 0000000000..302fd7bd63 --- /dev/null +++ b/Code/Framework/AzCore/AzCore/DOM/Backends/JSON/JsonSerializationUtils.cpp @@ -0,0 +1,675 @@ +/* + * Copyright (c) Contributors to the Open 3D Engine Project. + * For complete copyright and license terms please see the LICENSE at the root of this distribution. + * + * SPDX-License-Identifier: Apache-2.0 OR MIT + * + */ + +#include + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +namespace AZ::DOM::Json +{ + // + // class DocumentWriter + // + // Visitor that produces a rapidjson::Document + class DocumentWriter final : public Visitor + { + public: + VisitorFlags GetVisitorFlags() const override + { + return VisitorFlags::SupportsRawKeys | VisitorFlags::SupportsArrays | VisitorFlags::SupportsObjects; + } + + Result Null() override + { + CurrentValue().SetNull(); + return FinishWrite(); + } + + Result Bool(bool value) override + { + CurrentValue().SetBool(value); + return FinishWrite(); + } + + Result Int64(AZ::s64 value) override + { + CurrentValue().SetInt64(value); + return FinishWrite(); + } + + Result Uint64(AZ::u64 value) override + { + CurrentValue().SetUint64(value); + return FinishWrite(); + } + + Result Double(double value) override + { + CurrentValue().SetDouble(value); + return FinishWrite(); + } + + Result String(AZStd::string_view value, Lifetime lifetime) override + { + if (lifetime == Lifetime::Temporary) + { + CurrentValue().SetString(value.data(), static_cast(value.length()), m_result.GetAllocator()); + } + else + { + CurrentValue().SetString(value.data(), static_cast(value.size())); + } + return FinishWrite(); + } + + Result StartObject() override + { + CurrentValue().SetObject(); + + const bool isObject = true; + m_entryStack.emplace_front(isObject, CurrentValue()); + return VisitorSuccess(); + } + + Result EndObject(AZ::u64 attributeCount) override + { + if (m_entryStack.empty()) + { + return VisitorFailure(VisitorErrorCode::InternalError, "EndObject called without a matching BeginObject call"); + } + + if (!m_entryStack.front().m_isObject) + { + return VisitorFailure(VisitorErrorCode::InternalError, "Expected EndArray and received EndObject instead"); + } + + if (m_entryStack.front().m_entryCount != attributeCount) + { + return FormatVisitorFailure( + VisitorErrorCode::InternalError, "EndObject: Expected %lu attributes but received %lu attributes instead", + attributeCount, m_entryStack.front().m_entryCount); + } + + m_entryStack.pop_front(); + return FinishWrite(); + } + + Result Key(AZ::Name key) override + { + return RawKey(key.GetStringView(), Lifetime::Persistent); + } + + Result RawKey(AZStd::string_view key, Lifetime lifetime) override + { + AZ_Assert(!m_entryStack.empty(), "Attempmted to push a key with no object"); + AZ_Assert(m_entryStack.front().m_isObject, "Attempted to push a key to an array"); + if (lifetime == Lifetime::Persistent) + { + m_entryStack.front().m_key.SetString(key.data(), static_cast(key.size())); + } + else + { + m_entryStack.front().m_key.SetString(key.data(), static_cast(key.size()), m_result.GetAllocator()); + } + return VisitorSuccess(); + } + + Result StartArray() override + { + CurrentValue().SetArray(); + + const bool isObject = false; + m_entryStack.emplace_front(isObject, CurrentValue()); + return VisitorSuccess(); + } + + Result EndArray(AZ::u64 elementCount) override + { + if (m_entryStack.empty()) + { + return VisitorFailure(VisitorErrorCode::InternalError, "EndArray called without a matching BeginArray call"); + } + + if (m_entryStack.front().m_isObject) + { + return VisitorFailure(VisitorErrorCode::InternalError, "Expected EndObject and received EndArray instead"); + } + + if (m_entryStack.front().m_entryCount != elementCount) + { + return FormatVisitorFailure( + VisitorErrorCode::InternalError, "EndArray: Expected %lu elements but received %lu elements instead", elementCount, + m_entryStack.front().m_entryCount); + } + + m_entryStack.pop_front(); + return FinishWrite(); + } + + rapidjson::Document&& TakeDocument() + { + return AZStd::move(m_result); + } + + private: + Result FinishWrite() + { + if (m_entryStack.empty()) + { + return VisitorSuccess(); + } + + // Retrieve the top value of the stack and replace it with a null value + rapidjson::Value value; + m_entryStack.front().m_value.Swap(value); + ++m_entryStack.front().m_entryCount; + + if (m_entryStack.front().m_key.IsString()) + { + m_entryStack.front().m_container.AddMember(m_entryStack.front().m_key.Move(), AZStd::move(value), m_result.GetAllocator()); + m_entryStack.front().m_key.SetNull(); + } + else + { + m_entryStack.front().m_container.PushBack(AZStd::move(value), m_result.GetAllocator()); + } + + return VisitorSuccess(); + } + + rapidjson::Value& CurrentValue() + { + if (m_entryStack.empty()) + { + return m_result; + } + return m_entryStack.front().m_value; + } + + struct ValueInfo + { + ValueInfo(bool isObject, rapidjson::Value& container) + : m_isObject(isObject) + , m_container(container) + { + } + + bool m_isObject; + rapidjson::Value& m_container; + rapidjson::Value m_value; + AZ::u64 m_entryCount = 0; + rapidjson::Value m_key; + }; + + rapidjson::Document m_result; + AZStd::deque m_entryStack; + AZ::u64 m_entryCount = 0; + }; + + // + // class StreamWriter + // + // Visitor that writes to a rapidjson::Writer + template + class StreamWriter : public Visitor + { + public: + StreamWriter(AZ::IO::GenericStream* stream) + : m_streamWriter(stream) + , m_writer(TWriter(m_streamWriter)) + { + } + + VisitorFlags GetVisitorFlags() const override + { + return VisitorFlags::SupportsRawKeys | VisitorFlags::SupportsArrays | VisitorFlags::SupportsObjects; + } + + Result Null() override + { + return CheckWrite(m_writer.Null()); + } + + Result Bool(bool value) override + { + return CheckWrite(m_writer.Bool(value)); + } + + Result Int64(AZ::s64 value) override + { + return CheckWrite(m_writer.Int64(value)); + } + + Result Uint64(AZ::u64 value) override + { + return CheckWrite(m_writer.Uint64(value)); + } + + Result Double(double value) override + { + return CheckWrite(m_writer.Double(value)); + } + + Result String(AZStd::string_view value, Lifetime lifetime) override + { + const bool shouldCopy = lifetime == Lifetime::Temporary; + return CheckWrite(m_writer.String(value.data(), static_cast(value.size()), shouldCopy)); + } + + Result StartObject() override + { + return CheckWrite(m_writer.StartObject()); + } + + Result EndObject(AZ::u64 attributeCount) override + { + return CheckWrite(m_writer.EndObject(static_cast(attributeCount))); + } + + Result Key(AZ::Name key) override + { + return RawKey(key.GetStringView(), Lifetime::Persistent); + } + + Result RawKey(AZStd::string_view key, Lifetime lifetime) override + { + const bool shouldCopy = lifetime == Lifetime::Temporary; + return CheckWrite(m_writer.Key(key.data(), static_cast(key.size()), shouldCopy)); + } + + Result StartArray() override + { + return CheckWrite(m_writer.StartArray()); + } + + Result EndArray(AZ::u64 elementCount) override + { + return CheckWrite(m_writer.EndArray(static_cast(elementCount))); + } + + private: + Result CheckWrite(bool writeSucceeded) + { + if (!writeSucceeded) + { + return VisitorFailure(VisitorErrorCode::InternalError, "Failed to write JSON"); + } + return VisitorSuccess(); + } + + AZ::IO::RapidJSONStreamWriter m_streamWriter; + TWriter m_writer; + }; + + // + // struct JsonReadHandler + // + // Handler for a rapidjson::Reader that translates reads into an AZ::DOM::Visitor + struct JsonReadHandler : public rapidjson::BaseReaderHandler, JsonReadHandler> + { + public: + JsonReadHandler(Visitor* visitor, Lifetime stringLifetime) + : m_visitor(visitor) + , m_stringLifetime(stringLifetime) + , m_outcome(AZ::Success()) + { + } + + bool Null() + { + return CheckResult(m_visitor->Null()); + } + + bool Bool(bool b) + { + return CheckResult(m_visitor->Bool(b)); + } + + bool Int(int i) + { + return CheckResult(m_visitor->Int64(static_cast(i))); + } + + bool Uint(unsigned i) + { + return CheckResult(m_visitor->Uint64(static_cast(i))); + } + + bool Int64(int64_t i) + { + return CheckResult(m_visitor->Int64(i)); + } + + bool Uint64(uint64_t i) + { + return CheckResult(m_visitor->Uint64(i)); + } + + bool Double(double d) + { + return CheckResult(m_visitor->Double(d)); + } + + bool RawNumber([[maybe_unused]] const Ch* str, [[maybe_unused]] rapidjson::SizeType length, [[maybe_unused]] bool copy) + { + AZ_Assert(false, "Raw numbers are unsupported"); + return false; + } + + bool String(const Ch* str, rapidjson::SizeType length, bool copy) + { + Lifetime lifetime = m_stringLifetime; + if (!copy) + { + lifetime = Lifetime::Temporary; + } + return CheckResult(m_visitor->String(AZStd::string_view(str, length), lifetime)); + } + + bool StartObject() + { + return CheckResult(m_visitor->StartObject()); + } + + bool Key(const Ch* str, rapidjson::SizeType length, [[maybe_unused]] bool copy) + { + AZStd::string_view key = AZStd::string_view(str, length); + if (!m_visitor->SupportsRawKeys()) + { + m_visitor->Key(AZ::Name(key)); + } + Lifetime lifetime = m_stringLifetime; + if (!copy) + { + lifetime = Lifetime::Temporary; + } + return CheckResult(m_visitor->RawKey(key, lifetime)); + } + + bool EndObject([[maybe_unused]] rapidjson::SizeType memberCount) + { + return CheckResult(m_visitor->EndObject(memberCount)); + } + + bool StartArray() + { + return CheckResult(m_visitor->StartArray()); + } + + bool EndArray([[maybe_unused]] rapidjson::SizeType elementCount) + { + return CheckResult(m_visitor->EndArray(elementCount)); + } + + Visitor::Result&& TakeOutcome() + { + return AZStd::move(m_outcome); + } + + private: + bool CheckResult(Visitor::Result result) + { + if (!result.IsSuccess()) + { + m_outcome = AZStd::move(result); + return false; + } + return true; + } + + Visitor::Result m_outcome; + Visitor* m_visitor; + Lifetime m_stringLifetime; + }; + + // + // struct AzStringStream + // + // rapidjson stream wrapper for AZStd::string suitable for in-situ parsing + struct AzStringStream + { + using Ch = char; + + AzStringStream(AZStd::string& buffer) + { + m_cursor = buffer.data(); + m_begin = m_cursor; + } + + char Peek() const + { + return *m_cursor; + } + + char Take() + { + return *m_cursor++; + } + + size_t Tell() const + { + return static_cast(m_cursor - m_begin); + } + + char* PutBegin() + { + m_write = m_cursor; + return m_cursor; + } + + void Put(char c) + { + (*m_write++) = c; + } + + void Flush() + { + } + + size_t PutEnd(char* begin) + { + return m_write - begin; + } + + const char* Peek4() const + { + AZ_Assert(false, "Not implemented, encoding is hard-coded to UTF-8"); + } + + char* m_cursor; //!< Current read position. + char* m_write; //!< Current write position. + const char* m_begin; //!< Head of string. + }; + + // + // Serialized JSON util functions + // + AZStd::unique_ptr GetJsonStreamWriter(AZ::IO::GenericStream* stream, OutputFormatting format) + { + if (format == OutputFormatting::MinifiedJson) + { + using WriterType = rapidjson::Writer; + return AZStd::make_unique>(stream); + } + else + { + using WriterType = rapidjson::PrettyWriter; + return AZStd::make_unique>(stream); + } + } + + Visitor::Result VisitSerializedJson(AZStd::string_view buffer, Lifetime lifetime, Visitor* visitor) + { + rapidjson::Reader reader; + rapidjson::MemoryStream stream(buffer.data(), buffer.size()); + JsonReadHandler handler(visitor, lifetime); + + constexpr int flags = rapidjson::kParseCommentsFlag; + reader.Parse(stream, handler); + return handler.TakeOutcome(); + } + + Visitor::Result VisitSerializedJsonInPlace(AZStd::string& buffer, Visitor* visitor) + { + rapidjson::Reader reader; + AzStringStream stream(buffer); + JsonReadHandler handler(visitor, Lifetime::Persistent); + + constexpr int flags = rapidjson::kParseCommentsFlag | rapidjson::kParseInsituFlag; + reader.Parse(stream, handler); + return handler.TakeOutcome(); + } + + // + // In-memory rapidjson util functions + // + AZ::Outcome WriteToRapidJsonDocument(Backend::WriteCallback writeCallback) + { + DocumentWriter writer; + auto result = writeCallback(&writer); + if (!result.IsSuccess()) + { + return AZ::Failure(result.TakeError().FormatVisitorErrorMessage()); + } + return AZ::Success(writer.TakeDocument()); + } + + Visitor::Result VisitRapidJsonValue(const rapidjson::Value& value, Visitor* visitor, Lifetime lifetime) + { + enum class EndMarker + { + EndArray, + EndObject + }; + + // Processing stack consists of values comprised of one of a: + // - rapidjson::Value to process + // - EndMarker denoting the end of an array or object + // - string denoting a key at the beginning of a key/value pair + using Entry = AZStd::variant; + AZStd::stack entryStack; + AZStd::stack entryCountStack; + entryStack.push(&value); + + while (!entryStack.empty()) + { + const auto currentEntry = entryStack.top(); + entryStack.pop(); + + if (AZStd::holds_alternative(currentEntry)) + { + EndMarker marker = AZStd::get(currentEntry); + if (marker == EndMarker::EndArray) + { + visitor->EndArray(entryCountStack.top()); + } + else + { + visitor->EndObject(entryCountStack.top()); + } + entryCountStack.pop(); + continue; + } + + if (AZStd::holds_alternative(currentEntry)) + { + AZStd::string_view key = AZStd::get(currentEntry); + if (visitor->SupportsRawKeys()) + { + visitor->RawKey(key, lifetime); + } + else + { + visitor->Key(AZ::Name(key)); + } + continue; + } + + const rapidjson::Value& currentValue = *AZStd::get(currentEntry); + if (!entryCountStack.empty()) + { + ++entryCountStack.top(); + } + + Visitor::Result result = AZ::Success(); + + switch (currentValue.GetType()) + { + case rapidjson::kNullType: + result = visitor->Null(); + break; + case rapidjson::kFalseType: + result = visitor->Bool(false); + break; + case rapidjson::kTrueType: + result = visitor->Bool(true); + break; + case rapidjson::kObjectType: + entryStack.push(EndMarker::EndObject); + entryCountStack.push(0); + result = visitor->StartObject(); + for (auto it = currentValue.MemberEnd(); it != currentValue.MemberBegin(); --it) + { + auto entry = (it - 1); + const AZStd::string_view key(entry->name.GetString(), static_cast(entry->name.GetStringLength())); + entryStack.push(&entry->value); + entryStack.push(key); + } + break; + case rapidjson::kArrayType: + entryStack.push(EndMarker::EndArray); + entryCountStack.push(0); + result = visitor->StartArray(); + for (auto it = currentValue.End(); it != currentValue.Begin(); --it) + { + auto entry = (it - 1); + entryStack.push(entry); + } + break; + case rapidjson::kStringType: + result = visitor->String( + AZStd::string_view(currentValue.GetString(), static_cast(currentValue.GetStringLength())), lifetime); + break; + case rapidjson::kNumberType: + if (currentValue.IsFloat() || currentValue.IsDouble()) + { + result = visitor->Double(currentValue.GetDouble()); + } + else if (currentValue.IsInt64() || currentValue.IsInt()) + { + result = visitor->Int64(currentValue.GetInt64()); + } + else + { + result = visitor->Uint64(currentValue.GetUint64()); + } + break; + default: + return AZ::Failure(VisitorError(VisitorErrorCode::InvalidData, "Value with invalid type specified")); + } + + if (!result.IsSuccess()) + { + return result; + } + } + + return AZ::Success(); + } +} // namespace AZ::DOM::Json diff --git a/Code/Framework/AzCore/AzCore/DOM/Backends/JSON/JsonSerializationUtils.h b/Code/Framework/AzCore/AzCore/DOM/Backends/JSON/JsonSerializationUtils.h new file mode 100644 index 0000000000..323e0b880c --- /dev/null +++ b/Code/Framework/AzCore/AzCore/DOM/Backends/JSON/JsonSerializationUtils.h @@ -0,0 +1,60 @@ +/* + * Copyright (c) Contributors to the Open 3D Engine Project. + * For complete copyright and license terms please see the LICENSE at the root of this distribution. + * + * SPDX-License-Identifier: Apache-2.0 OR MIT + * + */ + +#pragma once + +#include +#include +#include +#include +#include +#include +#include + +namespace AZ::DOM::Json +{ + //! Specifies how JSON should be formatted when serialized. + enum class OutputFormatting + { + MinifiedJson, //!< Formats JSON in compact minified form, focusing on minimizing output size. + PrettyPrintedJson, //!< Formats JSON in a pretty printed form, focusing on legibility to readers. + }; + + //! Creates a Visitor that will write serialized JSON to the specified stream. + //! \param stream The stream the visitor will write to. + //! \param format The format to write in. + //! \return A Visitor that will write to stream when visited. + AZStd::unique_ptr GetJsonStreamWriter( + AZ::IO::GenericStream* stream, OutputFormatting format = OutputFormatting::PrettyPrintedJson); + //! Reads serialized JSON from a string and applies it to a visitor. + //! \param buffer The UTF-8 serialized JSON to read. + //! \param lifetime Specifies the lifetime of the specified buffer. If the string specified by buffer might be deallocated, + //! ensure specify Lifetime::Temporary is specified. + //! \param visitor The visitor to visit with the JSON buffer's contents. + //! \return The aggregate result specifying whether the visitor operations were successful. + Visitor::Result VisitSerializedJson(AZStd::string_view buffer, Lifetime lifetime, Visitor* visitor); + //! Reads serialized JSON from a string in-place and applies it to a visitor. + //! \param buffer The UTF-8 serialized JSON to read. This buffer will be modified as part of the deserialization process to + //! apply null terminators. + //! \param visitor The visitor to visit with the JSON buffer's contents. The strings provided to the visitor will only + //! be valid for the lifetime of buffer. + //! \return The aggregate result specifying whether the visitor operations were successful. + Visitor::Result VisitSerializedJsonInPlace(AZStd::string& buffer, Visitor* visitor); + + //! Takes a visitor specified by a callback and produces a rapidjson::Document. + //! \param writeCallback A callback specifying a visitor to accept to build the resulting document. + //! \return An outcome with either the rapidjson::Document or an error message. + AZ::Outcome WriteToRapidJsonDocument(Backend::WriteCallback writeCallback); + //! Accepts a visitor with the contents of a rapidjson::Value. + //! \param value The rapidjson::Value to apply to visitor. + //! \param visitor The visitor to receive the contents of value. + //! \param lifetime The lifetime to specify for visiting strings. If the rapidjson::Value might be destroyed or changed + //! before the visitor is finished using these values, Lifetime::Temporary should be specified. + //! \return The aggregate result specifying whether the visitor operations were successful. + Visitor::Result VisitRapidJsonValue(const rapidjson::Value& value, Visitor* visitor, Lifetime lifetime); +} // namespace AZ::DOM::Json diff --git a/Code/Framework/AzCore/AzCore/DOM/DomBackend.cpp b/Code/Framework/AzCore/AzCore/DOM/DomBackend.cpp new file mode 100644 index 0000000000..f26e34acdf --- /dev/null +++ b/Code/Framework/AzCore/AzCore/DOM/DomBackend.cpp @@ -0,0 +1,77 @@ +/* + * Copyright (c) Contributors to the Open 3D Engine Project. + * For complete copyright and license terms please see the LICENSE at the root of this distribution. + * + * SPDX-License-Identifier: Apache-2.0 OR MIT + * + */ + +#include + +#include +#include +#include "DomBackend.h" + +namespace AZ::DOM +{ + Visitor::Result Backend::ReadFromPath(AZ::IO::PathView pathName, Visitor* visitor, size_t maxFileSize) + { + auto readResult = AZ::Utils::ReadFile(pathName.Native(), maxFileSize); + if (!readResult.IsSuccess()) + { + return AZ::Failure(VisitorError(VisitorErrorCode::InternalError, readResult.TakeError())); + } + + AZStd::string fileContents = readResult.TakeValue(); + return ReadFromString(fileContents, Lifetime::Temporary, visitor); + } + + Visitor::Result Backend::ReadFromStream(AZ::IO::GenericStream* stream, Visitor* visitor, size_t maxSize) + { + size_t length = stream->GetLength(); + if (length > maxSize) + { + return AZ::Failure(VisitorError(VisitorErrorCode::InternalError, "Stream is too large.")); + } + AZStd::string buffer; + buffer.resize(maxSize); + stream->Read(length, buffer.data()); + return ReadFromString(buffer, Lifetime::Temporary, visitor); + } + + Visitor::Result Backend::ReadFromStringInPlace(AZStd::string& buffer, Visitor* visitor) + { + return ReadFromString(buffer, Lifetime::Persistent, visitor); + } + + Visitor::Result Backend::WriteToStream(AZ::IO::GenericStream* stream, WriteCallback callback) + { + AZStd::unique_ptr writer = CreateStreamWriter(stream); + return callback(writer.get()); + } + + Visitor::Result Backend::WriteToPath(AZ::IO::PathView pathName, WriteCallback callback) + { + AZStd::string buffer; + auto serializeResult = WriteToString(buffer, callback); + if (!serializeResult.IsSuccess()) + { + return serializeResult; + } + + auto writeResult = AZ::Utils::WriteFile(buffer, pathName.Native()); + if (!writeResult.IsSuccess()) + { + return AZ::Failure(VisitorError(VisitorErrorCode::InternalError, writeResult.TakeError())); + } + + return AZ::Success(); + } + + Visitor::Result Backend::WriteToString(AZStd::string& buffer, WriteCallback callback) + { + AZ::IO::ByteContainerStream stream{&buffer}; + AZStd::unique_ptr writer = CreateStreamWriter(&stream); + return callback(writer.get()); + } +} diff --git a/Code/Framework/AzCore/AzCore/DOM/DomBackend.h b/Code/Framework/AzCore/AzCore/DOM/DomBackend.h new file mode 100644 index 0000000000..bb5c9e40fe --- /dev/null +++ b/Code/Framework/AzCore/AzCore/DOM/DomBackend.h @@ -0,0 +1,54 @@ +/* + * Copyright (c) Contributors to the Open 3D Engine Project. + * For complete copyright and license terms please see the LICENSE at the root of this distribution. + * + * SPDX-License-Identifier: Apache-2.0 OR MIT + * + */ + +#pragma once + +#include +#include +#include +#include +#include + +namespace AZ::DOM +{ + //! Backends are registered centrally and used to transition DOM formats to and from a textual format. + class Backend + { + public: + virtual ~Backend() = default; + + //! Attempt to read this format from the given file into the target Visitor. + Visitor::Result ReadFromPath( + AZ::IO::PathView pathName, Visitor* visitor, size_t maxFileSize = AZStd::numeric_limits::max()); + //! Attempt to read this format from the given stream into the target Visitor. + //! The base implementation reads the stream into memory and calls ReadFromString. + virtual Visitor::Result ReadFromStream( + AZ::IO::GenericStream* stream, Visitor* visitor, size_t maxSize = AZStd::numeric_limits::max()); + //! Attempt to read this format from a mutable string into the target Visitor. This enables some backends to + //! parse without making additional string allocations. + //! This string may be modified and read in place without being copied, so when calling this please ensure that: + //! - The string won't be deallocated until the visitor no longer needs the values and + //! - The string is safe to modify in place. + //! The base implementation simply calls ReadFromString. + virtual Visitor::Result ReadFromStringInPlace(AZStd::string& buffer, Visitor* visitor); + //! Attempt to read this format from an immutable buffer in memory into the target Visitor. + virtual Visitor::Result ReadFromString(AZStd::string_view buffer, Lifetime lifetime, Visitor* visitor) = 0; + + //! Acquire a visitor interface for writing to the target output file. + virtual AZStd::unique_ptr CreateStreamWriter(AZ::IO::GenericStream* stream) = 0; + //! A callback that accepts a Visitor, making DOM calls to inform the serializer, and returns an + //! aggregate error code to indicate whether or not the operation succeeded. + using WriteCallback = AZStd::function; + //! Attempt to write a value to a stream using a write callback. + Visitor::Result WriteToStream(AZ::IO::GenericStream* stream, WriteCallback callback); + //! Attempt to write a value to a file using a write callback. + Visitor::Result WriteToPath(AZ::IO::PathView pathName, WriteCallback callback); + //! Attempt to write a value to a string using a write callback. + Visitor::Result WriteToString(AZStd::string& buffer, WriteCallback callback); + }; +} // namespace AZ::DOM diff --git a/Code/Framework/AzCore/AzCore/DOM/DomBackendRegistry.cpp b/Code/Framework/AzCore/AzCore/DOM/DomBackendRegistry.cpp new file mode 100644 index 0000000000..c8f4050602 --- /dev/null +++ b/Code/Framework/AzCore/AzCore/DOM/DomBackendRegistry.cpp @@ -0,0 +1,68 @@ +/* + * Copyright (c) Contributors to the Open 3D Engine Project. + * For complete copyright and license terms please see the LICENSE at the root of this distribution. + * + * SPDX-License-Identifier: Apache-2.0 OR MIT + * + */ + +#include + +#include + +namespace AZ::DOM +{ + BackendRegistry* BackendRegistry::s_instance = nullptr; + + BackendRegistryInterface* BackendRegistry::Get() + { + return AZ::Interface::Get(); + } + + void BackendRegistry::Create() + { + AZ_Assert(!s_instance, "Attempted to register BackendRegistry when it's already registered"); + s_instance = aznew BackendRegistry; + AZ::Interface::Register(s_instance); + } + + void BackendRegistry::Destroy() + { + AZ_Assert(s_instance, "Attempted to unregister a non-existent BackendRegistry"); + AZ::Interface::Unregister(s_instance); + delete s_instance; + s_instance = nullptr; + } + + AZStd::unique_ptr BackendRegistry::GetBackendByName(AZStd::string_view name) + { + auto backendIterator = m_nameToBackend.find(name); + if (backendIterator != m_nameToBackend.end()) + { + return backendIterator->second(); + } + return nullptr; + } + + AZStd::unique_ptr BackendRegistry::GetBackendForExtension(AZStd::string_view extension) + { + auto extensionIterator = m_extensionToName.find(extension); + if (extensionIterator != m_extensionToName.end()) + { + return GetBackendByName(extensionIterator->second); + } + return nullptr; + } + + void BackendRegistry::RegisterBackendInternal( + BackendRegistryInterface::BackendFactory factory, AZStd::string name, AZStd::vector extensions) + { + AZ_Assert(m_nameToBackend.find(name) == m_nameToBackend.end(), "DOM Backend %s already registered", name.c_str()); + m_nameToBackend.insert({name, factory}); + for (AZStd::string& extension : extensions) + { + AZ_Assert(m_extensionToName.find(extension) == m_extensionToName.end(), "DOM Extensions already registered", extension.c_str()); + m_extensionToName.insert({AZStd::move(extension), name}); + } + } +} diff --git a/Code/Framework/AzCore/AzCore/DOM/DomBackendRegistry.h b/Code/Framework/AzCore/AzCore/DOM/DomBackendRegistry.h new file mode 100644 index 0000000000..c71e6798fc --- /dev/null +++ b/Code/Framework/AzCore/AzCore/DOM/DomBackendRegistry.h @@ -0,0 +1,44 @@ +/* + * Copyright (c) Contributors to the Open 3D Engine Project. + * For complete copyright and license terms please see the LICENSE at the root of this distribution. + * + * SPDX-License-Identifier: Apache-2.0 OR MIT + * + */ + +#pragma once + +#include +#include + +namespace AZ::DOM +{ + //! DOM backend registry implementation. + //! \see BackendRegistryInterface + class BackendRegistry final : public BackendRegistryInterface + { + public: + AZ_RTTI(BackendRegistry, "{95533861-6201-4E45-BD03-097A20850C48}", BackendRegistryInterface); + AZ_CLASS_ALLOCATOR(BackendRegistry, AZ::SystemAllocator, 0); + + AZStd::unique_ptr GetBackendByName(AZStd::string_view name) override; + AZStd::unique_ptr GetBackendForExtension(AZStd::string_view extension) override; + + //! Convenience method, gets the current instance of the BackendRegistry via AZ::Interface. + static BackendRegistryInterface* Get(); + //! Creates a singleton BackendRegistry and registers it via AZ::Interface. + static void Create(); + //! Destroys the singleton BackendRegistry created by Create and unregisters it from AZ::Interface. + static void Destroy(); + + protected: + void RegisterBackendInternal(BackendFactory factory, AZStd::string name, AZStd::vector extensions) override; + + private: + using BackendFactory = AZStd::function()>; + AZStd::unordered_map m_nameToBackend; + AZStd::unordered_map m_extensionToName; + + static BackendRegistry* s_instance; + }; +} // namespace AZ::DOM diff --git a/Code/Framework/AzCore/AzCore/DOM/DomBackendRegistryInterface.h b/Code/Framework/AzCore/AzCore/DOM/DomBackendRegistryInterface.h new file mode 100644 index 0000000000..4651f35961 --- /dev/null +++ b/Code/Framework/AzCore/AzCore/DOM/DomBackendRegistryInterface.h @@ -0,0 +1,54 @@ +/* + * Copyright (c) Contributors to the Open 3D Engine Project. + * For complete copyright and license terms please see the LICENSE at the root of this distribution. + * + * SPDX-License-Identifier: Apache-2.0 OR MIT + * + */ + +#pragma once + +#include + +namespace AZ::DOM +{ + //! Central interface for registering and looking up backend types by file extension. + class BackendRegistryInterface + { + public: + AZ_RTTI(BackendRegistryInterface, "{B72A88AC-DF15-4F92-9489-ABCDEA3D94E6}") + using BackendFactory = AZStd::function()>; + + virtual ~BackendRegistryInterface() = default; + + //! Registers a factory for a given backend. + //! \param name A unique name to identify this backend. + //! \param extensions A set of file extensions this backend supports. + //! Extensions should by the entire file suffix including any dots (.), for example: + //! ".json" or ".tar.gz" + template + void RegisterBackend(AZStd::string name, AZStd::vector extensions); + + //! Looks up a DOM backend based on its name. + virtual AZStd::unique_ptr GetBackendByName(AZStd::string_view name) = 0; + + //! Looks up a DOM backend based on a file extension. + virtual AZStd::unique_ptr GetBackendForExtension(AZStd::string_view extension) = 0; + + protected: + //! Registers a factory for a given backend, called by RegisterBackend. + //! \param factory The factory function to create new backend instances. + //! \param name The unique name to identify this backend. + //! \param extensions A set of file extensions this backend supports. + //! \see RegisterBackend + virtual void RegisterBackendInternal(BackendFactory factory, AZStd::string name, AZStd::vector extensions) = 0; + }; + + template + void BackendRegistryInterface::RegisterBackend(AZStd::string name, AZStd::vector extensions) + { + RegisterBackendInternal([](){ + return AZStd::make_unique(); + }, AZStd::move(name), AZStd::move(extensions)); + } +} // namespace AZ::DOM diff --git a/Code/Framework/AzCore/AzCore/DOM/DomVisitor.h b/Code/Framework/AzCore/AzCore/DOM/DomVisitor.h index 584cfce4ed..eb6f20ae1b 100644 --- a/Code/Framework/AzCore/AzCore/DOM/DomVisitor.h +++ b/Code/Framework/AzCore/AzCore/DOM/DomVisitor.h @@ -17,7 +17,7 @@ namespace AZ::DOM { // // Lifetime enum - // + // //! Specifies the period in which a reference value will still be alive and safe to read. enum class Lifetime { @@ -30,7 +30,7 @@ namespace AZ::DOM // // VisitorErrorCode enum - // + // //! Error code specifying the reason a Visitor operation failed. enum class VisitorErrorCode { @@ -75,7 +75,7 @@ namespace AZ::DOM }; //! A type alias for opaque DOM types that aren't meant to be serializable. - //! /see VisitorInterface::OpaqueValue + //! \see VisitorInterface::OpaqueValue using OpaqueType = AZStd::any; // @@ -116,7 +116,7 @@ namespace AZ::DOM //! - \ref Double: 64 bit double precision float //! - \ref Null: sentinel "empty" type with no value representation //! - \ref String: UTF8 encoded string - //! - \ref Object: an ordered container of key/value pairs where keys are AZ::Names and values may be any DOM type + //! - \ref Object: an ordered container of key/value pairs where keys are \ref AZ::Name and values may be any DOM type //! (including Object) //! - \ref Array: an ordered container of values, in which values are any DOM value type (including Array) //! - \ref Node: a container @@ -144,17 +144,17 @@ namespace AZ::DOM //! Raw (\see VisitorFlags::SupportsRawValues) and opaque values (\see VisitorFlags::SupportsOpaqueValues) //! are disallowed by default, as their handling is intended to be implementation-specific. virtual VisitorFlags GetVisitorFlags() const; - //! /see VisitorFlags::SupportsRawValues + //! \see VisitorFlags::SupportsRawValues bool SupportsRawValues() const; - //! /see VisitorFlags::SupportsRawKeys + //! \see VisitorFlags::SupportsRawKeys bool SupportsRawKeys() const; - //! /see VisitorFlags::SupportsObjects + //! \see VisitorFlags::SupportsObjects bool SupportsObjects() const; - //! /see VisitorFlags::SupportsArrays + //! \see VisitorFlags::SupportsArrays bool SupportsArrays() const; - //! /see VisitorFlags::SupportsNodes + //! \see VisitorFlags::SupportsNodes bool SupportsNodes() const; - //! /see VisitorFlags::SupportsOpaqueValues + //! \see VisitorFlags::SupportsOpaqueValues bool SupportsOpaqueValues() const; //! Operates on an empty null value. @@ -231,6 +231,15 @@ namespace AZ::DOM static Result VisitorFailure(VisitorErrorCode code, AZStd::string additionalInfo); //! Helper method, constructs a failure \ref Result with the specified error. static Result VisitorFailure(VisitorError error); + + //! Helper method, constructs a failure \ref Result with the specified code and supplemental info specified by a format string + //! and its arguments. + template + static Result FormatVisitorFailure(VisitorErrorCode code, TArgs... formatArgs) + { + return VisitorFailure(code, AZStd::string::format(formatArgs...)); + } + //! Helper method, constructs a success \ref Result. static Result VisitorSuccess(); }; diff --git a/Code/Framework/AzCore/AzCore/azcore_files.cmake b/Code/Framework/AzCore/AzCore/azcore_files.cmake index a5cc3fdcd4..54f2c9ecca 100644 --- a/Code/Framework/AzCore/AzCore/azcore_files.cmake +++ b/Code/Framework/AzCore/AzCore/azcore_files.cmake @@ -125,8 +125,18 @@ set(FILES Debug/TraceMessagesDrillerBus.h Debug/TraceReflection.cpp Debug/TraceReflection.h + DOM/DomAdapter.h + DOM/DomBackend.cpp + DOM/DomBackend.h + DOM/DomBackendRegistry.cpp + DOM/DomBackendRegistry.h + DOM/DomBackendRegistryInterface.h DOM/DomVisitor.cpp DOM/DomVisitor.h + DOM/Backends/JSON/JsonBackend.cpp + DOM/Backends/JSON/JsonBackend.h + DOM/Backends/JSON/JsonSerializationUtils.cpp + DOM/Backends/JSON/JsonSerializationUtils.h Driller/DefaultStringPool.h Driller/Driller.cpp Driller/Driller.h diff --git a/Code/Framework/AzCore/Tests/DOM/DomJsonBenchmarks.cpp b/Code/Framework/AzCore/Tests/DOM/DomJsonBenchmarks.cpp new file mode 100644 index 0000000000..238802a35f --- /dev/null +++ b/Code/Framework/AzCore/Tests/DOM/DomJsonBenchmarks.cpp @@ -0,0 +1,184 @@ +/* + * Copyright (c) Contributors to the Open 3D Engine Project. + * For complete copyright and license terms please see the LICENSE at the root of this distribution. + * + * SPDX-License-Identifier: Apache-2.0 OR MIT + * + */ + +#if defined(HAVE_BENCHMARK) + +#include +#include +#include +#include +#include +#include + +namespace Benchmark +{ + class DomJsonBenchmark : public UnitTest::AllocatorsBenchmarkFixture + { + public: + void SetUp([[maybe_unused]] const ::benchmark::State& st) override + { + UnitTest::AllocatorsBenchmarkFixture::SetUp(st); + AZ::NameDictionary::Create(); + } + + void SetUp([[maybe_unused]] ::benchmark::State& st) override + { + UnitTest::AllocatorsBenchmarkFixture::SetUp(st); + AZ::NameDictionary::Create(); + } + + void TearDown([[maybe_unused]] ::benchmark::State& st) override + { + AZ::NameDictionary::Destroy(); + UnitTest::AllocatorsBenchmarkFixture::TearDown(st); + } + + void TearDown([[maybe_unused]] const ::benchmark::State& st) override + { + AZ::NameDictionary::Destroy(); + UnitTest::AllocatorsBenchmarkFixture::TearDown(st); + } + + AZStd::string GenerateDomJsonBenchmarkPayload(int64_t entryCount = 100, int64_t stringTemplateLength = 5) + { + rapidjson::Document document; + document.SetObject(); + + AZStd::string entryTemplate; + while (entryTemplate.size() < static_cast(stringTemplateLength)) + { + entryTemplate += "Lorem ipsum dolor sit amet, consectetur adipiscing elit, sed do eiusmod tempor "; + } + entryTemplate.resize(stringTemplateLength); + AZStd::string buffer; + + auto createString = [&](int n) -> rapidjson::Value + { + buffer = AZStd::string::format("#%i %s", n, entryTemplate.c_str()); + return rapidjson::Value(buffer.data(), static_cast(buffer.size()), document.GetAllocator()); + }; + + auto createEntry = [&](int n) -> rapidjson::Value + { + rapidjson::Value entry(rapidjson::kObjectType); + entry.AddMember("string", createString(n), document.GetAllocator()); + entry.AddMember("int", rapidjson::Value(n), document.GetAllocator()); + entry.AddMember("double", rapidjson::Value(static_cast(n) * 0.5), document.GetAllocator()); + entry.AddMember("bool", rapidjson::Value(n % 2 == 0), document.GetAllocator()); + entry.AddMember("null", rapidjson::Value(rapidjson::kNullType), document.GetAllocator()); + return entry; + }; + + auto createArray = [&]() -> rapidjson::Value + { + rapidjson::Value array; + array.SetArray(); + for (int i = 0; i < entryCount; ++i) + { + array.PushBack(createEntry(i), document.GetAllocator()); + } + return array; + }; + + auto createObject = [&]() -> rapidjson::Value + { + rapidjson::Value object; + object.SetObject(); + for (int i = 0; i < entryCount; ++i) + { + buffer = AZStd::string::format("Key%i", i); + rapidjson::Value key; + key.SetString(buffer.data(), static_cast(buffer.length()), document.GetAllocator()); + object.AddMember(key.Move(), createArray(), document.GetAllocator()); + } + return object; + }; + + document.SetObject(); + document.AddMember("entries", createObject(), document.GetAllocator()); + + AZStd::string serializedJson; + auto result = AZ::JsonSerializationUtils::WriteJsonString(document, serializedJson); + AZ_Assert(result.IsSuccess(), "Failed to serialize generated JSON"); + return serializedJson; + } + }; + +// Helper macro for registering JSON benchmarks +#define BENCHMARK_REGISTER_JSON(BaseClass, Method) \ + BENCHMARK_REGISTER_F(BaseClass, Method) \ + ->Args({ 10, 5 }) \ + ->Args({ 10, 500 }) \ + ->Args({ 100, 5 }) \ + ->Args({ 100, 500 }) \ + ->Unit(benchmark::kMillisecond); + + BENCHMARK_DEFINE_F(DomJsonBenchmark, DomDeserializeToDocumentInPlace)(benchmark::State& state) + { + AZ::DOM::JsonBackend backend; + AZStd::string serializedPayload = GenerateDomJsonBenchmarkPayload(state.range(0), state.range(1)); + + for (auto _ : state) + { + state.PauseTiming(); + AZStd::string payloadCopy = serializedPayload; + state.ResumeTiming(); + + auto result = AZ::DOM::Json::WriteToRapidJsonDocument( + [&](AZ::DOM::Visitor* visitor) + { + return backend.ReadFromStringInPlace(payloadCopy, visitor); + }); + + benchmark::DoNotOptimize(result.GetValue()); + } + + state.SetBytesProcessed(serializedPayload.size() * state.iterations()); + } + BENCHMARK_REGISTER_JSON(DomJsonBenchmark, DomDeserializeToDocumentInPlace) + + BENCHMARK_DEFINE_F(DomJsonBenchmark, DomDeserializeToDocument)(benchmark::State& state) + { + AZ::DOM::JsonBackend backend; + AZStd::string serializedPayload = GenerateDomJsonBenchmarkPayload(state.range(0), state.range(1)); + + for (auto _ : state) + { + auto result = AZ::DOM::Json::WriteToRapidJsonDocument( + [&](AZ::DOM::Visitor* visitor) + { + return backend.ReadFromString(serializedPayload, AZ::DOM::Lifetime::Temporary, visitor); + }); + + benchmark::DoNotOptimize(result.GetValue()); + } + + state.SetBytesProcessed(serializedPayload.size() * state.iterations()); + } + BENCHMARK_REGISTER_JSON(DomJsonBenchmark, DomDeserializeToDocument) + + BENCHMARK_DEFINE_F(DomJsonBenchmark, JsonUtilsDeserializeToDocument)(benchmark::State& state) + { + AZ::DOM::JsonBackend backend; + AZStd::string serializedPayload = GenerateDomJsonBenchmarkPayload(state.range(0), state.range(1)); + + for (auto _ : state) + { + auto result = AZ::JsonSerializationUtils::ReadJsonString(serializedPayload); + + benchmark::DoNotOptimize(result.GetValue()); + } + + state.SetBytesProcessed(serializedPayload.size() * state.iterations()); + } + BENCHMARK_REGISTER_JSON(DomJsonBenchmark, JsonUtilsDeserializeToDocument) + +#undef BENCHMARK_REGISTER_JSON +} // namespace Benchmark + +#endif // defined(HAVE_BENCHMARK) diff --git a/Code/Framework/AzCore/Tests/DOM/DomJsonTests.cpp b/Code/Framework/AzCore/Tests/DOM/DomJsonTests.cpp new file mode 100644 index 0000000000..e778a449d5 --- /dev/null +++ b/Code/Framework/AzCore/Tests/DOM/DomJsonTests.cpp @@ -0,0 +1,288 @@ +/* + * Copyright (c) Contributors to the Open 3D Engine Project. + * For complete copyright and license terms please see the LICENSE at the root of this distribution. + * + * SPDX-License-Identifier: Apache-2.0 OR MIT + * + */ + +#include +#include +#include +#include +#include + +namespace AZ::DOM::Tests +{ + class DomJsonTests : public UnitTest::AllocatorsFixture + { + public: + void SetUp() override + { + UnitTest::AllocatorsFixture::SetUp(); + NameDictionary::Create(); + m_document = AZStd::make_unique(); + } + + void TearDown() override + { + m_document.reset(); + NameDictionary::Destroy(); + UnitTest::AllocatorsFixture::TearDown(); + } + + static bool DeepCompare(const rapidjson::Value& lhs, const rapidjson::Value& rhs) + { + if (lhs.GetType() != rhs.GetType()) + { + return false; + } + + switch (lhs.GetType()) + { + case rapidjson::kNullType: + return true; + case rapidjson::kFalseType: + return true; + case rapidjson::kTrueType: + return true; + case rapidjson::kObjectType: + { + if (lhs.MemberCount() != rhs.MemberCount()) + { + return false; + } + + auto lhsIt = lhs.MemberBegin(); + auto rhsIt = rhs.MemberBegin(); + while (lhsIt != lhs.MemberEnd()) + { + if (lhsIt->name != rhsIt->name) + { + return false; + } + + if (!DeepCompare(lhsIt->value, rhsIt->value)) + { + return false; + } + + ++lhsIt; + ++rhsIt; + } + return true; + } + case rapidjson::kArrayType: + { + if (lhs.Size() != rhs.Size()) + { + return false; + } + + auto lhsIt = lhs.Begin(); + auto rhsIt = rhs.Begin(); + while (lhsIt != lhs.End()) + { + if (!DeepCompare(*lhsIt, *rhsIt)) + { + return false; + } + + ++lhsIt; + ++rhsIt; + } + return true; + } + case rapidjson::kStringType: + return lhs == rhs; + case rapidjson::kNumberType: + return lhs == rhs; + } + + AZ_Assert(false, "Unexpected JSON value type"); + return false; + } + + rapidjson::Value CreateString(const AZStd::string& text) + { + rapidjson::Value key; + key.SetString(text.c_str(), static_cast(text.length()), m_document->GetAllocator()); + return key; + } + + template + void AddValue(const AZStd::string& key, T value) + { + m_document->AddMember(CreateString(key), rapidjson::Value(value), m_document->GetAllocator()); + } + + // Validate round-trip serialization to and from rapidjson::Document and a UTF-8 encoded string + void PerformSerializationChecks() + { + // Generate a canonical serializaed representation of this document using rapidjson + // This will be pretty-printed using the same rapidjson pretty printer we use, so should be binary identical + // to any output generated by the visitor API + AZStd::string canonicalSerializedDocument; + AZ::JsonSerializationUtils::WriteJsonString(*m_document, canonicalSerializedDocument); + + auto visitDocumentFn = [this](AZ::DOM::Visitor* visitor) + { + return Json::VisitRapidJsonValue(*m_document, visitor, Lifetime::Persistent); + }; + + // Document -> Document + { + auto result = Json::WriteToRapidJsonDocument(visitDocumentFn); + EXPECT_TRUE(result.IsSuccess()); + EXPECT_TRUE(DeepCompare(*m_document, result.GetValue())); + } + + // Document -> string + { + AZStd::string serializedDocument; + JsonBackend backend; + auto result = backend.WriteToString(serializedDocument, visitDocumentFn); + EXPECT_TRUE(result.IsSuccess()); + EXPECT_EQ(canonicalSerializedDocument, serializedDocument); + } + + // string -> Document + { + auto result = Json::WriteToRapidJsonDocument( + [&canonicalSerializedDocument](AZ::DOM::Visitor* visitor) + { + JsonBackend backend; + return backend.ReadFromString(canonicalSerializedDocument, AZ::DOM::Lifetime::Temporary, visitor); + }); + EXPECT_TRUE(result.IsSuccess()); + EXPECT_TRUE(DeepCompare(*m_document, result.GetValue())); + } + + // string -> string + { + AZStd::string serializedDocument; + JsonBackend backend; + auto result = backend.WriteToString( + serializedDocument, + [&backend, &canonicalSerializedDocument](AZ::DOM::Visitor* visitor) + { + return backend.ReadFromString(canonicalSerializedDocument, AZ::DOM::Lifetime::Temporary, visitor); + }); + EXPECT_TRUE(result.IsSuccess()); + EXPECT_EQ(canonicalSerializedDocument, serializedDocument); + } + } + + AZStd::unique_ptr m_document; + }; + + TEST_F(DomJsonTests, EmptyArray) + { + m_document->SetArray(); + PerformSerializationChecks(); + } + + TEST_F(DomJsonTests, SimpleArray) + { + m_document->SetArray(); + for (int i = 0; i < 5; ++i) + { + m_document->PushBack(i, m_document->GetAllocator()); + } + PerformSerializationChecks(); + } + + TEST_F(DomJsonTests, NestedArrays) + { + m_document->SetArray(); + for (int j = 0; j < 7; ++j) + { + rapidjson::Value nestedArray(rapidjson::kArrayType); + for (int i = 0; i < 5; ++i) + { + nestedArray.PushBack(i, m_document->GetAllocator()); + } + m_document->PushBack(nestedArray.Move(), m_document->GetAllocator()); + } + PerformSerializationChecks(); + } + + TEST_F(DomJsonTests, EmptyObject) + { + m_document->SetObject(); + PerformSerializationChecks(); + } + + TEST_F(DomJsonTests, SimpleObject) + { + m_document->SetObject(); + for (int i = 0; i < 5; ++i) + { + m_document->AddMember(CreateString(AZStd::string::format("Key%i", i)), rapidjson::Value(i), m_document->GetAllocator()); + } + PerformSerializationChecks(); + } + + TEST_F(DomJsonTests, NestedObjects) + { + m_document->SetObject(); + for (int j = 0; j < 7; ++j) + { + rapidjson::Value nestedObject(rapidjson::kObjectType); + for (int i = 0; i < 5; ++i) + { + nestedObject.AddMember(CreateString(AZStd::string::format("Key%i", i)), rapidjson::Value(i), m_document->GetAllocator()); + } + m_document->AddMember(CreateString(AZStd::string::format("Obj%i", j)), nestedObject.Move(), m_document->GetAllocator()); + } + PerformSerializationChecks(); + } + + TEST_F(DomJsonTests, Int64) + { + m_document->SetObject(); + AddValue("int64_min", AZStd::numeric_limits::min()); + AddValue("int64_max", AZStd::numeric_limits::max()); + PerformSerializationChecks(); + } + + TEST_F(DomJsonTests, Uint64) + { + m_document->SetObject(); + AddValue("uint64_min", AZStd::numeric_limits::min()); + AddValue("uint64_max", AZStd::numeric_limits::max()); + PerformSerializationChecks(); + } + + TEST_F(DomJsonTests, Double) + { + m_document->SetObject(); + AddValue("double_min", AZStd::numeric_limits::min()); + AddValue("double_max", AZStd::numeric_limits::max()); + PerformSerializationChecks(); + } + + TEST_F(DomJsonTests, Null) + { + m_document->SetObject(); + m_document->AddMember(CreateString("null_value"), rapidjson::Value(rapidjson::kNullType), m_document->GetAllocator()); + PerformSerializationChecks(); + } + + TEST_F(DomJsonTests, Bool) + { + m_document->SetObject(); + AddValue("true_value", true); + AddValue("false_value", false); + PerformSerializationChecks(); + } + + TEST_F(DomJsonTests, String) + { + m_document->SetObject(); + m_document->AddMember(CreateString("empty_string"), CreateString(""), m_document->GetAllocator()); + m_document->AddMember(CreateString("short_string"), CreateString("test"), m_document->GetAllocator()); + m_document->AddMember(CreateString("long_string"), CreateString("abcdefghijklmnopqrstuvwxyz0123456789"), m_document->GetAllocator()); + PerformSerializationChecks(); + } +} // namespace AZ::DOM::Tests diff --git a/Code/Framework/AzCore/Tests/azcoretests_files.cmake b/Code/Framework/AzCore/Tests/azcoretests_files.cmake index d39595c45e..0fca75e2a8 100644 --- a/Code/Framework/AzCore/Tests/azcoretests_files.cmake +++ b/Code/Framework/AzCore/Tests/azcoretests_files.cmake @@ -213,6 +213,8 @@ set(FILES AZStd/Variant.cpp AZStd/VariantSerialization.cpp AZStd/VectorAndArray.cpp + DOM/DomJsonTests.cpp + DOM/DomJsonBenchmarks.cpp ) # Prevent the following files from being grouped in UNITY builds From 609469d5e44fc8707e8d477492bab55f8e32f6c4 Mon Sep 17 00:00:00 2001 From: Nicholas Van Sickle Date: Wed, 10 Nov 2021 14:08:59 -0800 Subject: [PATCH 02/18] Fix a couple build issues Signed-off-by: Nicholas Van Sickle --- .../AzCore/AzCore/DOM/Backends/JSON/JsonSerializationUtils.cpp | 2 +- Code/Framework/AzCore/AzCore/azcore_files.cmake | 1 - 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/Code/Framework/AzCore/AzCore/DOM/Backends/JSON/JsonSerializationUtils.cpp b/Code/Framework/AzCore/AzCore/DOM/Backends/JSON/JsonSerializationUtils.cpp index 302fd7bd63..aa0b24a5ad 100644 --- a/Code/Framework/AzCore/AzCore/DOM/Backends/JSON/JsonSerializationUtils.cpp +++ b/Code/Framework/AzCore/AzCore/DOM/Backends/JSON/JsonSerializationUtils.cpp @@ -219,7 +219,6 @@ namespace AZ::DOM::Json rapidjson::Document m_result; AZStd::deque m_entryStack; - AZ::u64 m_entryCount = 0; }; // @@ -490,6 +489,7 @@ namespace AZ::DOM::Json const char* Peek4() const { AZ_Assert(false, "Not implemented, encoding is hard-coded to UTF-8"); + return m_cursor; } char* m_cursor; //!< Current read position. diff --git a/Code/Framework/AzCore/AzCore/azcore_files.cmake b/Code/Framework/AzCore/AzCore/azcore_files.cmake index 54f2c9ecca..abb16d61a0 100644 --- a/Code/Framework/AzCore/AzCore/azcore_files.cmake +++ b/Code/Framework/AzCore/AzCore/azcore_files.cmake @@ -125,7 +125,6 @@ set(FILES Debug/TraceMessagesDrillerBus.h Debug/TraceReflection.cpp Debug/TraceReflection.h - DOM/DomAdapter.h DOM/DomBackend.cpp DOM/DomBackend.h DOM/DomBackendRegistry.cpp From 5dbe9e387b3fc827aa2f7598f116e7f78a87faa9 Mon Sep 17 00:00:00 2001 From: Nicholas Van Sickle Date: Tue, 30 Nov 2021 12:10:16 -0800 Subject: [PATCH 03/18] Address some review feedback, remove DomBackendRegistry Signed-off-by: Nicholas Van Sickle --- .../AzCore/DOM/Backends/JSON/JsonBackend.cpp | 12 +- .../AzCore/DOM/Backends/JSON/JsonBackend.h | 8 +- .../Backends/JSON/JsonSerializationUtils.cpp | 226 +++++++++--------- .../Backends/JSON/JsonSerializationUtils.h | 4 +- .../AzCore/AzCore/DOM/DomBackend.cpp | 34 +-- Code/Framework/AzCore/AzCore/DOM/DomBackend.h | 9 +- .../AzCore/AzCore/DOM/DomBackendRegistry.cpp | 68 ------ .../AzCore/AzCore/DOM/DomBackendRegistry.h | 44 ---- .../AzCore/DOM/DomBackendRegistryInterface.h | 54 ----- .../AzCore/AzCore/DOM/DomVisitor.cpp | 4 +- Code/Framework/AzCore/AzCore/DOM/DomVisitor.h | 4 +- .../AzCore/AzCore/azcore_files.cmake | 3 - .../AzCore/Tests/DOM/DomJsonBenchmarks.cpp | 26 +- .../AzCore/Tests/DOM/DomJsonTests.cpp | 14 +- 14 files changed, 149 insertions(+), 361 deletions(-) delete mode 100644 Code/Framework/AzCore/AzCore/DOM/DomBackendRegistry.cpp delete mode 100644 Code/Framework/AzCore/AzCore/DOM/DomBackendRegistry.h delete mode 100644 Code/Framework/AzCore/AzCore/DOM/DomBackendRegistryInterface.h diff --git a/Code/Framework/AzCore/AzCore/DOM/Backends/JSON/JsonBackend.cpp b/Code/Framework/AzCore/AzCore/DOM/Backends/JSON/JsonBackend.cpp index 4abeeaf07e..db7b209310 100644 --- a/Code/Framework/AzCore/AzCore/DOM/Backends/JSON/JsonBackend.cpp +++ b/Code/Framework/AzCore/AzCore/DOM/Backends/JSON/JsonBackend.cpp @@ -8,9 +8,7 @@ #include -#include - -namespace AZ::DOM +namespace AZ::Dom { Visitor::Result JsonBackend::ReadFromStringInPlace(AZStd::string& buffer, Visitor* visitor) { @@ -26,12 +24,4 @@ namespace AZ::DOM { return Json::GetJsonStreamWriter(stream, Json::OutputFormatting::PrettyPrintedJson); } - - void JsonBackend::Register() - { - if (auto backendRegistry = BackendRegistry::Get()) - { - backendRegistry->RegisterBackend(kName, {kExtension}); - } - } } diff --git a/Code/Framework/AzCore/AzCore/DOM/Backends/JSON/JsonBackend.h b/Code/Framework/AzCore/AzCore/DOM/Backends/JSON/JsonBackend.h index 5e93ece1de..9de3db573d 100644 --- a/Code/Framework/AzCore/AzCore/DOM/Backends/JSON/JsonBackend.h +++ b/Code/Framework/AzCore/AzCore/DOM/Backends/JSON/JsonBackend.h @@ -11,7 +11,7 @@ #include #include -namespace AZ::DOM +namespace AZ::Dom { class JsonBackend final : public Backend { @@ -19,9 +19,5 @@ namespace AZ::DOM Visitor::Result ReadFromStringInPlace(AZStd::string& buffer, Visitor* visitor) override; Visitor::Result ReadFromString(AZStd::string_view buffer, Lifetime lifetime, Visitor* visitor) override; AZStd::unique_ptr CreateStreamWriter(AZ::IO::GenericStream* stream) override; - - static constexpr const char* kName = "JSON"; - static constexpr const char* kExtension = ".json"; - static void Register(); }; -} // namespace AZ::DOM +} // namespace AZ::Dom diff --git a/Code/Framework/AzCore/AzCore/DOM/Backends/JSON/JsonSerializationUtils.cpp b/Code/Framework/AzCore/AzCore/DOM/Backends/JSON/JsonSerializationUtils.cpp index aa0b24a5ad..4cdf680aff 100644 --- a/Code/Framework/AzCore/AzCore/DOM/Backends/JSON/JsonSerializationUtils.cpp +++ b/Code/Framework/AzCore/AzCore/DOM/Backends/JSON/JsonSerializationUtils.cpp @@ -21,7 +21,7 @@ #include #include -namespace AZ::DOM::Json +namespace AZ::Dom::Json { // // class DocumentWriter @@ -73,7 +73,7 @@ namespace AZ::DOM::Json } else { - CurrentValue().SetString(value.data(), static_cast(value.size())); + CurrentValue().SetString(value.data(), static_cast(value.length())); } return FinishWrite(); } @@ -210,11 +210,11 @@ namespace AZ::DOM::Json { } - bool m_isObject; - rapidjson::Value& m_container; + rapidjson::Value m_key; rapidjson::Value m_value; + rapidjson::Value& m_container; AZ::u64 m_entryCount = 0; - rapidjson::Value m_key; + bool m_isObject; }; rapidjson::Document m_result; @@ -225,13 +225,13 @@ namespace AZ::DOM::Json // class StreamWriter // // Visitor that writes to a rapidjson::Writer - template + template class StreamWriter : public Visitor { public: StreamWriter(AZ::IO::GenericStream* stream) : m_streamWriter(stream) - , m_writer(TWriter(m_streamWriter)) + , m_writer(Writer(m_streamWriter)) { } @@ -305,22 +305,25 @@ namespace AZ::DOM::Json private: Result CheckWrite(bool writeSucceeded) { - if (!writeSucceeded) + if (writeSucceeded) + { + return VisitorSuccess(); + } + else { return VisitorFailure(VisitorErrorCode::InternalError, "Failed to write JSON"); } - return VisitorSuccess(); } AZ::IO::RapidJSONStreamWriter m_streamWriter; - TWriter m_writer; + Writer m_writer; }; // // struct JsonReadHandler // - // Handler for a rapidjson::Reader that translates reads into an AZ::DOM::Visitor - struct JsonReadHandler : public rapidjson::BaseReaderHandler, JsonReadHandler> + // Handler for a rapidjson::Reader that translates reads into an AZ::Dom::Visitor + struct JsonReadHandler { public: JsonReadHandler(Visitor* visitor, Lifetime stringLifetime) @@ -365,13 +368,13 @@ namespace AZ::DOM::Json return CheckResult(m_visitor->Double(d)); } - bool RawNumber([[maybe_unused]] const Ch* str, [[maybe_unused]] rapidjson::SizeType length, [[maybe_unused]] bool copy) + bool RawNumber([[maybe_unused]] const char* str, [[maybe_unused]] rapidjson::SizeType length, [[maybe_unused]] bool copy) { - AZ_Assert(false, "Raw numbers are unsupported"); + AZ_Assert(false, "Raw numbers are unsupported in the rapidjson DOM backend"); return false; } - bool String(const Ch* str, rapidjson::SizeType length, bool copy) + bool String(const char* str, rapidjson::SizeType length, bool copy) { Lifetime lifetime = m_stringLifetime; if (!copy) @@ -386,7 +389,7 @@ namespace AZ::DOM::Json return CheckResult(m_visitor->StartObject()); } - bool Key(const Ch* str, rapidjson::SizeType length, [[maybe_unused]] bool copy) + bool Key(const char* str, rapidjson::SizeType length, [[maybe_unused]] bool copy) { AZStd::string_view key = AZStd::string_view(str, length); if (!m_visitor->SupportsRawKeys()) @@ -552,17 +555,18 @@ namespace AZ::DOM::Json Visitor::Result VisitRapidJsonValue(const rapidjson::Value& value, Visitor* visitor, Lifetime lifetime) { - enum class EndMarker + struct EndArrayMarker + { + }; + struct EndObjectMarker { - EndArray, - EndObject }; // Processing stack consists of values comprised of one of a: // - rapidjson::Value to process - // - EndMarker denoting the end of an array or object + // - EndArrayMarker or EndObjectMarker denoting the end of an array or object // - string denoting a key at the beginning of a key/value pair - using Entry = AZStd::variant; + using Entry = AZStd::variant; AZStd::stack entryStack; AZStd::stack entryCountStack; entryStack.push(&value); @@ -572,97 +576,99 @@ namespace AZ::DOM::Json const auto currentEntry = entryStack.top(); entryStack.pop(); - if (AZStd::holds_alternative(currentEntry)) - { - EndMarker marker = AZStd::get(currentEntry); - if (marker == EndMarker::EndArray) - { - visitor->EndArray(entryCountStack.top()); - } - else - { - visitor->EndObject(entryCountStack.top()); - } - entryCountStack.pop(); - continue; - } - - if (AZStd::holds_alternative(currentEntry)) - { - AZStd::string_view key = AZStd::get(currentEntry); - if (visitor->SupportsRawKeys()) - { - visitor->RawKey(key, lifetime); - } - else - { - visitor->Key(AZ::Name(key)); - } - continue; - } - - const rapidjson::Value& currentValue = *AZStd::get(currentEntry); - if (!entryCountStack.empty()) - { - ++entryCountStack.top(); - } - Visitor::Result result = AZ::Success(); - switch (currentValue.GetType()) - { - case rapidjson::kNullType: - result = visitor->Null(); - break; - case rapidjson::kFalseType: - result = visitor->Bool(false); - break; - case rapidjson::kTrueType: - result = visitor->Bool(true); - break; - case rapidjson::kObjectType: - entryStack.push(EndMarker::EndObject); - entryCountStack.push(0); - result = visitor->StartObject(); - for (auto it = currentValue.MemberEnd(); it != currentValue.MemberBegin(); --it) - { - auto entry = (it - 1); - const AZStd::string_view key(entry->name.GetString(), static_cast(entry->name.GetStringLength())); - entryStack.push(&entry->value); - entryStack.push(key); - } - break; - case rapidjson::kArrayType: - entryStack.push(EndMarker::EndArray); - entryCountStack.push(0); - result = visitor->StartArray(); - for (auto it = currentValue.End(); it != currentValue.Begin(); --it) + AZStd::visit( + [visitor, &entryStack, &entryCountStack, &result, lifetime](auto&& arg) { - auto entry = (it - 1); - entryStack.push(entry); - } - break; - case rapidjson::kStringType: - result = visitor->String( - AZStd::string_view(currentValue.GetString(), static_cast(currentValue.GetStringLength())), lifetime); - break; - case rapidjson::kNumberType: - if (currentValue.IsFloat() || currentValue.IsDouble()) - { - result = visitor->Double(currentValue.GetDouble()); - } - else if (currentValue.IsInt64() || currentValue.IsInt()) - { - result = visitor->Int64(currentValue.GetInt64()); - } - else - { - result = visitor->Uint64(currentValue.GetUint64()); - } - break; - default: - return AZ::Failure(VisitorError(VisitorErrorCode::InvalidData, "Value with invalid type specified")); - } + using Alternative = AZStd::decay_t; + if constexpr (AZStd::is_same_v) + { + const rapidjson::Value& currentValue = *arg; + if (!entryCountStack.empty()) + { + ++entryCountStack.top(); + } + + switch (currentValue.GetType()) + { + case rapidjson::kNullType: + result = visitor->Null(); + break; + case rapidjson::kFalseType: + result = visitor->Bool(false); + break; + case rapidjson::kTrueType: + result = visitor->Bool(true); + break; + case rapidjson::kObjectType: + entryStack.push(EndObjectMarker{}); + entryCountStack.push(0); + result = visitor->StartObject(); + for (auto it = currentValue.MemberEnd(); it != currentValue.MemberBegin(); --it) + { + auto entry = (it - 1); + const AZStd::string_view key(entry->name.GetString(), static_cast(entry->name.GetStringLength())); + entryStack.push(&entry->value); + entryStack.push(key); + } + break; + case rapidjson::kArrayType: + entryStack.push(EndArrayMarker{}); + entryCountStack.push(0); + result = visitor->StartArray(); + for (auto it = currentValue.End(); it != currentValue.Begin(); --it) + { + auto entry = (it - 1); + entryStack.push(entry); + } + break; + case rapidjson::kStringType: + result = visitor->String( + AZStd::string_view(currentValue.GetString(), static_cast(currentValue.GetStringLength())), + lifetime); + break; + case rapidjson::kNumberType: + if (currentValue.IsFloat() || currentValue.IsDouble()) + { + result = visitor->Double(currentValue.GetDouble()); + } + else if (currentValue.IsInt64() || currentValue.IsInt()) + { + result = visitor->Int64(currentValue.GetInt64()); + } + else + { + result = visitor->Uint64(currentValue.GetUint64()); + } + break; + default: + result = AZ::Failure(VisitorError(VisitorErrorCode::InvalidData, "Value with invalid type specified")); + } + } + else if constexpr (AZStd::is_same_v) + { + result = visitor->EndArray(entryCountStack.top()); + entryCountStack.pop(); + } + else if constexpr (AZStd::is_same_v) + { + result = visitor->EndObject(entryCountStack.top()); + entryCountStack.pop(); + } + else if constexpr (AZStd::is_same_v) + { + if (visitor->SupportsRawKeys()) + { + visitor->RawKey(arg, lifetime); + } + else + { + visitor->Key(AZ::Name(arg)); + } + } + }, + currentEntry); if (!result.IsSuccess()) { @@ -672,4 +678,4 @@ namespace AZ::DOM::Json return AZ::Success(); } -} // namespace AZ::DOM::Json +} // namespace AZ::Dom::Json diff --git a/Code/Framework/AzCore/AzCore/DOM/Backends/JSON/JsonSerializationUtils.h b/Code/Framework/AzCore/AzCore/DOM/Backends/JSON/JsonSerializationUtils.h index 323e0b880c..1f9b58ac51 100644 --- a/Code/Framework/AzCore/AzCore/DOM/Backends/JSON/JsonSerializationUtils.h +++ b/Code/Framework/AzCore/AzCore/DOM/Backends/JSON/JsonSerializationUtils.h @@ -16,7 +16,7 @@ #include #include -namespace AZ::DOM::Json +namespace AZ::Dom::Json { //! Specifies how JSON should be formatted when serialized. enum class OutputFormatting @@ -57,4 +57,4 @@ namespace AZ::DOM::Json //! before the visitor is finished using these values, Lifetime::Temporary should be specified. //! \return The aggregate result specifying whether the visitor operations were successful. Visitor::Result VisitRapidJsonValue(const rapidjson::Value& value, Visitor* visitor, Lifetime lifetime); -} // namespace AZ::DOM::Json +} // namespace AZ::Dom::Json diff --git a/Code/Framework/AzCore/AzCore/DOM/DomBackend.cpp b/Code/Framework/AzCore/AzCore/DOM/DomBackend.cpp index f26e34acdf..925897c299 100644 --- a/Code/Framework/AzCore/AzCore/DOM/DomBackend.cpp +++ b/Code/Framework/AzCore/AzCore/DOM/DomBackend.cpp @@ -12,20 +12,8 @@ #include #include "DomBackend.h" -namespace AZ::DOM +namespace AZ::Dom { - Visitor::Result Backend::ReadFromPath(AZ::IO::PathView pathName, Visitor* visitor, size_t maxFileSize) - { - auto readResult = AZ::Utils::ReadFile(pathName.Native(), maxFileSize); - if (!readResult.IsSuccess()) - { - return AZ::Failure(VisitorError(VisitorErrorCode::InternalError, readResult.TakeError())); - } - - AZStd::string fileContents = readResult.TakeValue(); - return ReadFromString(fileContents, Lifetime::Temporary, visitor); - } - Visitor::Result Backend::ReadFromStream(AZ::IO::GenericStream* stream, Visitor* visitor, size_t maxSize) { size_t length = stream->GetLength(); @@ -34,7 +22,7 @@ namespace AZ::DOM return AZ::Failure(VisitorError(VisitorErrorCode::InternalError, "Stream is too large.")); } AZStd::string buffer; - buffer.resize(maxSize); + buffer.resize(length); stream->Read(length, buffer.data()); return ReadFromString(buffer, Lifetime::Temporary, visitor); } @@ -50,24 +38,6 @@ namespace AZ::DOM return callback(writer.get()); } - Visitor::Result Backend::WriteToPath(AZ::IO::PathView pathName, WriteCallback callback) - { - AZStd::string buffer; - auto serializeResult = WriteToString(buffer, callback); - if (!serializeResult.IsSuccess()) - { - return serializeResult; - } - - auto writeResult = AZ::Utils::WriteFile(buffer, pathName.Native()); - if (!writeResult.IsSuccess()) - { - return AZ::Failure(VisitorError(VisitorErrorCode::InternalError, writeResult.TakeError())); - } - - return AZ::Success(); - } - Visitor::Result Backend::WriteToString(AZStd::string& buffer, WriteCallback callback) { AZ::IO::ByteContainerStream stream{&buffer}; diff --git a/Code/Framework/AzCore/AzCore/DOM/DomBackend.h b/Code/Framework/AzCore/AzCore/DOM/DomBackend.h index bb5c9e40fe..d11335262e 100644 --- a/Code/Framework/AzCore/AzCore/DOM/DomBackend.h +++ b/Code/Framework/AzCore/AzCore/DOM/DomBackend.h @@ -14,7 +14,7 @@ #include #include -namespace AZ::DOM +namespace AZ::Dom { //! Backends are registered centrally and used to transition DOM formats to and from a textual format. class Backend @@ -22,9 +22,6 @@ namespace AZ::DOM public: virtual ~Backend() = default; - //! Attempt to read this format from the given file into the target Visitor. - Visitor::Result ReadFromPath( - AZ::IO::PathView pathName, Visitor* visitor, size_t maxFileSize = AZStd::numeric_limits::max()); //! Attempt to read this format from the given stream into the target Visitor. //! The base implementation reads the stream into memory and calls ReadFromString. virtual Visitor::Result ReadFromStream( @@ -46,9 +43,7 @@ namespace AZ::DOM using WriteCallback = AZStd::function; //! Attempt to write a value to a stream using a write callback. Visitor::Result WriteToStream(AZ::IO::GenericStream* stream, WriteCallback callback); - //! Attempt to write a value to a file using a write callback. - Visitor::Result WriteToPath(AZ::IO::PathView pathName, WriteCallback callback); //! Attempt to write a value to a string using a write callback. Visitor::Result WriteToString(AZStd::string& buffer, WriteCallback callback); }; -} // namespace AZ::DOM +} // namespace AZ::Dom diff --git a/Code/Framework/AzCore/AzCore/DOM/DomBackendRegistry.cpp b/Code/Framework/AzCore/AzCore/DOM/DomBackendRegistry.cpp deleted file mode 100644 index c8f4050602..0000000000 --- a/Code/Framework/AzCore/AzCore/DOM/DomBackendRegistry.cpp +++ /dev/null @@ -1,68 +0,0 @@ -/* - * Copyright (c) Contributors to the Open 3D Engine Project. - * For complete copyright and license terms please see the LICENSE at the root of this distribution. - * - * SPDX-License-Identifier: Apache-2.0 OR MIT - * - */ - -#include - -#include - -namespace AZ::DOM -{ - BackendRegistry* BackendRegistry::s_instance = nullptr; - - BackendRegistryInterface* BackendRegistry::Get() - { - return AZ::Interface::Get(); - } - - void BackendRegistry::Create() - { - AZ_Assert(!s_instance, "Attempted to register BackendRegistry when it's already registered"); - s_instance = aznew BackendRegistry; - AZ::Interface::Register(s_instance); - } - - void BackendRegistry::Destroy() - { - AZ_Assert(s_instance, "Attempted to unregister a non-existent BackendRegistry"); - AZ::Interface::Unregister(s_instance); - delete s_instance; - s_instance = nullptr; - } - - AZStd::unique_ptr BackendRegistry::GetBackendByName(AZStd::string_view name) - { - auto backendIterator = m_nameToBackend.find(name); - if (backendIterator != m_nameToBackend.end()) - { - return backendIterator->second(); - } - return nullptr; - } - - AZStd::unique_ptr BackendRegistry::GetBackendForExtension(AZStd::string_view extension) - { - auto extensionIterator = m_extensionToName.find(extension); - if (extensionIterator != m_extensionToName.end()) - { - return GetBackendByName(extensionIterator->second); - } - return nullptr; - } - - void BackendRegistry::RegisterBackendInternal( - BackendRegistryInterface::BackendFactory factory, AZStd::string name, AZStd::vector extensions) - { - AZ_Assert(m_nameToBackend.find(name) == m_nameToBackend.end(), "DOM Backend %s already registered", name.c_str()); - m_nameToBackend.insert({name, factory}); - for (AZStd::string& extension : extensions) - { - AZ_Assert(m_extensionToName.find(extension) == m_extensionToName.end(), "DOM Extensions already registered", extension.c_str()); - m_extensionToName.insert({AZStd::move(extension), name}); - } - } -} diff --git a/Code/Framework/AzCore/AzCore/DOM/DomBackendRegistry.h b/Code/Framework/AzCore/AzCore/DOM/DomBackendRegistry.h deleted file mode 100644 index c71e6798fc..0000000000 --- a/Code/Framework/AzCore/AzCore/DOM/DomBackendRegistry.h +++ /dev/null @@ -1,44 +0,0 @@ -/* - * Copyright (c) Contributors to the Open 3D Engine Project. - * For complete copyright and license terms please see the LICENSE at the root of this distribution. - * - * SPDX-License-Identifier: Apache-2.0 OR MIT - * - */ - -#pragma once - -#include -#include - -namespace AZ::DOM -{ - //! DOM backend registry implementation. - //! \see BackendRegistryInterface - class BackendRegistry final : public BackendRegistryInterface - { - public: - AZ_RTTI(BackendRegistry, "{95533861-6201-4E45-BD03-097A20850C48}", BackendRegistryInterface); - AZ_CLASS_ALLOCATOR(BackendRegistry, AZ::SystemAllocator, 0); - - AZStd::unique_ptr GetBackendByName(AZStd::string_view name) override; - AZStd::unique_ptr GetBackendForExtension(AZStd::string_view extension) override; - - //! Convenience method, gets the current instance of the BackendRegistry via AZ::Interface. - static BackendRegistryInterface* Get(); - //! Creates a singleton BackendRegistry and registers it via AZ::Interface. - static void Create(); - //! Destroys the singleton BackendRegistry created by Create and unregisters it from AZ::Interface. - static void Destroy(); - - protected: - void RegisterBackendInternal(BackendFactory factory, AZStd::string name, AZStd::vector extensions) override; - - private: - using BackendFactory = AZStd::function()>; - AZStd::unordered_map m_nameToBackend; - AZStd::unordered_map m_extensionToName; - - static BackendRegistry* s_instance; - }; -} // namespace AZ::DOM diff --git a/Code/Framework/AzCore/AzCore/DOM/DomBackendRegistryInterface.h b/Code/Framework/AzCore/AzCore/DOM/DomBackendRegistryInterface.h deleted file mode 100644 index 4651f35961..0000000000 --- a/Code/Framework/AzCore/AzCore/DOM/DomBackendRegistryInterface.h +++ /dev/null @@ -1,54 +0,0 @@ -/* - * Copyright (c) Contributors to the Open 3D Engine Project. - * For complete copyright and license terms please see the LICENSE at the root of this distribution. - * - * SPDX-License-Identifier: Apache-2.0 OR MIT - * - */ - -#pragma once - -#include - -namespace AZ::DOM -{ - //! Central interface for registering and looking up backend types by file extension. - class BackendRegistryInterface - { - public: - AZ_RTTI(BackendRegistryInterface, "{B72A88AC-DF15-4F92-9489-ABCDEA3D94E6}") - using BackendFactory = AZStd::function()>; - - virtual ~BackendRegistryInterface() = default; - - //! Registers a factory for a given backend. - //! \param name A unique name to identify this backend. - //! \param extensions A set of file extensions this backend supports. - //! Extensions should by the entire file suffix including any dots (.), for example: - //! ".json" or ".tar.gz" - template - void RegisterBackend(AZStd::string name, AZStd::vector extensions); - - //! Looks up a DOM backend based on its name. - virtual AZStd::unique_ptr GetBackendByName(AZStd::string_view name) = 0; - - //! Looks up a DOM backend based on a file extension. - virtual AZStd::unique_ptr GetBackendForExtension(AZStd::string_view extension) = 0; - - protected: - //! Registers a factory for a given backend, called by RegisterBackend. - //! \param factory The factory function to create new backend instances. - //! \param name The unique name to identify this backend. - //! \param extensions A set of file extensions this backend supports. - //! \see RegisterBackend - virtual void RegisterBackendInternal(BackendFactory factory, AZStd::string name, AZStd::vector extensions) = 0; - }; - - template - void BackendRegistryInterface::RegisterBackend(AZStd::string name, AZStd::vector extensions) - { - RegisterBackendInternal([](){ - return AZStd::make_unique(); - }, AZStd::move(name), AZStd::move(extensions)); - } -} // namespace AZ::DOM diff --git a/Code/Framework/AzCore/AzCore/DOM/DomVisitor.cpp b/Code/Framework/AzCore/AzCore/DOM/DomVisitor.cpp index 5d66bb6ac5..ad314da385 100644 --- a/Code/Framework/AzCore/AzCore/DOM/DomVisitor.cpp +++ b/Code/Framework/AzCore/AzCore/DOM/DomVisitor.cpp @@ -8,7 +8,7 @@ #include -namespace AZ::DOM +namespace AZ::Dom { const char* VisitorError::CodeToString(VisitorErrorCode code) { @@ -236,4 +236,4 @@ namespace AZ::DOM { return (GetVisitorFlags() & VisitorFlags::SupportsOpaqueValues) != VisitorFlags::Null; } -} // namespace AZ::DOM +} // namespace AZ::Dom diff --git a/Code/Framework/AzCore/AzCore/DOM/DomVisitor.h b/Code/Framework/AzCore/AzCore/DOM/DomVisitor.h index eb6f20ae1b..f36ad8d827 100644 --- a/Code/Framework/AzCore/AzCore/DOM/DomVisitor.h +++ b/Code/Framework/AzCore/AzCore/DOM/DomVisitor.h @@ -13,7 +13,7 @@ #include #include -namespace AZ::DOM +namespace AZ::Dom { // // Lifetime enum @@ -243,4 +243,4 @@ namespace AZ::DOM //! Helper method, constructs a success \ref Result. static Result VisitorSuccess(); }; -} // namespace AZ::DOM +} // namespace AZ::Dom diff --git a/Code/Framework/AzCore/AzCore/azcore_files.cmake b/Code/Framework/AzCore/AzCore/azcore_files.cmake index aa5d49dccc..ecc7a41ba1 100644 --- a/Code/Framework/AzCore/AzCore/azcore_files.cmake +++ b/Code/Framework/AzCore/AzCore/azcore_files.cmake @@ -127,9 +127,6 @@ set(FILES Debug/TraceReflection.h DOM/DomBackend.cpp DOM/DomBackend.h - DOM/DomBackendRegistry.cpp - DOM/DomBackendRegistry.h - DOM/DomBackendRegistryInterface.h DOM/DomVisitor.cpp DOM/DomVisitor.h DOM/Backends/JSON/JsonBackend.cpp diff --git a/Code/Framework/AzCore/Tests/DOM/DomJsonBenchmarks.cpp b/Code/Framework/AzCore/Tests/DOM/DomJsonBenchmarks.cpp index 238802a35f..4596316213 100644 --- a/Code/Framework/AzCore/Tests/DOM/DomJsonBenchmarks.cpp +++ b/Code/Framework/AzCore/Tests/DOM/DomJsonBenchmarks.cpp @@ -20,31 +20,31 @@ namespace Benchmark class DomJsonBenchmark : public UnitTest::AllocatorsBenchmarkFixture { public: - void SetUp([[maybe_unused]] const ::benchmark::State& st) override + void SetUp(const ::benchmark::State& st) override { UnitTest::AllocatorsBenchmarkFixture::SetUp(st); AZ::NameDictionary::Create(); } - void SetUp([[maybe_unused]] ::benchmark::State& st) override + void SetUp(::benchmark::State& st) override { UnitTest::AllocatorsBenchmarkFixture::SetUp(st); AZ::NameDictionary::Create(); } - void TearDown([[maybe_unused]] ::benchmark::State& st) override + void TearDown(::benchmark::State& st) override { AZ::NameDictionary::Destroy(); UnitTest::AllocatorsBenchmarkFixture::TearDown(st); } - void TearDown([[maybe_unused]] const ::benchmark::State& st) override + void TearDown(const ::benchmark::State& st) override { AZ::NameDictionary::Destroy(); UnitTest::AllocatorsBenchmarkFixture::TearDown(st); } - AZStd::string GenerateDomJsonBenchmarkPayload(int64_t entryCount = 100, int64_t stringTemplateLength = 5) + AZStd::string GenerateDomJsonBenchmarkPayload(int64_t entryCount, int64_t stringTemplateLength) { rapidjson::Document document; document.SetObject(); @@ -120,7 +120,7 @@ namespace Benchmark BENCHMARK_DEFINE_F(DomJsonBenchmark, DomDeserializeToDocumentInPlace)(benchmark::State& state) { - AZ::DOM::JsonBackend backend; + AZ::Dom::JsonBackend backend; AZStd::string serializedPayload = GenerateDomJsonBenchmarkPayload(state.range(0), state.range(1)); for (auto _ : state) @@ -129,8 +129,8 @@ namespace Benchmark AZStd::string payloadCopy = serializedPayload; state.ResumeTiming(); - auto result = AZ::DOM::Json::WriteToRapidJsonDocument( - [&](AZ::DOM::Visitor* visitor) + auto result = AZ::Dom::Json::WriteToRapidJsonDocument( + [&](AZ::Dom::Visitor* visitor) { return backend.ReadFromStringInPlace(payloadCopy, visitor); }); @@ -144,15 +144,15 @@ namespace Benchmark BENCHMARK_DEFINE_F(DomJsonBenchmark, DomDeserializeToDocument)(benchmark::State& state) { - AZ::DOM::JsonBackend backend; + AZ::Dom::JsonBackend backend; AZStd::string serializedPayload = GenerateDomJsonBenchmarkPayload(state.range(0), state.range(1)); for (auto _ : state) { - auto result = AZ::DOM::Json::WriteToRapidJsonDocument( - [&](AZ::DOM::Visitor* visitor) + auto result = AZ::Dom::Json::WriteToRapidJsonDocument( + [&](AZ::Dom::Visitor* visitor) { - return backend.ReadFromString(serializedPayload, AZ::DOM::Lifetime::Temporary, visitor); + return backend.ReadFromString(serializedPayload, AZ::Dom::Lifetime::Temporary, visitor); }); benchmark::DoNotOptimize(result.GetValue()); @@ -164,7 +164,7 @@ namespace Benchmark BENCHMARK_DEFINE_F(DomJsonBenchmark, JsonUtilsDeserializeToDocument)(benchmark::State& state) { - AZ::DOM::JsonBackend backend; + AZ::Dom::JsonBackend backend; AZStd::string serializedPayload = GenerateDomJsonBenchmarkPayload(state.range(0), state.range(1)); for (auto _ : state) diff --git a/Code/Framework/AzCore/Tests/DOM/DomJsonTests.cpp b/Code/Framework/AzCore/Tests/DOM/DomJsonTests.cpp index e778a449d5..25deeb8a82 100644 --- a/Code/Framework/AzCore/Tests/DOM/DomJsonTests.cpp +++ b/Code/Framework/AzCore/Tests/DOM/DomJsonTests.cpp @@ -12,7 +12,7 @@ #include #include -namespace AZ::DOM::Tests +namespace AZ::Dom::Tests { class DomJsonTests : public UnitTest::AllocatorsFixture { @@ -125,7 +125,7 @@ namespace AZ::DOM::Tests AZStd::string canonicalSerializedDocument; AZ::JsonSerializationUtils::WriteJsonString(*m_document, canonicalSerializedDocument); - auto visitDocumentFn = [this](AZ::DOM::Visitor* visitor) + auto visitDocumentFn = [this](AZ::Dom::Visitor* visitor) { return Json::VisitRapidJsonValue(*m_document, visitor, Lifetime::Persistent); }; @@ -149,10 +149,10 @@ namespace AZ::DOM::Tests // string -> Document { auto result = Json::WriteToRapidJsonDocument( - [&canonicalSerializedDocument](AZ::DOM::Visitor* visitor) + [&canonicalSerializedDocument](AZ::Dom::Visitor* visitor) { JsonBackend backend; - return backend.ReadFromString(canonicalSerializedDocument, AZ::DOM::Lifetime::Temporary, visitor); + return backend.ReadFromString(canonicalSerializedDocument, AZ::Dom::Lifetime::Temporary, visitor); }); EXPECT_TRUE(result.IsSuccess()); EXPECT_TRUE(DeepCompare(*m_document, result.GetValue())); @@ -164,9 +164,9 @@ namespace AZ::DOM::Tests JsonBackend backend; auto result = backend.WriteToString( serializedDocument, - [&backend, &canonicalSerializedDocument](AZ::DOM::Visitor* visitor) + [&backend, &canonicalSerializedDocument](AZ::Dom::Visitor* visitor) { - return backend.ReadFromString(canonicalSerializedDocument, AZ::DOM::Lifetime::Temporary, visitor); + return backend.ReadFromString(canonicalSerializedDocument, AZ::Dom::Lifetime::Temporary, visitor); }); EXPECT_TRUE(result.IsSuccess()); EXPECT_EQ(canonicalSerializedDocument, serializedDocument); @@ -285,4 +285,4 @@ namespace AZ::DOM::Tests m_document->AddMember(CreateString("long_string"), CreateString("abcdefghijklmnopqrstuvwxyz0123456789"), m_document->GetAllocator()); PerformSerializationChecks(); } -} // namespace AZ::DOM::Tests +} // namespace AZ::Dom::Tests From a9c05372d526f35af31ae5906793f71fdcd4ea3e Mon Sep 17 00:00:00 2001 From: Nicholas Van Sickle Date: Tue, 30 Nov 2021 18:00:59 -0800 Subject: [PATCH 04/18] API tweaks - Use ref types - Support using rapidjson::Value in lieu of rapidjson::Document - Use the existing JSON comparison util function in tests Signed-off-by: Nicholas Van Sickle --- .../AzCore/DOM/Backends/JSON/JsonBackend.cpp | 6 +- .../AzCore/DOM/Backends/JSON/JsonBackend.h | 6 +- .../Backends/JSON/JsonSerializationUtils.cpp | 346 +++++++++--------- .../Backends/JSON/JsonSerializationUtils.h | 57 ++- .../AzCore/AzCore/DOM/DomBackend.cpp | 14 +- Code/Framework/AzCore/AzCore/DOM/DomBackend.h | 12 +- .../AzCore/Tests/DOM/DomJsonBenchmarks.cpp | 4 +- .../AzCore/Tests/DOM/DomJsonTests.cpp | 88 +---- 8 files changed, 251 insertions(+), 282 deletions(-) diff --git a/Code/Framework/AzCore/AzCore/DOM/Backends/JSON/JsonBackend.cpp b/Code/Framework/AzCore/AzCore/DOM/Backends/JSON/JsonBackend.cpp index db7b209310..288d3c0699 100644 --- a/Code/Framework/AzCore/AzCore/DOM/Backends/JSON/JsonBackend.cpp +++ b/Code/Framework/AzCore/AzCore/DOM/Backends/JSON/JsonBackend.cpp @@ -10,17 +10,17 @@ namespace AZ::Dom { - Visitor::Result JsonBackend::ReadFromStringInPlace(AZStd::string& buffer, Visitor* visitor) + Visitor::Result JsonBackend::ReadFromStringInPlace(AZStd::string& buffer, Visitor& visitor) { return Json::VisitSerializedJsonInPlace(buffer, visitor); } - Visitor::Result JsonBackend::ReadFromString(AZStd::string_view buffer, Lifetime lifetime, Visitor* visitor) + Visitor::Result JsonBackend::ReadFromString(AZStd::string_view buffer, Lifetime lifetime, Visitor& visitor) { return Json::VisitSerializedJson(buffer, lifetime, visitor); } - AZStd::unique_ptr JsonBackend::CreateStreamWriter(AZ::IO::GenericStream* stream) + AZStd::unique_ptr JsonBackend::CreateStreamWriter(AZ::IO::GenericStream& stream) { return Json::GetJsonStreamWriter(stream, Json::OutputFormatting::PrettyPrintedJson); } diff --git a/Code/Framework/AzCore/AzCore/DOM/Backends/JSON/JsonBackend.h b/Code/Framework/AzCore/AzCore/DOM/Backends/JSON/JsonBackend.h index 9de3db573d..536fec7418 100644 --- a/Code/Framework/AzCore/AzCore/DOM/Backends/JSON/JsonBackend.h +++ b/Code/Framework/AzCore/AzCore/DOM/Backends/JSON/JsonBackend.h @@ -16,8 +16,8 @@ namespace AZ::Dom class JsonBackend final : public Backend { public: - Visitor::Result ReadFromStringInPlace(AZStd::string& buffer, Visitor* visitor) override; - Visitor::Result ReadFromString(AZStd::string_view buffer, Lifetime lifetime, Visitor* visitor) override; - AZStd::unique_ptr CreateStreamWriter(AZ::IO::GenericStream* stream) override; + Visitor::Result ReadFromStringInPlace(AZStd::string& buffer, Visitor& visitor) override; + Visitor::Result ReadFromString(AZStd::string_view buffer, Lifetime lifetime, Visitor& visitor) override; + AZStd::unique_ptr CreateStreamWriter(AZ::IO::GenericStream& stream) override; }; } // namespace AZ::Dom diff --git a/Code/Framework/AzCore/AzCore/DOM/Backends/JSON/JsonSerializationUtils.cpp b/Code/Framework/AzCore/AzCore/DOM/Backends/JSON/JsonSerializationUtils.cpp index 4cdf680aff..743ef3df43 100644 --- a/Code/Framework/AzCore/AzCore/DOM/Backends/JSON/JsonSerializationUtils.cpp +++ b/Code/Framework/AzCore/AzCore/DOM/Backends/JSON/JsonSerializationUtils.cpp @@ -16,7 +16,6 @@ #include #include #include -#include #include #include #include @@ -24,202 +23,185 @@ namespace AZ::Dom::Json { // - // class DocumentWriter + // class RapidJsonValueWriter // - // Visitor that produces a rapidjson::Document - class DocumentWriter final : public Visitor + RapidJsonValueWriter::RapidJsonValueWriter(rapidjson::Value& outputValue, rapidjson::Value::AllocatorType& allocator) + : m_result(outputValue) + , m_allocator(allocator) { - public: - VisitorFlags GetVisitorFlags() const override - { - return VisitorFlags::SupportsRawKeys | VisitorFlags::SupportsArrays | VisitorFlags::SupportsObjects; - } + } - Result Null() override - { - CurrentValue().SetNull(); - return FinishWrite(); - } + VisitorFlags RapidJsonValueWriter::GetVisitorFlags() const + { + return VisitorFlags::SupportsRawKeys | VisitorFlags::SupportsArrays | VisitorFlags::SupportsObjects; + } - Result Bool(bool value) override - { - CurrentValue().SetBool(value); - return FinishWrite(); - } + Visitor::Result RapidJsonValueWriter::Null() + { + CurrentValue().SetNull(); + return FinishWrite(); + } - Result Int64(AZ::s64 value) override - { - CurrentValue().SetInt64(value); - return FinishWrite(); - } + Visitor::Result RapidJsonValueWriter::Bool(bool value) + { + CurrentValue().SetBool(value); + return FinishWrite(); + } - Result Uint64(AZ::u64 value) override - { - CurrentValue().SetUint64(value); - return FinishWrite(); - } + Visitor::Result RapidJsonValueWriter::Int64(AZ::s64 value) + { + CurrentValue().SetInt64(value); + return FinishWrite(); + } - Result Double(double value) override - { - CurrentValue().SetDouble(value); - return FinishWrite(); - } + Visitor::Result RapidJsonValueWriter::Uint64(AZ::u64 value) + { + CurrentValue().SetUint64(value); + return FinishWrite(); + } - Result String(AZStd::string_view value, Lifetime lifetime) override - { - if (lifetime == Lifetime::Temporary) - { - CurrentValue().SetString(value.data(), static_cast(value.length()), m_result.GetAllocator()); - } - else - { - CurrentValue().SetString(value.data(), static_cast(value.length())); - } - return FinishWrite(); - } + Visitor::Result RapidJsonValueWriter::Double(double value) + { + CurrentValue().SetDouble(value); + return FinishWrite(); + } - Result StartObject() override + Visitor::Result RapidJsonValueWriter::String(AZStd::string_view value, Lifetime lifetime) + { + if (lifetime == Lifetime::Temporary) { - CurrentValue().SetObject(); - - const bool isObject = true; - m_entryStack.emplace_front(isObject, CurrentValue()); - return VisitorSuccess(); + CurrentValue().SetString(value.data(), static_cast(value.length()), m_allocator); } - - Result EndObject(AZ::u64 attributeCount) override + else { - if (m_entryStack.empty()) - { - return VisitorFailure(VisitorErrorCode::InternalError, "EndObject called without a matching BeginObject call"); - } + CurrentValue().SetString(value.data(), static_cast(value.length())); + } + return FinishWrite(); + } - if (!m_entryStack.front().m_isObject) - { - return VisitorFailure(VisitorErrorCode::InternalError, "Expected EndArray and received EndObject instead"); - } + Visitor::Result RapidJsonValueWriter::StartObject() + { + CurrentValue().SetObject(); - if (m_entryStack.front().m_entryCount != attributeCount) - { - return FormatVisitorFailure( - VisitorErrorCode::InternalError, "EndObject: Expected %lu attributes but received %lu attributes instead", - attributeCount, m_entryStack.front().m_entryCount); - } + const bool isObject = true; + m_entryStack.emplace_front(isObject, CurrentValue()); + return VisitorSuccess(); + } - m_entryStack.pop_front(); - return FinishWrite(); + Visitor::Result RapidJsonValueWriter::EndObject(AZ::u64 attributeCount) + { + if (m_entryStack.empty()) + { + return VisitorFailure(VisitorErrorCode::InternalError, "EndObject called without a matching BeginObject call"); } - Result Key(AZ::Name key) override + if (!m_entryStack.front().m_isObject) { - return RawKey(key.GetStringView(), Lifetime::Persistent); + return VisitorFailure(VisitorErrorCode::InternalError, "Expected EndArray and received EndObject instead"); } - Result RawKey(AZStd::string_view key, Lifetime lifetime) override + if (m_entryStack.front().m_entryCount != attributeCount) { - AZ_Assert(!m_entryStack.empty(), "Attempmted to push a key with no object"); - AZ_Assert(m_entryStack.front().m_isObject, "Attempted to push a key to an array"); - if (lifetime == Lifetime::Persistent) - { - m_entryStack.front().m_key.SetString(key.data(), static_cast(key.size())); - } - else - { - m_entryStack.front().m_key.SetString(key.data(), static_cast(key.size()), m_result.GetAllocator()); - } - return VisitorSuccess(); + return FormatVisitorFailure( + VisitorErrorCode::InternalError, "EndObject: Expected %lu attributes but received %lu attributes instead", attributeCount, + m_entryStack.front().m_entryCount); } - Result StartArray() override - { - CurrentValue().SetArray(); + m_entryStack.pop_front(); + return FinishWrite(); + } - const bool isObject = false; - m_entryStack.emplace_front(isObject, CurrentValue()); - return VisitorSuccess(); - } + Visitor::Result RapidJsonValueWriter::Key(AZ::Name key) + { + return RawKey(key.GetStringView(), Lifetime::Persistent); + } - Result EndArray(AZ::u64 elementCount) override + Visitor::Result RapidJsonValueWriter::RawKey(AZStd::string_view key, Lifetime lifetime) + { + AZ_Assert(!m_entryStack.empty(), "Attempmted to push a key with no object"); + AZ_Assert(m_entryStack.front().m_isObject, "Attempted to push a key to an array"); + if (lifetime == Lifetime::Persistent) { - if (m_entryStack.empty()) - { - return VisitorFailure(VisitorErrorCode::InternalError, "EndArray called without a matching BeginArray call"); - } + m_entryStack.front().m_key.SetString(key.data(), static_cast(key.size())); + } + else + { + m_entryStack.front().m_key.SetString(key.data(), static_cast(key.size()), m_allocator); + } + return VisitorSuccess(); + } - if (m_entryStack.front().m_isObject) - { - return VisitorFailure(VisitorErrorCode::InternalError, "Expected EndObject and received EndArray instead"); - } + Visitor::Result RapidJsonValueWriter::StartArray() + { + CurrentValue().SetArray(); - if (m_entryStack.front().m_entryCount != elementCount) - { - return FormatVisitorFailure( - VisitorErrorCode::InternalError, "EndArray: Expected %lu elements but received %lu elements instead", elementCount, - m_entryStack.front().m_entryCount); - } + const bool isObject = false; + m_entryStack.emplace_front(isObject, CurrentValue()); + return VisitorSuccess(); + } - m_entryStack.pop_front(); - return FinishWrite(); + Visitor::Result RapidJsonValueWriter::EndArray(AZ::u64 elementCount) + { + if (m_entryStack.empty()) + { + return VisitorFailure(VisitorErrorCode::InternalError, "EndArray called without a matching BeginArray call"); } - rapidjson::Document&& TakeDocument() + if (m_entryStack.front().m_isObject) { - return AZStd::move(m_result); + return VisitorFailure(VisitorErrorCode::InternalError, "Expected EndObject and received EndArray instead"); } - private: - Result FinishWrite() + if (m_entryStack.front().m_entryCount != elementCount) { - if (m_entryStack.empty()) - { - return VisitorSuccess(); - } - - // Retrieve the top value of the stack and replace it with a null value - rapidjson::Value value; - m_entryStack.front().m_value.Swap(value); - ++m_entryStack.front().m_entryCount; + return FormatVisitorFailure( + VisitorErrorCode::InternalError, "EndArray: Expected %lu elements but received %lu elements instead", elementCount, + m_entryStack.front().m_entryCount); + } - if (m_entryStack.front().m_key.IsString()) - { - m_entryStack.front().m_container.AddMember(m_entryStack.front().m_key.Move(), AZStd::move(value), m_result.GetAllocator()); - m_entryStack.front().m_key.SetNull(); - } - else - { - m_entryStack.front().m_container.PushBack(AZStd::move(value), m_result.GetAllocator()); - } + m_entryStack.pop_front(); + return FinishWrite(); + } + Visitor::Result RapidJsonValueWriter::FinishWrite() + { + if (m_entryStack.empty()) + { return VisitorSuccess(); } - rapidjson::Value& CurrentValue() + // Retrieve the top value of the stack and replace it with a null value + rapidjson::Value value; + m_entryStack.front().m_value.Swap(value); + ++m_entryStack.front().m_entryCount; + + if (m_entryStack.front().m_key.IsString()) { - if (m_entryStack.empty()) - { - return m_result; - } - return m_entryStack.front().m_value; + m_entryStack.front().m_container.AddMember(m_entryStack.front().m_key.Move(), AZStd::move(value), m_allocator); + m_entryStack.front().m_key.SetNull(); } - - struct ValueInfo + else { - ValueInfo(bool isObject, rapidjson::Value& container) - : m_isObject(isObject) - , m_container(container) - { - } + m_entryStack.front().m_container.PushBack(AZStd::move(value), m_allocator); + } - rapidjson::Value m_key; - rapidjson::Value m_value; - rapidjson::Value& m_container; - AZ::u64 m_entryCount = 0; - bool m_isObject; - }; + return VisitorSuccess(); + } - rapidjson::Document m_result; - AZStd::deque m_entryStack; - }; + rapidjson::Value& RapidJsonValueWriter::CurrentValue() + { + if (m_entryStack.empty()) + { + return m_result; + } + return m_entryStack.front().m_value; + } + + RapidJsonValueWriter::ValueInfo::ValueInfo(bool isObject, rapidjson::Value& container) + : m_isObject(isObject) + , m_container(container) + { + } // // class StreamWriter @@ -503,36 +485,36 @@ namespace AZ::Dom::Json // // Serialized JSON util functions // - AZStd::unique_ptr GetJsonStreamWriter(AZ::IO::GenericStream* stream, OutputFormatting format) + AZStd::unique_ptr GetJsonStreamWriter(AZ::IO::GenericStream& stream, OutputFormatting format) { if (format == OutputFormatting::MinifiedJson) { using WriterType = rapidjson::Writer; - return AZStd::make_unique>(stream); + return AZStd::make_unique>(&stream); } else { using WriterType = rapidjson::PrettyWriter; - return AZStd::make_unique>(stream); + return AZStd::make_unique>(&stream); } } - Visitor::Result VisitSerializedJson(AZStd::string_view buffer, Lifetime lifetime, Visitor* visitor) + Visitor::Result VisitSerializedJson(AZStd::string_view buffer, Lifetime lifetime, Visitor& visitor) { rapidjson::Reader reader; rapidjson::MemoryStream stream(buffer.data(), buffer.size()); - JsonReadHandler handler(visitor, lifetime); + JsonReadHandler handler(&visitor, lifetime); constexpr int flags = rapidjson::kParseCommentsFlag; reader.Parse(stream, handler); return handler.TakeOutcome(); } - Visitor::Result VisitSerializedJsonInPlace(AZStd::string& buffer, Visitor* visitor) + Visitor::Result VisitSerializedJsonInPlace(AZStd::string& buffer, Visitor& visitor) { rapidjson::Reader reader; AzStringStream stream(buffer); - JsonReadHandler handler(visitor, Lifetime::Persistent); + JsonReadHandler handler(&visitor, Lifetime::Persistent); constexpr int flags = rapidjson::kParseCommentsFlag | rapidjson::kParseInsituFlag; reader.Parse(stream, handler); @@ -544,16 +526,24 @@ namespace AZ::Dom::Json // AZ::Outcome WriteToRapidJsonDocument(Backend::WriteCallback writeCallback) { - DocumentWriter writer; - auto result = writeCallback(&writer); + rapidjson::Document document; + RapidJsonValueWriter writer(document, document.GetAllocator()); + auto result = writeCallback(writer); if (!result.IsSuccess()) { return AZ::Failure(result.TakeError().FormatVisitorErrorMessage()); } - return AZ::Success(writer.TakeDocument()); + return AZ::Success(AZStd::move(document)); + } + + Visitor::Result WriteToRapidJsonValue( + rapidjson::Value& value, rapidjson::Value::AllocatorType& allocator, Backend::WriteCallback writeCallback) + { + RapidJsonValueWriter writer(value, allocator); + return writeCallback(writer); } - Visitor::Result VisitRapidJsonValue(const rapidjson::Value& value, Visitor* visitor, Lifetime lifetime) + Visitor::Result VisitRapidJsonValue(const rapidjson::Value& value, Visitor& visitor, Lifetime lifetime) { struct EndArrayMarker { @@ -579,7 +569,7 @@ namespace AZ::Dom::Json Visitor::Result result = AZ::Success(); AZStd::visit( - [visitor, &entryStack, &entryCountStack, &result, lifetime](auto&& arg) + [&visitor, &entryStack, &entryCountStack, &result, lifetime](auto&& arg) { using Alternative = AZStd::decay_t; if constexpr (AZStd::is_same_v) @@ -593,18 +583,18 @@ namespace AZ::Dom::Json switch (currentValue.GetType()) { case rapidjson::kNullType: - result = visitor->Null(); + result = visitor.Null(); break; case rapidjson::kFalseType: - result = visitor->Bool(false); + result = visitor.Bool(false); break; case rapidjson::kTrueType: - result = visitor->Bool(true); + result = visitor.Bool(true); break; case rapidjson::kObjectType: entryStack.push(EndObjectMarker{}); entryCountStack.push(0); - result = visitor->StartObject(); + result = visitor.StartObject(); for (auto it = currentValue.MemberEnd(); it != currentValue.MemberBegin(); --it) { auto entry = (it - 1); @@ -616,7 +606,7 @@ namespace AZ::Dom::Json case rapidjson::kArrayType: entryStack.push(EndArrayMarker{}); entryCountStack.push(0); - result = visitor->StartArray(); + result = visitor.StartArray(); for (auto it = currentValue.End(); it != currentValue.Begin(); --it) { auto entry = (it - 1); @@ -624,22 +614,22 @@ namespace AZ::Dom::Json } break; case rapidjson::kStringType: - result = visitor->String( + result = visitor.String( AZStd::string_view(currentValue.GetString(), static_cast(currentValue.GetStringLength())), lifetime); break; case rapidjson::kNumberType: if (currentValue.IsFloat() || currentValue.IsDouble()) { - result = visitor->Double(currentValue.GetDouble()); + result = visitor.Double(currentValue.GetDouble()); } else if (currentValue.IsInt64() || currentValue.IsInt()) { - result = visitor->Int64(currentValue.GetInt64()); + result = visitor.Int64(currentValue.GetInt64()); } else { - result = visitor->Uint64(currentValue.GetUint64()); + result = visitor.Uint64(currentValue.GetUint64()); } break; default: @@ -648,23 +638,23 @@ namespace AZ::Dom::Json } else if constexpr (AZStd::is_same_v) { - result = visitor->EndArray(entryCountStack.top()); + result = visitor.EndArray(entryCountStack.top()); entryCountStack.pop(); } else if constexpr (AZStd::is_same_v) { - result = visitor->EndObject(entryCountStack.top()); + result = visitor.EndObject(entryCountStack.top()); entryCountStack.pop(); } else if constexpr (AZStd::is_same_v) { - if (visitor->SupportsRawKeys()) + if (visitor.SupportsRawKeys()) { - visitor->RawKey(arg, lifetime); + visitor.RawKey(arg, lifetime); } else { - visitor->Key(AZ::Name(arg)); + visitor.Key(AZ::Name(arg)); } } }, diff --git a/Code/Framework/AzCore/AzCore/DOM/Backends/JSON/JsonSerializationUtils.h b/Code/Framework/AzCore/AzCore/DOM/Backends/JSON/JsonSerializationUtils.h index 1f9b58ac51..007134227e 100644 --- a/Code/Framework/AzCore/AzCore/DOM/Backends/JSON/JsonSerializationUtils.h +++ b/Code/Framework/AzCore/AzCore/DOM/Backends/JSON/JsonSerializationUtils.h @@ -13,6 +13,7 @@ #include #include #include +#include #include #include @@ -25,36 +26,84 @@ namespace AZ::Dom::Json PrettyPrintedJson, //!< Formats JSON in a pretty printed form, focusing on legibility to readers. }; + //! Visitor that feeds into a rapidjson::Value + class RapidJsonValueWriter final : public Visitor + { + public: + RapidJsonValueWriter(rapidjson::Value& outputValue, rapidjson::Value::AllocatorType& allocator); + + VisitorFlags GetVisitorFlags() const override; + Result Null() override; + Result Bool(bool value) override; + Result Int64(AZ::s64 value) override; + Result Uint64(AZ::u64 value) override; + Result Double(double value) override; + + Result String(AZStd::string_view value, Lifetime lifetime) override; + Result StartObject() override; + Result EndObject(AZ::u64 attributeCount) override; + Result Key(AZ::Name key) override; + Result RawKey(AZStd::string_view key, Lifetime lifetime) override; + Result StartArray() override; + Result EndArray(AZ::u64 elementCount) override; + + private: + Result FinishWrite(); + rapidjson::Value& CurrentValue(); + + struct ValueInfo + { + ValueInfo(bool isObject, rapidjson::Value& container); + + rapidjson::Value m_key; + rapidjson::Value m_value; + rapidjson::Value& m_container; + AZ::u64 m_entryCount = 0; + bool m_isObject; + }; + + rapidjson::Value& m_result; + rapidjson::Value::AllocatorType& m_allocator; + AZStd::deque m_entryStack; + }; + //! Creates a Visitor that will write serialized JSON to the specified stream. //! \param stream The stream the visitor will write to. //! \param format The format to write in. //! \return A Visitor that will write to stream when visited. AZStd::unique_ptr GetJsonStreamWriter( - AZ::IO::GenericStream* stream, OutputFormatting format = OutputFormatting::PrettyPrintedJson); + AZ::IO::GenericStream& stream, OutputFormatting format = OutputFormatting::PrettyPrintedJson); //! Reads serialized JSON from a string and applies it to a visitor. //! \param buffer The UTF-8 serialized JSON to read. //! \param lifetime Specifies the lifetime of the specified buffer. If the string specified by buffer might be deallocated, //! ensure specify Lifetime::Temporary is specified. //! \param visitor The visitor to visit with the JSON buffer's contents. //! \return The aggregate result specifying whether the visitor operations were successful. - Visitor::Result VisitSerializedJson(AZStd::string_view buffer, Lifetime lifetime, Visitor* visitor); + Visitor::Result VisitSerializedJson(AZStd::string_view buffer, Lifetime lifetime, Visitor& visitor); //! Reads serialized JSON from a string in-place and applies it to a visitor. //! \param buffer The UTF-8 serialized JSON to read. This buffer will be modified as part of the deserialization process to //! apply null terminators. //! \param visitor The visitor to visit with the JSON buffer's contents. The strings provided to the visitor will only //! be valid for the lifetime of buffer. //! \return The aggregate result specifying whether the visitor operations were successful. - Visitor::Result VisitSerializedJsonInPlace(AZStd::string& buffer, Visitor* visitor); + Visitor::Result VisitSerializedJsonInPlace(AZStd::string& buffer, Visitor& visitor); //! Takes a visitor specified by a callback and produces a rapidjson::Document. //! \param writeCallback A callback specifying a visitor to accept to build the resulting document. //! \return An outcome with either the rapidjson::Document or an error message. AZ::Outcome WriteToRapidJsonDocument(Backend::WriteCallback writeCallback); + //! Takes a visitor specified by a callback and reads them into a rapidjson::Value. + //! \param value The value to read into, its contents will be overridden. + //! \param allocator The allocator to use when performing rapidjson allocations (generally provded by the rapidjson::Document). + //! \param writeCallback A callback specifying a visitor to accept to build the resulting document. + //! \return An outcome with either the rapidjson::Document or an error message. + Visitor::Result WriteToRapidJsonValue( + rapidjson::Value& value, rapidjson::Value::AllocatorType& allocator, Backend::WriteCallback writeCallback); //! Accepts a visitor with the contents of a rapidjson::Value. //! \param value The rapidjson::Value to apply to visitor. //! \param visitor The visitor to receive the contents of value. //! \param lifetime The lifetime to specify for visiting strings. If the rapidjson::Value might be destroyed or changed //! before the visitor is finished using these values, Lifetime::Temporary should be specified. //! \return The aggregate result specifying whether the visitor operations were successful. - Visitor::Result VisitRapidJsonValue(const rapidjson::Value& value, Visitor* visitor, Lifetime lifetime); + Visitor::Result VisitRapidJsonValue(const rapidjson::Value& value, Visitor& visitor, Lifetime lifetime); } // namespace AZ::Dom::Json diff --git a/Code/Framework/AzCore/AzCore/DOM/DomBackend.cpp b/Code/Framework/AzCore/AzCore/DOM/DomBackend.cpp index 925897c299..258975eff3 100644 --- a/Code/Framework/AzCore/AzCore/DOM/DomBackend.cpp +++ b/Code/Framework/AzCore/AzCore/DOM/DomBackend.cpp @@ -10,11 +10,11 @@ #include #include -#include "DomBackend.h" +#include namespace AZ::Dom { - Visitor::Result Backend::ReadFromStream(AZ::IO::GenericStream* stream, Visitor* visitor, size_t maxSize) + Visitor::Result Backend::ReadFromStream(AZ::IO::GenericStream* stream, Visitor& visitor, size_t maxSize) { size_t length = stream->GetLength(); if (length > maxSize) @@ -27,21 +27,21 @@ namespace AZ::Dom return ReadFromString(buffer, Lifetime::Temporary, visitor); } - Visitor::Result Backend::ReadFromStringInPlace(AZStd::string& buffer, Visitor* visitor) + Visitor::Result Backend::ReadFromStringInPlace(AZStd::string& buffer, Visitor& visitor) { return ReadFromString(buffer, Lifetime::Persistent, visitor); } - Visitor::Result Backend::WriteToStream(AZ::IO::GenericStream* stream, WriteCallback callback) + Visitor::Result Backend::WriteToStream(AZ::IO::GenericStream& stream, WriteCallback callback) { AZStd::unique_ptr writer = CreateStreamWriter(stream); - return callback(writer.get()); + return callback(*writer.get()); } Visitor::Result Backend::WriteToString(AZStd::string& buffer, WriteCallback callback) { AZ::IO::ByteContainerStream stream{&buffer}; - AZStd::unique_ptr writer = CreateStreamWriter(&stream); - return callback(writer.get()); + AZStd::unique_ptr writer = CreateStreamWriter(stream); + return callback(*writer.get()); } } diff --git a/Code/Framework/AzCore/AzCore/DOM/DomBackend.h b/Code/Framework/AzCore/AzCore/DOM/DomBackend.h index d11335262e..d1eee5d676 100644 --- a/Code/Framework/AzCore/AzCore/DOM/DomBackend.h +++ b/Code/Framework/AzCore/AzCore/DOM/DomBackend.h @@ -25,24 +25,24 @@ namespace AZ::Dom //! Attempt to read this format from the given stream into the target Visitor. //! The base implementation reads the stream into memory and calls ReadFromString. virtual Visitor::Result ReadFromStream( - AZ::IO::GenericStream* stream, Visitor* visitor, size_t maxSize = AZStd::numeric_limits::max()); + AZ::IO::GenericStream* stream, Visitor& visitor, size_t maxSize = AZStd::numeric_limits::max()); //! Attempt to read this format from a mutable string into the target Visitor. This enables some backends to //! parse without making additional string allocations. //! This string may be modified and read in place without being copied, so when calling this please ensure that: //! - The string won't be deallocated until the visitor no longer needs the values and //! - The string is safe to modify in place. //! The base implementation simply calls ReadFromString. - virtual Visitor::Result ReadFromStringInPlace(AZStd::string& buffer, Visitor* visitor); + virtual Visitor::Result ReadFromStringInPlace(AZStd::string& buffer, Visitor& visitor); //! Attempt to read this format from an immutable buffer in memory into the target Visitor. - virtual Visitor::Result ReadFromString(AZStd::string_view buffer, Lifetime lifetime, Visitor* visitor) = 0; + virtual Visitor::Result ReadFromString(AZStd::string_view buffer, Lifetime lifetime, Visitor& visitor) = 0; //! Acquire a visitor interface for writing to the target output file. - virtual AZStd::unique_ptr CreateStreamWriter(AZ::IO::GenericStream* stream) = 0; + virtual AZStd::unique_ptr CreateStreamWriter(AZ::IO::GenericStream& stream) = 0; //! A callback that accepts a Visitor, making DOM calls to inform the serializer, and returns an //! aggregate error code to indicate whether or not the operation succeeded. - using WriteCallback = AZStd::function; + using WriteCallback = AZStd::function; //! Attempt to write a value to a stream using a write callback. - Visitor::Result WriteToStream(AZ::IO::GenericStream* stream, WriteCallback callback); + Visitor::Result WriteToStream(AZ::IO::GenericStream& stream, WriteCallback callback); //! Attempt to write a value to a string using a write callback. Visitor::Result WriteToString(AZStd::string& buffer, WriteCallback callback); }; diff --git a/Code/Framework/AzCore/Tests/DOM/DomJsonBenchmarks.cpp b/Code/Framework/AzCore/Tests/DOM/DomJsonBenchmarks.cpp index 4596316213..ea72092001 100644 --- a/Code/Framework/AzCore/Tests/DOM/DomJsonBenchmarks.cpp +++ b/Code/Framework/AzCore/Tests/DOM/DomJsonBenchmarks.cpp @@ -130,7 +130,7 @@ namespace Benchmark state.ResumeTiming(); auto result = AZ::Dom::Json::WriteToRapidJsonDocument( - [&](AZ::Dom::Visitor* visitor) + [&](AZ::Dom::Visitor& visitor) { return backend.ReadFromStringInPlace(payloadCopy, visitor); }); @@ -150,7 +150,7 @@ namespace Benchmark for (auto _ : state) { auto result = AZ::Dom::Json::WriteToRapidJsonDocument( - [&](AZ::Dom::Visitor* visitor) + [&](AZ::Dom::Visitor& visitor) { return backend.ReadFromString(serializedPayload, AZ::Dom::Lifetime::Temporary, visitor); }); diff --git a/Code/Framework/AzCore/Tests/DOM/DomJsonTests.cpp b/Code/Framework/AzCore/Tests/DOM/DomJsonTests.cpp index 25deeb8a82..cde5e2eb3f 100644 --- a/Code/Framework/AzCore/Tests/DOM/DomJsonTests.cpp +++ b/Code/Framework/AzCore/Tests/DOM/DomJsonTests.cpp @@ -9,6 +9,7 @@ #include #include #include +#include #include #include @@ -31,78 +32,6 @@ namespace AZ::Dom::Tests UnitTest::AllocatorsFixture::TearDown(); } - static bool DeepCompare(const rapidjson::Value& lhs, const rapidjson::Value& rhs) - { - if (lhs.GetType() != rhs.GetType()) - { - return false; - } - - switch (lhs.GetType()) - { - case rapidjson::kNullType: - return true; - case rapidjson::kFalseType: - return true; - case rapidjson::kTrueType: - return true; - case rapidjson::kObjectType: - { - if (lhs.MemberCount() != rhs.MemberCount()) - { - return false; - } - - auto lhsIt = lhs.MemberBegin(); - auto rhsIt = rhs.MemberBegin(); - while (lhsIt != lhs.MemberEnd()) - { - if (lhsIt->name != rhsIt->name) - { - return false; - } - - if (!DeepCompare(lhsIt->value, rhsIt->value)) - { - return false; - } - - ++lhsIt; - ++rhsIt; - } - return true; - } - case rapidjson::kArrayType: - { - if (lhs.Size() != rhs.Size()) - { - return false; - } - - auto lhsIt = lhs.Begin(); - auto rhsIt = rhs.Begin(); - while (lhsIt != lhs.End()) - { - if (!DeepCompare(*lhsIt, *rhsIt)) - { - return false; - } - - ++lhsIt; - ++rhsIt; - } - return true; - } - case rapidjson::kStringType: - return lhs == rhs; - case rapidjson::kNumberType: - return lhs == rhs; - } - - AZ_Assert(false, "Unexpected JSON value type"); - return false; - } - rapidjson::Value CreateString(const AZStd::string& text) { rapidjson::Value key; @@ -110,7 +39,7 @@ namespace AZ::Dom::Tests return key; } - template + template void AddValue(const AZStd::string& key, T value) { m_document->AddMember(CreateString(key), rapidjson::Value(value), m_document->GetAllocator()); @@ -125,7 +54,7 @@ namespace AZ::Dom::Tests AZStd::string canonicalSerializedDocument; AZ::JsonSerializationUtils::WriteJsonString(*m_document, canonicalSerializedDocument); - auto visitDocumentFn = [this](AZ::Dom::Visitor* visitor) + auto visitDocumentFn = [this](AZ::Dom::Visitor& visitor) { return Json::VisitRapidJsonValue(*m_document, visitor, Lifetime::Persistent); }; @@ -134,7 +63,7 @@ namespace AZ::Dom::Tests { auto result = Json::WriteToRapidJsonDocument(visitDocumentFn); EXPECT_TRUE(result.IsSuccess()); - EXPECT_TRUE(DeepCompare(*m_document, result.GetValue())); + EXPECT_EQ(AZ::JsonSerialization::Compare(*m_document, result.GetValue()), AZ::JsonSerializerCompareResult::Equal); } // Document -> string @@ -149,13 +78,13 @@ namespace AZ::Dom::Tests // string -> Document { auto result = Json::WriteToRapidJsonDocument( - [&canonicalSerializedDocument](AZ::Dom::Visitor* visitor) + [&canonicalSerializedDocument](AZ::Dom::Visitor& visitor) { JsonBackend backend; return backend.ReadFromString(canonicalSerializedDocument, AZ::Dom::Lifetime::Temporary, visitor); }); EXPECT_TRUE(result.IsSuccess()); - EXPECT_TRUE(DeepCompare(*m_document, result.GetValue())); + EXPECT_EQ(AZ::JsonSerialization::Compare(*m_document, result.GetValue()), JsonSerializerCompareResult::Equal); } // string -> string @@ -164,7 +93,7 @@ namespace AZ::Dom::Tests JsonBackend backend; auto result = backend.WriteToString( serializedDocument, - [&backend, &canonicalSerializedDocument](AZ::Dom::Visitor* visitor) + [&backend, &canonicalSerializedDocument](AZ::Dom::Visitor& visitor) { return backend.ReadFromString(canonicalSerializedDocument, AZ::Dom::Lifetime::Temporary, visitor); }); @@ -282,7 +211,8 @@ namespace AZ::Dom::Tests m_document->SetObject(); m_document->AddMember(CreateString("empty_string"), CreateString(""), m_document->GetAllocator()); m_document->AddMember(CreateString("short_string"), CreateString("test"), m_document->GetAllocator()); - m_document->AddMember(CreateString("long_string"), CreateString("abcdefghijklmnopqrstuvwxyz0123456789"), m_document->GetAllocator()); + m_document->AddMember( + CreateString("long_string"), CreateString("abcdefghijklmnopqrstuvwxyz0123456789"), m_document->GetAllocator()); PerformSerializationChecks(); } } // namespace AZ::Dom::Tests From f87021d8561f17dd05d9e2fd0239fcf7657ebbd8 Mon Sep 17 00:00:00 2001 From: Nicholas Van Sickle Date: Tue, 30 Nov 2021 22:08:48 -0800 Subject: [PATCH 05/18] Expose JSON parse flag options Signed-off-by: Nicholas Van Sickle --- .../Backends/JSON/JsonSerializationUtils.cpp | 282 ++++++++---------- .../Backends/JSON/JsonSerializationUtils.h | 82 ++++- 2 files changed, 202 insertions(+), 162 deletions(-) diff --git a/Code/Framework/AzCore/AzCore/DOM/Backends/JSON/JsonSerializationUtils.cpp b/Code/Framework/AzCore/AzCore/DOM/Backends/JSON/JsonSerializationUtils.cpp index 743ef3df43..84f663f97c 100644 --- a/Code/Framework/AzCore/AzCore/DOM/Backends/JSON/JsonSerializationUtils.cpp +++ b/Code/Framework/AzCore/AzCore/DOM/Backends/JSON/JsonSerializationUtils.cpp @@ -304,183 +304,165 @@ namespace AZ::Dom::Json // // struct JsonReadHandler // - // Handler for a rapidjson::Reader that translates reads into an AZ::Dom::Visitor - struct JsonReadHandler + RapidJsonReadHandler::RapidJsonReadHandler(Visitor* visitor, Lifetime stringLifetime) + : m_visitor(visitor) + , m_stringLifetime(stringLifetime) + , m_outcome(AZ::Success()) { - public: - JsonReadHandler(Visitor* visitor, Lifetime stringLifetime) - : m_visitor(visitor) - , m_stringLifetime(stringLifetime) - , m_outcome(AZ::Success()) - { - } + } - bool Null() - { - return CheckResult(m_visitor->Null()); - } + bool RapidJsonReadHandler::Null() + { + return CheckResult(m_visitor->Null()); + } - bool Bool(bool b) - { - return CheckResult(m_visitor->Bool(b)); - } + bool RapidJsonReadHandler::Bool(bool b) + { + return CheckResult(m_visitor->Bool(b)); + } - bool Int(int i) - { - return CheckResult(m_visitor->Int64(static_cast(i))); - } + bool RapidJsonReadHandler::Int(int i) + { + return CheckResult(m_visitor->Int64(static_cast(i))); + } - bool Uint(unsigned i) - { - return CheckResult(m_visitor->Uint64(static_cast(i))); - } + bool RapidJsonReadHandler::Uint(unsigned i) + { + return CheckResult(m_visitor->Uint64(static_cast(i))); + } - bool Int64(int64_t i) - { - return CheckResult(m_visitor->Int64(i)); - } + bool RapidJsonReadHandler::Int64(int64_t i) + { + return CheckResult(m_visitor->Int64(i)); + } - bool Uint64(uint64_t i) - { - return CheckResult(m_visitor->Uint64(i)); - } + bool RapidJsonReadHandler::Uint64(uint64_t i) + { + return CheckResult(m_visitor->Uint64(i)); + } - bool Double(double d) - { - return CheckResult(m_visitor->Double(d)); - } + bool RapidJsonReadHandler::Double(double d) + { + return CheckResult(m_visitor->Double(d)); + } - bool RawNumber([[maybe_unused]] const char* str, [[maybe_unused]] rapidjson::SizeType length, [[maybe_unused]] bool copy) - { - AZ_Assert(false, "Raw numbers are unsupported in the rapidjson DOM backend"); - return false; - } + bool RapidJsonReadHandler::RawNumber( + [[maybe_unused]] const char* str, [[maybe_unused]] rapidjson::SizeType length, [[maybe_unused]] bool copy) + { + AZ_Assert(false, "Raw numbers are unsupported in the rapidjson DOM backend"); + return false; + } - bool String(const char* str, rapidjson::SizeType length, bool copy) + bool RapidJsonReadHandler::String(const char* str, rapidjson::SizeType length, bool copy) + { + Lifetime lifetime = m_stringLifetime; + if (!copy) { - Lifetime lifetime = m_stringLifetime; - if (!copy) - { - lifetime = Lifetime::Temporary; - } - return CheckResult(m_visitor->String(AZStd::string_view(str, length), lifetime)); + lifetime = Lifetime::Temporary; } + return CheckResult(m_visitor->String(AZStd::string_view(str, length), lifetime)); + } - bool StartObject() - { - return CheckResult(m_visitor->StartObject()); - } + bool RapidJsonReadHandler::StartObject() + { + return CheckResult(m_visitor->StartObject()); + } - bool Key(const char* str, rapidjson::SizeType length, [[maybe_unused]] bool copy) + bool RapidJsonReadHandler::Key(const char* str, rapidjson::SizeType length, [[maybe_unused]] bool copy) + { + AZStd::string_view key = AZStd::string_view(str, length); + if (!m_visitor->SupportsRawKeys()) { - AZStd::string_view key = AZStd::string_view(str, length); - if (!m_visitor->SupportsRawKeys()) - { - m_visitor->Key(AZ::Name(key)); - } - Lifetime lifetime = m_stringLifetime; - if (!copy) - { - lifetime = Lifetime::Temporary; - } - return CheckResult(m_visitor->RawKey(key, lifetime)); + m_visitor->Key(AZ::Name(key)); } - - bool EndObject([[maybe_unused]] rapidjson::SizeType memberCount) + Lifetime lifetime = m_stringLifetime; + if (!copy) { - return CheckResult(m_visitor->EndObject(memberCount)); + lifetime = Lifetime::Temporary; } + return CheckResult(m_visitor->RawKey(key, lifetime)); + } - bool StartArray() - { - return CheckResult(m_visitor->StartArray()); - } + bool RapidJsonReadHandler::EndObject([[maybe_unused]] rapidjson::SizeType memberCount) + { + return CheckResult(m_visitor->EndObject(memberCount)); + } - bool EndArray([[maybe_unused]] rapidjson::SizeType elementCount) - { - return CheckResult(m_visitor->EndArray(elementCount)); - } + bool RapidJsonReadHandler::StartArray() + { + return CheckResult(m_visitor->StartArray()); + } - Visitor::Result&& TakeOutcome() - { - return AZStd::move(m_outcome); - } + bool RapidJsonReadHandler::EndArray([[maybe_unused]] rapidjson::SizeType elementCount) + { + return CheckResult(m_visitor->EndArray(elementCount)); + } - private: - bool CheckResult(Visitor::Result result) + Visitor::Result&& RapidJsonReadHandler::TakeOutcome() + { + return AZStd::move(m_outcome); + } + + bool RapidJsonReadHandler::CheckResult(Visitor::Result result) + { + if (!result.IsSuccess()) { - if (!result.IsSuccess()) - { - m_outcome = AZStd::move(result); - return false; - } - return true; + m_outcome = AZStd::move(result); + return false; } - - Visitor::Result m_outcome; - Visitor* m_visitor; - Lifetime m_stringLifetime; - }; + return true; + } // // struct AzStringStream // // rapidjson stream wrapper for AZStd::string suitable for in-situ parsing - struct AzStringStream + AzStringStream::AzStringStream(AZStd::string& buffer) { - using Ch = char; - - AzStringStream(AZStd::string& buffer) - { - m_cursor = buffer.data(); - m_begin = m_cursor; - } - - char Peek() const - { - return *m_cursor; - } + m_cursor = buffer.data(); + m_begin = m_cursor; + } - char Take() - { - return *m_cursor++; - } + char AzStringStream::Peek() const + { + return *m_cursor; + } - size_t Tell() const - { - return static_cast(m_cursor - m_begin); - } + char AzStringStream::Take() + { + return *m_cursor++; + } - char* PutBegin() - { - m_write = m_cursor; - return m_cursor; - } + size_t AzStringStream::Tell() const + { + return static_cast(m_cursor - m_begin); + } - void Put(char c) - { - (*m_write++) = c; - } + char* AzStringStream::PutBegin() + { + m_write = m_cursor; + return m_cursor; + } - void Flush() - { - } + void AzStringStream::Put(char c) + { + (*m_write++) = c; + } - size_t PutEnd(char* begin) - { - return m_write - begin; - } + void AzStringStream::Flush() + { + } - const char* Peek4() const - { - AZ_Assert(false, "Not implemented, encoding is hard-coded to UTF-8"); - return m_cursor; - } + size_t AzStringStream::PutEnd(char* begin) + { + return m_write - begin; + } - char* m_cursor; //!< Current read position. - char* m_write; //!< Current write position. - const char* m_begin; //!< Head of string. - }; + const char* AzStringStream::Peek4() const + { + AZ_Assert(false, "Not implemented, encoding is hard-coded to UTF-8"); + return m_cursor; + } // // Serialized JSON util functions @@ -499,28 +481,6 @@ namespace AZ::Dom::Json } } - Visitor::Result VisitSerializedJson(AZStd::string_view buffer, Lifetime lifetime, Visitor& visitor) - { - rapidjson::Reader reader; - rapidjson::MemoryStream stream(buffer.data(), buffer.size()); - JsonReadHandler handler(&visitor, lifetime); - - constexpr int flags = rapidjson::kParseCommentsFlag; - reader.Parse(stream, handler); - return handler.TakeOutcome(); - } - - Visitor::Result VisitSerializedJsonInPlace(AZStd::string& buffer, Visitor& visitor) - { - rapidjson::Reader reader; - AzStringStream stream(buffer); - JsonReadHandler handler(&visitor, Lifetime::Persistent); - - constexpr int flags = rapidjson::kParseCommentsFlag | rapidjson::kParseInsituFlag; - reader.Parse(stream, handler); - return handler.TakeOutcome(); - } - // // In-memory rapidjson util functions // diff --git a/Code/Framework/AzCore/AzCore/DOM/Backends/JSON/JsonSerializationUtils.h b/Code/Framework/AzCore/AzCore/DOM/Backends/JSON/JsonSerializationUtils.h index 007134227e..be055d7f5c 100644 --- a/Code/Framework/AzCore/AzCore/DOM/Backends/JSON/JsonSerializationUtils.h +++ b/Code/Framework/AzCore/AzCore/DOM/Backends/JSON/JsonSerializationUtils.h @@ -67,6 +67,56 @@ namespace AZ::Dom::Json AZStd::deque m_entryStack; }; + //! Handler for a rapidjson::Reader that translates reads into an AZ::Dom::Visitor + struct RapidJsonReadHandler + { + public: + RapidJsonReadHandler(Visitor* visitor, Lifetime stringLifetime); + + bool Null(); + bool Bool(bool b); + bool Int(int i); + bool Uint(unsigned i); + bool Int64(int64_t i); + bool Uint64(uint64_t i); + bool Double(double d); + bool RawNumber(const char* str, rapidjson::SizeType length, bool copy); + bool String(const char* str, rapidjson::SizeType length, bool copy); + bool StartObject(); + bool Key(const char* str, rapidjson::SizeType length, bool copy); + bool EndObject(rapidjson::SizeType memberCount); + bool StartArray(); + bool EndArray(rapidjson::SizeType elementCount); + Visitor::Result&& TakeOutcome(); + + private: + bool CheckResult(Visitor::Result result); + + Visitor::Result m_outcome; + Visitor* m_visitor; + Lifetime m_stringLifetime; + }; + + //! rapidjson stream wrapper for AZStd::string suitable for in-situ parsing + struct AzStringStream + { + using Ch = char; // Visitor::Result VisitSerializedJson(AZStd::string_view buffer, Lifetime lifetime, Visitor& visitor); //! Reads serialized JSON from a string in-place and applies it to a visitor. //! \param buffer The UTF-8 serialized JSON to read. This buffer will be modified as part of the deserialization process to @@ -86,6 +137,7 @@ namespace AZ::Dom::Json //! \param visitor The visitor to visit with the JSON buffer's contents. The strings provided to the visitor will only //! be valid for the lifetime of buffer. //! \return The aggregate result specifying whether the visitor operations were successful. + template Visitor::Result VisitSerializedJsonInPlace(AZStd::string& buffer, Visitor& visitor); //! Takes a visitor specified by a callback and produces a rapidjson::Document. @@ -106,4 +158,32 @@ namespace AZ::Dom::Json //! before the visitor is finished using these values, Lifetime::Temporary should be specified. //! \return The aggregate result specifying whether the visitor operations were successful. Visitor::Result VisitRapidJsonValue(const rapidjson::Value& value, Visitor& visitor, Lifetime lifetime); + + template + Visitor::Result VisitSerializedJson(AZStd::string_view buffer, Lifetime lifetime, Visitor& visitor) + { + static_assert( + (ParseFlags & rapidjson::kParseInsituFlag) == 0, + "VisitSerializedJsonInPlace requires kParseInSituFlag not to be set, use VisitSerializedJsonInPlace to parse in-situ"); + + rapidjson::Reader reader; + rapidjson::MemoryStream stream(buffer.data(), buffer.size()); + RapidJsonReadHandler handler(&visitor, lifetime); + + reader.Parse(stream, handler); + return handler.TakeOutcome(); + } + + template + Visitor::Result VisitSerializedJsonInPlace(AZStd::string& buffer, Visitor& visitor) + { + static_assert((ParseFlags & rapidjson::kParseInsituFlag) != 0, "VisitSerializedJsonInPlace requires kParseInSituFlag to be set"); + + rapidjson::Reader reader; + AzStringStream stream(buffer); + RapidJsonReadHandler handler(&visitor, Lifetime::Persistent); + + reader.Parse(stream, handler); + return handler.TakeOutcome(); + } } // namespace AZ::Dom::Json From e0508ddefc315f3d693a9b5c82a0fe835af1969e Mon Sep 17 00:00:00 2001 From: Nicholas Van Sickle Date: Wed, 1 Dec 2021 12:10:22 -0800 Subject: [PATCH 06/18] Make JsonBackend templated Signed-off-by: Nicholas Van Sickle --- .../AzCore/DOM/Backends/JSON/JsonBackend.cpp | 27 -------------- .../AzCore/DOM/Backends/JSON/JsonBackend.h | 21 +++++++++-- .../Backends/JSON/JsonSerializationUtils.h | 35 ++++++++++++------- .../AzCore/AzCore/azcore_files.cmake | 1 - 4 files changed, 41 insertions(+), 43 deletions(-) delete mode 100644 Code/Framework/AzCore/AzCore/DOM/Backends/JSON/JsonBackend.cpp diff --git a/Code/Framework/AzCore/AzCore/DOM/Backends/JSON/JsonBackend.cpp b/Code/Framework/AzCore/AzCore/DOM/Backends/JSON/JsonBackend.cpp deleted file mode 100644 index 288d3c0699..0000000000 --- a/Code/Framework/AzCore/AzCore/DOM/Backends/JSON/JsonBackend.cpp +++ /dev/null @@ -1,27 +0,0 @@ -/* - * Copyright (c) Contributors to the Open 3D Engine Project. - * For complete copyright and license terms please see the LICENSE at the root of this distribution. - * - * SPDX-License-Identifier: Apache-2.0 OR MIT - * - */ - -#include - -namespace AZ::Dom -{ - Visitor::Result JsonBackend::ReadFromStringInPlace(AZStd::string& buffer, Visitor& visitor) - { - return Json::VisitSerializedJsonInPlace(buffer, visitor); - } - - Visitor::Result JsonBackend::ReadFromString(AZStd::string_view buffer, Lifetime lifetime, Visitor& visitor) - { - return Json::VisitSerializedJson(buffer, lifetime, visitor); - } - - AZStd::unique_ptr JsonBackend::CreateStreamWriter(AZ::IO::GenericStream& stream) - { - return Json::GetJsonStreamWriter(stream, Json::OutputFormatting::PrettyPrintedJson); - } -} diff --git a/Code/Framework/AzCore/AzCore/DOM/Backends/JSON/JsonBackend.h b/Code/Framework/AzCore/AzCore/DOM/Backends/JSON/JsonBackend.h index 536fec7418..1ecfa3fb37 100644 --- a/Code/Framework/AzCore/AzCore/DOM/Backends/JSON/JsonBackend.h +++ b/Code/Framework/AzCore/AzCore/DOM/Backends/JSON/JsonBackend.h @@ -13,11 +13,26 @@ namespace AZ::Dom { + //! A DOM backend for serializing and deserializing JSON <=> UTF-8 text + //! \param ParseFlags Controls how deserialized JSON is parsed. + //! \param WriteFormat Controls how serialized JSON is formatted. + template class JsonBackend final : public Backend { public: - Visitor::Result ReadFromStringInPlace(AZStd::string& buffer, Visitor& visitor) override; - Visitor::Result ReadFromString(AZStd::string_view buffer, Lifetime lifetime, Visitor& visitor) override; - AZStd::unique_ptr CreateStreamWriter(AZ::IO::GenericStream& stream) override; + Visitor::Result ReadFromStringInPlace(AZStd::string& buffer, Visitor& visitor) override + { + return Json::VisitSerializedJsonInPlace(buffer, visitor); + } + + Visitor::Result ReadFromString(AZStd::string_view buffer, Lifetime lifetime, Visitor& visitor) override + { + return Json::VisitSerializedJson(buffer, lifetime, visitor); + } + + AZStd::unique_ptr CreateStreamWriter(AZ::IO::GenericStream& stream) override + { + return Json::GetJsonStreamWriter(stream, WriteFormat); + } }; } // namespace AZ::Dom diff --git a/Code/Framework/AzCore/AzCore/DOM/Backends/JSON/JsonSerializationUtils.h b/Code/Framework/AzCore/AzCore/DOM/Backends/JSON/JsonSerializationUtils.h index be055d7f5c..4197209b8e 100644 --- a/Code/Framework/AzCore/AzCore/DOM/Backends/JSON/JsonSerializationUtils.h +++ b/Code/Framework/AzCore/AzCore/DOM/Backends/JSON/JsonSerializationUtils.h @@ -26,6 +26,21 @@ namespace AZ::Dom::Json PrettyPrintedJson, //!< Formats JSON in a pretty printed form, focusing on legibility to readers. }; + //! Specifies parsing behavior when deserializing JSON. + enum class ParseFlags : int + { + Null = 0, + StopWhenDone = rapidjson::kParseStopWhenDoneFlag, + FullFloatingPointPrecision = rapidjson::kParseFullPrecisionFlag, + ParseComments = rapidjson::kParseCommentsFlag, + ParseNumbersAsStrings = rapidjson::kParseNumbersAsStringsFlag, + ParseTrailingCommas = rapidjson::kParseTrailingCommasFlag, + ParseNanAndInfinity = rapidjson::kParseNanAndInfFlag, + ParseEscapedApostrophies = rapidjson::kParseEscapedApostropheFlag, + }; + + AZ_DEFINE_ENUM_BITWISE_OPERATORS(ParseFlags); + //! Visitor that feeds into a rapidjson::Value class RapidJsonValueWriter final : public Visitor { @@ -128,16 +143,18 @@ namespace AZ::Dom::Json //! \param lifetime Specifies the lifetime of the specified buffer. If the string specified by buffer might be deallocated, //! ensure Lifetime::Temporary is specified. //! \param visitor The visitor to visit with the JSON buffer's contents. + //! \param parseFlags (template) Settings for adjusting parser behavior. //! \return The aggregate result specifying whether the visitor operations were successful. - template + template Visitor::Result VisitSerializedJson(AZStd::string_view buffer, Lifetime lifetime, Visitor& visitor); //! Reads serialized JSON from a string in-place and applies it to a visitor. //! \param buffer The UTF-8 serialized JSON to read. This buffer will be modified as part of the deserialization process to //! apply null terminators. //! \param visitor The visitor to visit with the JSON buffer's contents. The strings provided to the visitor will only //! be valid for the lifetime of buffer. + //! \param parseFlags (template) Settings for adjusting parser behavior. //! \return The aggregate result specifying whether the visitor operations were successful. - template + template Visitor::Result VisitSerializedJsonInPlace(AZStd::string& buffer, Visitor& visitor); //! Takes a visitor specified by a callback and produces a rapidjson::Document. @@ -159,31 +176,25 @@ namespace AZ::Dom::Json //! \return The aggregate result specifying whether the visitor operations were successful. Visitor::Result VisitRapidJsonValue(const rapidjson::Value& value, Visitor& visitor, Lifetime lifetime); - template + template Visitor::Result VisitSerializedJson(AZStd::string_view buffer, Lifetime lifetime, Visitor& visitor) { - static_assert( - (ParseFlags & rapidjson::kParseInsituFlag) == 0, - "VisitSerializedJsonInPlace requires kParseInSituFlag not to be set, use VisitSerializedJsonInPlace to parse in-situ"); - rapidjson::Reader reader; rapidjson::MemoryStream stream(buffer.data(), buffer.size()); RapidJsonReadHandler handler(&visitor, lifetime); - reader.Parse(stream, handler); + reader.Parse(parseFlags)>(stream, handler); return handler.TakeOutcome(); } - template + template Visitor::Result VisitSerializedJsonInPlace(AZStd::string& buffer, Visitor& visitor) { - static_assert((ParseFlags & rapidjson::kParseInsituFlag) != 0, "VisitSerializedJsonInPlace requires kParseInSituFlag to be set"); - rapidjson::Reader reader; AzStringStream stream(buffer); RapidJsonReadHandler handler(&visitor, Lifetime::Persistent); - reader.Parse(stream, handler); + reader.Parse(parseFlags) | rapidjson::kParseInsituFlag>(stream, handler); return handler.TakeOutcome(); } } // namespace AZ::Dom::Json diff --git a/Code/Framework/AzCore/AzCore/azcore_files.cmake b/Code/Framework/AzCore/AzCore/azcore_files.cmake index ecc7a41ba1..8557773c93 100644 --- a/Code/Framework/AzCore/AzCore/azcore_files.cmake +++ b/Code/Framework/AzCore/AzCore/azcore_files.cmake @@ -129,7 +129,6 @@ set(FILES DOM/DomBackend.h DOM/DomVisitor.cpp DOM/DomVisitor.h - DOM/Backends/JSON/JsonBackend.cpp DOM/Backends/JSON/JsonBackend.h DOM/Backends/JSON/JsonSerializationUtils.cpp DOM/Backends/JSON/JsonSerializationUtils.h From d519d4cb030bda67b4023c9677116801c858330c Mon Sep 17 00:00:00 2001 From: Nicholas Van Sickle Date: Wed, 1 Dec 2021 16:22:48 -0800 Subject: [PATCH 07/18] Improve performance for both in-place and copy parsing (faster than `AZ::JsonSerializationUtils::ReadJsonString` now!) Signed-off-by: Nicholas Van Sickle --- .../Backends/JSON/JsonSerializationUtils.cpp | 51 ------------ .../Backends/JSON/JsonSerializationUtils.h | 77 ++++++++++++++++--- 2 files changed, 66 insertions(+), 62 deletions(-) diff --git a/Code/Framework/AzCore/AzCore/DOM/Backends/JSON/JsonSerializationUtils.cpp b/Code/Framework/AzCore/AzCore/DOM/Backends/JSON/JsonSerializationUtils.cpp index 84f663f97c..104082ce54 100644 --- a/Code/Framework/AzCore/AzCore/DOM/Backends/JSON/JsonSerializationUtils.cpp +++ b/Code/Framework/AzCore/AzCore/DOM/Backends/JSON/JsonSerializationUtils.cpp @@ -413,57 +413,6 @@ namespace AZ::Dom::Json return true; } - // - // struct AzStringStream - // - // rapidjson stream wrapper for AZStd::string suitable for in-situ parsing - AzStringStream::AzStringStream(AZStd::string& buffer) - { - m_cursor = buffer.data(); - m_begin = m_cursor; - } - - char AzStringStream::Peek() const - { - return *m_cursor; - } - - char AzStringStream::Take() - { - return *m_cursor++; - } - - size_t AzStringStream::Tell() const - { - return static_cast(m_cursor - m_begin); - } - - char* AzStringStream::PutBegin() - { - m_write = m_cursor; - return m_cursor; - } - - void AzStringStream::Put(char c) - { - (*m_write++) = c; - } - - void AzStringStream::Flush() - { - } - - size_t AzStringStream::PutEnd(char* begin) - { - return m_write - begin; - } - - const char* AzStringStream::Peek4() const - { - AZ_Assert(false, "Not implemented, encoding is hard-coded to UTF-8"); - return m_cursor; - } - // // Serialized JSON util functions // diff --git a/Code/Framework/AzCore/AzCore/DOM/Backends/JSON/JsonSerializationUtils.h b/Code/Framework/AzCore/AzCore/DOM/Backends/JSON/JsonSerializationUtils.h index 4197209b8e..5e7976568b 100644 --- a/Code/Framework/AzCore/AzCore/DOM/Backends/JSON/JsonSerializationUtils.h +++ b/Code/Framework/AzCore/AzCore/DOM/Backends/JSON/JsonSerializationUtils.h @@ -113,19 +113,65 @@ namespace AZ::Dom::Json }; //! rapidjson stream wrapper for AZStd::string suitable for in-situ parsing + //! Faster than rapidjson::MemoryStream for reading from AZStd::string / AZStd::string_view (because it requires a null terminator) + //! \note This needs to be inlined for performance reasons. struct AzStringStream { using Ch = char; //(buffer.data()); + m_begin = m_cursor; + } + + AZ_FORCE_INLINE char Peek() const + { + return *m_cursor; + } + + AZ_FORCE_INLINE char Take() + { + return *m_cursor++; + } + + AZ_FORCE_INLINE size_t Tell() const + { + return static_cast(m_cursor - m_begin); + } + + AZ_FORCE_INLINE char* PutBegin() + { + m_write = m_cursor; + return m_cursor; + } + + AZ_FORCE_INLINE void Put(char c) + { + (*m_write++) = c; + } + + AZ_FORCE_INLINE void Flush() + { + } + + AZ_FORCE_INLINE size_t PutEnd(char* begin) + { + return m_write - begin; + } + + AZ_FORCE_INLINE const char* Peek4() const + { + AZ_Assert(false, "Not implemented, encoding is hard-coded to UTF-8"); + return m_cursor; + } char* m_cursor; //!< Current read position. char* m_write; //!< Current write position. @@ -180,10 +226,19 @@ namespace AZ::Dom::Json Visitor::Result VisitSerializedJson(AZStd::string_view buffer, Lifetime lifetime, Visitor& visitor) { rapidjson::Reader reader; - rapidjson::MemoryStream stream(buffer.data(), buffer.size()); RapidJsonReadHandler handler(&visitor, lifetime); - reader.Parse(parseFlags)>(stream, handler); + // If the string is null terminated, we can use the faster AzStringStream path - otherwise we fall back on rapidjson::MemoryStream + if (buffer.data()[buffer.size()] == '\0') + { + AzStringStream stream(buffer); + reader.Parse(parseFlags)>(stream, handler); + } + else + { + rapidjson::MemoryStream stream(buffer.data(), buffer.size()); + reader.Parse(parseFlags)>(stream, handler); + } return handler.TakeOutcome(); } From 2dddb97b7c01ed13fdcd4a6f2987a071fce4dfea Mon Sep 17 00:00:00 2001 From: Nicholas Van Sickle Date: Wed, 1 Dec 2021 16:32:23 -0800 Subject: [PATCH 08/18] One last round of stray review feedback I missed Signed-off-by: Nicholas Van Sickle --- .../Backends/JSON/JsonSerializationUtils.cpp | 30 +++++++++---------- .../AzCore/AzCore/DOM/DomVisitor.cpp | 5 ---- Code/Framework/AzCore/AzCore/DOM/DomVisitor.h | 6 ++-- 3 files changed, 17 insertions(+), 24 deletions(-) diff --git a/Code/Framework/AzCore/AzCore/DOM/Backends/JSON/JsonSerializationUtils.cpp b/Code/Framework/AzCore/AzCore/DOM/Backends/JSON/JsonSerializationUtils.cpp index 104082ce54..8d8c8463ff 100644 --- a/Code/Framework/AzCore/AzCore/DOM/Backends/JSON/JsonSerializationUtils.cpp +++ b/Code/Framework/AzCore/AzCore/DOM/Backends/JSON/JsonSerializationUtils.cpp @@ -70,11 +70,11 @@ namespace AZ::Dom::Json { if (lifetime == Lifetime::Temporary) { - CurrentValue().SetString(value.data(), static_cast(value.length()), m_allocator); + CurrentValue().SetString(value.data(), aznumeric_cast(value.length()), m_allocator); } else { - CurrentValue().SetString(value.data(), static_cast(value.length())); + CurrentValue().SetString(value.data(), aznumeric_cast(value.length())); } return FinishWrite(); } @@ -102,7 +102,7 @@ namespace AZ::Dom::Json if (m_entryStack.front().m_entryCount != attributeCount) { - return FormatVisitorFailure( + return VisitorFailure( VisitorErrorCode::InternalError, "EndObject: Expected %lu attributes but received %lu attributes instead", attributeCount, m_entryStack.front().m_entryCount); } @@ -122,11 +122,11 @@ namespace AZ::Dom::Json AZ_Assert(m_entryStack.front().m_isObject, "Attempted to push a key to an array"); if (lifetime == Lifetime::Persistent) { - m_entryStack.front().m_key.SetString(key.data(), static_cast(key.size())); + m_entryStack.front().m_key.SetString(key.data(), aznumeric_cast(key.size())); } else { - m_entryStack.front().m_key.SetString(key.data(), static_cast(key.size()), m_allocator); + m_entryStack.front().m_key.SetString(key.data(), aznumeric_cast(key.size()), m_allocator); } return VisitorSuccess(); } @@ -154,7 +154,7 @@ namespace AZ::Dom::Json if (m_entryStack.front().m_entryCount != elementCount) { - return FormatVisitorFailure( + return VisitorFailure( VisitorErrorCode::InternalError, "EndArray: Expected %lu elements but received %lu elements instead", elementCount, m_entryStack.front().m_entryCount); } @@ -250,7 +250,7 @@ namespace AZ::Dom::Json Result String(AZStd::string_view value, Lifetime lifetime) override { const bool shouldCopy = lifetime == Lifetime::Temporary; - return CheckWrite(m_writer.String(value.data(), static_cast(value.size()), shouldCopy)); + return CheckWrite(m_writer.String(value.data(), aznumeric_cast(value.size()), shouldCopy)); } Result StartObject() override @@ -260,7 +260,7 @@ namespace AZ::Dom::Json Result EndObject(AZ::u64 attributeCount) override { - return CheckWrite(m_writer.EndObject(static_cast(attributeCount))); + return CheckWrite(m_writer.EndObject(aznumeric_cast(attributeCount))); } Result Key(AZ::Name key) override @@ -271,7 +271,7 @@ namespace AZ::Dom::Json Result RawKey(AZStd::string_view key, Lifetime lifetime) override { const bool shouldCopy = lifetime == Lifetime::Temporary; - return CheckWrite(m_writer.Key(key.data(), static_cast(key.size()), shouldCopy)); + return CheckWrite(m_writer.Key(key.data(), aznumeric_cast(key.size()), shouldCopy)); } Result StartArray() override @@ -281,7 +281,7 @@ namespace AZ::Dom::Json Result EndArray(AZ::u64 elementCount) override { - return CheckWrite(m_writer.EndArray(static_cast(elementCount))); + return CheckWrite(m_writer.EndArray(aznumeric_cast(elementCount))); } private: @@ -323,12 +323,12 @@ namespace AZ::Dom::Json bool RapidJsonReadHandler::Int(int i) { - return CheckResult(m_visitor->Int64(static_cast(i))); + return CheckResult(m_visitor->Int64(aznumeric_cast(i))); } bool RapidJsonReadHandler::Uint(unsigned i) { - return CheckResult(m_visitor->Uint64(static_cast(i))); + return CheckResult(m_visitor->Uint64(aznumeric_cast(i))); } bool RapidJsonReadHandler::Int64(int64_t i) @@ -472,7 +472,7 @@ namespace AZ::Dom::Json while (!entryStack.empty()) { - const auto currentEntry = entryStack.top(); + const Entry currentEntry = entryStack.top(); entryStack.pop(); Visitor::Result result = AZ::Success(); @@ -507,7 +507,7 @@ namespace AZ::Dom::Json for (auto it = currentValue.MemberEnd(); it != currentValue.MemberBegin(); --it) { auto entry = (it - 1); - const AZStd::string_view key(entry->name.GetString(), static_cast(entry->name.GetStringLength())); + const AZStd::string_view key(entry->name.GetString(), aznumeric_cast(entry->name.GetStringLength())); entryStack.push(&entry->value); entryStack.push(key); } @@ -524,7 +524,7 @@ namespace AZ::Dom::Json break; case rapidjson::kStringType: result = visitor.String( - AZStd::string_view(currentValue.GetString(), static_cast(currentValue.GetStringLength())), + AZStd::string_view(currentValue.GetString(), aznumeric_cast(currentValue.GetStringLength())), lifetime); break; case rapidjson::kNumberType: diff --git a/Code/Framework/AzCore/AzCore/DOM/DomVisitor.cpp b/Code/Framework/AzCore/AzCore/DOM/DomVisitor.cpp index ad314da385..d5190cbbac 100644 --- a/Code/Framework/AzCore/AzCore/DOM/DomVisitor.cpp +++ b/Code/Framework/AzCore/AzCore/DOM/DomVisitor.cpp @@ -60,11 +60,6 @@ namespace AZ::Dom return AZ::Failure(VisitorError(code)); } - Visitor::Result Visitor::VisitorFailure(VisitorErrorCode code, AZStd::string additionalInfo) - { - return AZ::Failure(VisitorError(code, AZStd::move(additionalInfo))); - } - Visitor::Result Visitor::VisitorFailure(VisitorError error) { return AZ::Failure(error); diff --git a/Code/Framework/AzCore/AzCore/DOM/DomVisitor.h b/Code/Framework/AzCore/AzCore/DOM/DomVisitor.h index f36ad8d827..c0da56036e 100644 --- a/Code/Framework/AzCore/AzCore/DOM/DomVisitor.h +++ b/Code/Framework/AzCore/AzCore/DOM/DomVisitor.h @@ -227,17 +227,15 @@ namespace AZ::Dom //! Helper method, constructs a failure \ref Result with the specified code. static Result VisitorFailure(VisitorErrorCode code); - //! Helper method, constructs a failure \ref Result with the specified code and supplemental info. - static Result VisitorFailure(VisitorErrorCode code, AZStd::string additionalInfo); //! Helper method, constructs a failure \ref Result with the specified error. static Result VisitorFailure(VisitorError error); //! Helper method, constructs a failure \ref Result with the specified code and supplemental info specified by a format string //! and its arguments. template - static Result FormatVisitorFailure(VisitorErrorCode code, TArgs... formatArgs) + static Result VisitorFailure(VisitorErrorCode code, TArgs... formatArgs) { - return VisitorFailure(code, AZStd::string::format(formatArgs...)); + return AZ::Failure(VisitorError(code, AZStd::string::format(formatArgs...))); } //! Helper method, constructs a success \ref Result. From 1aaa2675857da20b16b5b2b65e3dc542c00f6fe1 Mon Sep 17 00:00:00 2001 From: Nicholas Van Sickle Date: Wed, 1 Dec 2021 16:49:48 -0800 Subject: [PATCH 09/18] Fix one last bit of minor feedback Signed-off-by: Nicholas Van Sickle --- Code/Framework/AzCore/AzCore/DOM/Backends/JSON/JsonBackend.h | 2 +- .../AzCore/AzCore/DOM/Backends/JSON/JsonSerializationUtils.cpp | 2 +- .../AzCore/AzCore/DOM/Backends/JSON/JsonSerializationUtils.h | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/Code/Framework/AzCore/AzCore/DOM/Backends/JSON/JsonBackend.h b/Code/Framework/AzCore/AzCore/DOM/Backends/JSON/JsonBackend.h index 1ecfa3fb37..ace21d9be8 100644 --- a/Code/Framework/AzCore/AzCore/DOM/Backends/JSON/JsonBackend.h +++ b/Code/Framework/AzCore/AzCore/DOM/Backends/JSON/JsonBackend.h @@ -32,7 +32,7 @@ namespace AZ::Dom AZStd::unique_ptr CreateStreamWriter(AZ::IO::GenericStream& stream) override { - return Json::GetJsonStreamWriter(stream, WriteFormat); + return Json::CreateJsonStreamWriter(stream, WriteFormat); } }; } // namespace AZ::Dom diff --git a/Code/Framework/AzCore/AzCore/DOM/Backends/JSON/JsonSerializationUtils.cpp b/Code/Framework/AzCore/AzCore/DOM/Backends/JSON/JsonSerializationUtils.cpp index 8d8c8463ff..66851f1a40 100644 --- a/Code/Framework/AzCore/AzCore/DOM/Backends/JSON/JsonSerializationUtils.cpp +++ b/Code/Framework/AzCore/AzCore/DOM/Backends/JSON/JsonSerializationUtils.cpp @@ -416,7 +416,7 @@ namespace AZ::Dom::Json // // Serialized JSON util functions // - AZStd::unique_ptr GetJsonStreamWriter(AZ::IO::GenericStream& stream, OutputFormatting format) + AZStd::unique_ptr CreateJsonStreamWriter(AZ::IO::GenericStream& stream, OutputFormatting format) { if (format == OutputFormatting::MinifiedJson) { diff --git a/Code/Framework/AzCore/AzCore/DOM/Backends/JSON/JsonSerializationUtils.h b/Code/Framework/AzCore/AzCore/DOM/Backends/JSON/JsonSerializationUtils.h index 5e7976568b..88c6c42978 100644 --- a/Code/Framework/AzCore/AzCore/DOM/Backends/JSON/JsonSerializationUtils.h +++ b/Code/Framework/AzCore/AzCore/DOM/Backends/JSON/JsonSerializationUtils.h @@ -182,7 +182,7 @@ namespace AZ::Dom::Json //! \param stream The stream the visitor will write to. //! \param format The format to write in. //! \return A Visitor that will write to stream when visited. - AZStd::unique_ptr GetJsonStreamWriter( + AZStd::unique_ptr CreateJsonStreamWriter( AZ::IO::GenericStream& stream, OutputFormatting format = OutputFormatting::PrettyPrintedJson); //! Reads serialized JSON from a string and applies it to a visitor. //! \param buffer The UTF-8 serialized JSON to read. From 4dbce4275bd2b797c14414bb76ea577eae11d835 Mon Sep 17 00:00:00 2001 From: Nicholas Van Sickle Date: Thu, 2 Dec 2021 17:46:07 -0800 Subject: [PATCH 10/18] Refactor the interface after some chatting with @amazon-employee-dm Signed-off-by: Nicholas Van Sickle --- .../AzCore/DOM/Backends/JSON/JsonBackend.h | 19 +++++----- .../Backends/JSON/JsonSerializationUtils.h | 17 ++++----- .../AzCore/AzCore/DOM/DomBackend.cpp | 36 ++----------------- Code/Framework/AzCore/AzCore/DOM/DomBackend.h | 20 ++++------- .../AzCore/AzCore/azcore_files.cmake | 2 ++ .../AzCore/Tests/DOM/DomJsonBenchmarks.cpp | 5 +-- .../AzCore/Tests/DOM/DomJsonTests.cpp | 11 +++--- 7 files changed, 41 insertions(+), 69 deletions(-) diff --git a/Code/Framework/AzCore/AzCore/DOM/Backends/JSON/JsonBackend.h b/Code/Framework/AzCore/AzCore/DOM/Backends/JSON/JsonBackend.h index ace21d9be8..1c8f4eb607 100644 --- a/Code/Framework/AzCore/AzCore/DOM/Backends/JSON/JsonBackend.h +++ b/Code/Framework/AzCore/AzCore/DOM/Backends/JSON/JsonBackend.h @@ -8,31 +8,34 @@ #pragma once -#include #include +#include namespace AZ::Dom { //! A DOM backend for serializing and deserializing JSON <=> UTF-8 text //! \param ParseFlags Controls how deserialized JSON is parsed. //! \param WriteFormat Controls how serialized JSON is formatted. - template + template< + Json::ParseFlags ParseFlags = Json::ParseFlags::ParseComments, + Json::OutputFormatting WriteFormat = Json::OutputFormatting::PrettyPrintedJson> class JsonBackend final : public Backend { public: - Visitor::Result ReadFromStringInPlace(AZStd::string& buffer, Visitor& visitor) override + Visitor::Result ReadFromBuffer(const char* buffer, size_t size, AZ::Dom::Lifetime lifetime, Visitor& visitor) override { - return Json::VisitSerializedJsonInPlace(buffer, visitor); + return Json::VisitSerializedJson({ buffer, size }, lifetime, visitor); } - Visitor::Result ReadFromString(AZStd::string_view buffer, Lifetime lifetime, Visitor& visitor) override + Visitor::Result ReadFromBufferInPlace(char* buffer, Visitor& visitor) override { - return Json::VisitSerializedJson(buffer, lifetime, visitor); + return Json::VisitSerializedJsonInPlace(buffer, visitor); } - AZStd::unique_ptr CreateStreamWriter(AZ::IO::GenericStream& stream) override + Visitor::Result WriteToStream(AZ::IO::GenericStream& stream, WriteCallback callback) override { - return Json::CreateJsonStreamWriter(stream, WriteFormat); + AZStd::unique_ptr visitor = Json::CreateJsonStreamWriter(stream, WriteFormat); + return callback(*visitor.get()); } }; } // namespace AZ::Dom diff --git a/Code/Framework/AzCore/AzCore/DOM/Backends/JSON/JsonSerializationUtils.h b/Code/Framework/AzCore/AzCore/DOM/Backends/JSON/JsonSerializationUtils.h index 88c6c42978..db43395475 100644 --- a/Code/Framework/AzCore/AzCore/DOM/Backends/JSON/JsonSerializationUtils.h +++ b/Code/Framework/AzCore/AzCore/DOM/Backends/JSON/JsonSerializationUtils.h @@ -115,17 +115,17 @@ namespace AZ::Dom::Json //! rapidjson stream wrapper for AZStd::string suitable for in-situ parsing //! Faster than rapidjson::MemoryStream for reading from AZStd::string / AZStd::string_view (because it requires a null terminator) //! \note This needs to be inlined for performance reasons. - struct AzStringStream + struct NullDelimitedStringStream { using Ch = char; //(buffer.data()); @@ -193,6 +193,7 @@ namespace AZ::Dom::Json //! \return The aggregate result specifying whether the visitor operations were successful. template Visitor::Result VisitSerializedJson(AZStd::string_view buffer, Lifetime lifetime, Visitor& visitor); + //! Reads serialized JSON from a string in-place and applies it to a visitor. //! \param buffer The UTF-8 serialized JSON to read. This buffer will be modified as part of the deserialization process to //! apply null terminators. @@ -201,7 +202,7 @@ namespace AZ::Dom::Json //! \param parseFlags (template) Settings for adjusting parser behavior. //! \return The aggregate result specifying whether the visitor operations were successful. template - Visitor::Result VisitSerializedJsonInPlace(AZStd::string& buffer, Visitor& visitor); + Visitor::Result VisitSerializedJsonInPlace(char* buffer, Visitor& visitor); //! Takes a visitor specified by a callback and produces a rapidjson::Document. //! \param writeCallback A callback specifying a visitor to accept to build the resulting document. @@ -231,7 +232,7 @@ namespace AZ::Dom::Json // If the string is null terminated, we can use the faster AzStringStream path - otherwise we fall back on rapidjson::MemoryStream if (buffer.data()[buffer.size()] == '\0') { - AzStringStream stream(buffer); + NullDelimitedStringStream stream(buffer); reader.Parse(parseFlags)>(stream, handler); } else @@ -243,10 +244,10 @@ namespace AZ::Dom::Json } template - Visitor::Result VisitSerializedJsonInPlace(AZStd::string& buffer, Visitor& visitor) + Visitor::Result VisitSerializedJsonInPlace(char* buffer, Visitor& visitor) { rapidjson::Reader reader; - AzStringStream stream(buffer); + NullDelimitedStringStream stream(buffer); RapidJsonReadHandler handler(&visitor, Lifetime::Persistent); reader.Parse(parseFlags) | rapidjson::kParseInsituFlag>(stream, handler); diff --git a/Code/Framework/AzCore/AzCore/DOM/DomBackend.cpp b/Code/Framework/AzCore/AzCore/DOM/DomBackend.cpp index 258975eff3..8ff0e8a0bb 100644 --- a/Code/Framework/AzCore/AzCore/DOM/DomBackend.cpp +++ b/Code/Framework/AzCore/AzCore/DOM/DomBackend.cpp @@ -8,40 +8,10 @@ #include -#include -#include -#include - namespace AZ::Dom { - Visitor::Result Backend::ReadFromStream(AZ::IO::GenericStream* stream, Visitor& visitor, size_t maxSize) - { - size_t length = stream->GetLength(); - if (length > maxSize) - { - return AZ::Failure(VisitorError(VisitorErrorCode::InternalError, "Stream is too large.")); - } - AZStd::string buffer; - buffer.resize(length); - stream->Read(length, buffer.data()); - return ReadFromString(buffer, Lifetime::Temporary, visitor); - } - - Visitor::Result Backend::ReadFromStringInPlace(AZStd::string& buffer, Visitor& visitor) - { - return ReadFromString(buffer, Lifetime::Persistent, visitor); - } - - Visitor::Result Backend::WriteToStream(AZ::IO::GenericStream& stream, WriteCallback callback) - { - AZStd::unique_ptr writer = CreateStreamWriter(stream); - return callback(*writer.get()); - } - - Visitor::Result Backend::WriteToString(AZStd::string& buffer, WriteCallback callback) + Visitor::Result Backend::ReadFromBufferInPlace(char* buffer, Visitor& visitor) { - AZ::IO::ByteContainerStream stream{&buffer}; - AZStd::unique_ptr writer = CreateStreamWriter(stream); - return callback(*writer.get()); + return ReadFromBuffer(buffer, strlen(buffer), AZ::Dom::Lifetime::Persistent, visitor); } -} +} // namespace AZ::Dom diff --git a/Code/Framework/AzCore/AzCore/DOM/DomBackend.h b/Code/Framework/AzCore/AzCore/DOM/DomBackend.h index d1eee5d676..c70af72652 100644 --- a/Code/Framework/AzCore/AzCore/DOM/DomBackend.h +++ b/Code/Framework/AzCore/AzCore/DOM/DomBackend.h @@ -22,28 +22,22 @@ namespace AZ::Dom public: virtual ~Backend() = default; - //! Attempt to read this format from the given stream into the target Visitor. - //! The base implementation reads the stream into memory and calls ReadFromString. - virtual Visitor::Result ReadFromStream( - AZ::IO::GenericStream* stream, Visitor& visitor, size_t maxSize = AZStd::numeric_limits::max()); + //! Attempt to read this format from the given buffer into the target Visitor. + virtual Visitor::Result ReadFromBuffer( + const char* buffer, size_t size, AZ::Dom::Lifetime lifetime, Visitor& visitor) = 0; //! Attempt to read this format from a mutable string into the target Visitor. This enables some backends to //! parse without making additional string allocations. + //! This string must be null terminated. //! This string may be modified and read in place without being copied, so when calling this please ensure that: //! - The string won't be deallocated until the visitor no longer needs the values and //! - The string is safe to modify in place. - //! The base implementation simply calls ReadFromString. - virtual Visitor::Result ReadFromStringInPlace(AZStd::string& buffer, Visitor& visitor); - //! Attempt to read this format from an immutable buffer in memory into the target Visitor. - virtual Visitor::Result ReadFromString(AZStd::string_view buffer, Lifetime lifetime, Visitor& visitor) = 0; + //! The base implementation simply calls ReadFromBuffer. + virtual Visitor::Result ReadFromBufferInPlace(char* buffer, Visitor& visitor); - //! Acquire a visitor interface for writing to the target output file. - virtual AZStd::unique_ptr CreateStreamWriter(AZ::IO::GenericStream& stream) = 0; //! A callback that accepts a Visitor, making DOM calls to inform the serializer, and returns an //! aggregate error code to indicate whether or not the operation succeeded. using WriteCallback = AZStd::function; //! Attempt to write a value to a stream using a write callback. - Visitor::Result WriteToStream(AZ::IO::GenericStream& stream, WriteCallback callback); - //! Attempt to write a value to a string using a write callback. - Visitor::Result WriteToString(AZStd::string& buffer, WriteCallback callback); + virtual Visitor::Result WriteToStream(AZ::IO::GenericStream& stream, WriteCallback callback) = 0; }; } // namespace AZ::Dom diff --git a/Code/Framework/AzCore/AzCore/azcore_files.cmake b/Code/Framework/AzCore/AzCore/azcore_files.cmake index 8557773c93..ef6c7434cc 100644 --- a/Code/Framework/AzCore/AzCore/azcore_files.cmake +++ b/Code/Framework/AzCore/AzCore/azcore_files.cmake @@ -127,6 +127,8 @@ set(FILES Debug/TraceReflection.h DOM/DomBackend.cpp DOM/DomBackend.h + DOM/DomUtils.cpp + DOM/DomUtils.h DOM/DomVisitor.cpp DOM/DomVisitor.h DOM/Backends/JSON/JsonBackend.h diff --git a/Code/Framework/AzCore/Tests/DOM/DomJsonBenchmarks.cpp b/Code/Framework/AzCore/Tests/DOM/DomJsonBenchmarks.cpp index ea72092001..0baa4aeb53 100644 --- a/Code/Framework/AzCore/Tests/DOM/DomJsonBenchmarks.cpp +++ b/Code/Framework/AzCore/Tests/DOM/DomJsonBenchmarks.cpp @@ -8,6 +8,7 @@ #if defined(HAVE_BENCHMARK) +#include #include #include #include @@ -132,7 +133,7 @@ namespace Benchmark auto result = AZ::Dom::Json::WriteToRapidJsonDocument( [&](AZ::Dom::Visitor& visitor) { - return backend.ReadFromStringInPlace(payloadCopy, visitor); + return AZ::Dom::Utils::ReadFromStringInPlace(backend, payloadCopy, visitor); }); benchmark::DoNotOptimize(result.GetValue()); @@ -152,7 +153,7 @@ namespace Benchmark auto result = AZ::Dom::Json::WriteToRapidJsonDocument( [&](AZ::Dom::Visitor& visitor) { - return backend.ReadFromString(serializedPayload, AZ::Dom::Lifetime::Temporary, visitor); + return AZ::Dom::Utils::ReadFromString(backend, serializedPayload, AZ::Dom::Lifetime::Temporary, visitor); }); benchmark::DoNotOptimize(result.GetValue()); diff --git a/Code/Framework/AzCore/Tests/DOM/DomJsonTests.cpp b/Code/Framework/AzCore/Tests/DOM/DomJsonTests.cpp index cde5e2eb3f..a416bcf86b 100644 --- a/Code/Framework/AzCore/Tests/DOM/DomJsonTests.cpp +++ b/Code/Framework/AzCore/Tests/DOM/DomJsonTests.cpp @@ -8,6 +8,7 @@ #include #include +#include #include #include #include @@ -70,7 +71,7 @@ namespace AZ::Dom::Tests { AZStd::string serializedDocument; JsonBackend backend; - auto result = backend.WriteToString(serializedDocument, visitDocumentFn); + auto result = Dom::Utils::WriteToString(backend, serializedDocument, visitDocumentFn); EXPECT_TRUE(result.IsSuccess()); EXPECT_EQ(canonicalSerializedDocument, serializedDocument); } @@ -81,7 +82,7 @@ namespace AZ::Dom::Tests [&canonicalSerializedDocument](AZ::Dom::Visitor& visitor) { JsonBackend backend; - return backend.ReadFromString(canonicalSerializedDocument, AZ::Dom::Lifetime::Temporary, visitor); + return Dom::Utils::ReadFromString(backend, canonicalSerializedDocument, Dom::Lifetime::Persistent, visitor); }); EXPECT_TRUE(result.IsSuccess()); EXPECT_EQ(AZ::JsonSerialization::Compare(*m_document, result.GetValue()), JsonSerializerCompareResult::Equal); @@ -91,11 +92,11 @@ namespace AZ::Dom::Tests { AZStd::string serializedDocument; JsonBackend backend; - auto result = backend.WriteToString( - serializedDocument, + auto result = Dom::Utils::WriteToString( + backend, serializedDocument, [&backend, &canonicalSerializedDocument](AZ::Dom::Visitor& visitor) { - return backend.ReadFromString(canonicalSerializedDocument, AZ::Dom::Lifetime::Temporary, visitor); + return Dom::Utils::ReadFromString(backend, canonicalSerializedDocument, Dom::Lifetime::Persistent, visitor); }); EXPECT_TRUE(result.IsSuccess()); EXPECT_EQ(canonicalSerializedDocument, serializedDocument); From 59a22165847e58925541654cad308fc420ffdf19 Mon Sep 17 00:00:00 2001 From: Nicholas Van Sickle Date: Thu, 2 Dec 2021 17:49:29 -0800 Subject: [PATCH 11/18] Fix a build issue Signed-off-by: Nicholas Van Sickle --- Code/Framework/AzCore/AzCore/DOM/DomVisitor.h | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Code/Framework/AzCore/AzCore/DOM/DomVisitor.h b/Code/Framework/AzCore/AzCore/DOM/DomVisitor.h index c0da56036e..26fb2aa063 100644 --- a/Code/Framework/AzCore/AzCore/DOM/DomVisitor.h +++ b/Code/Framework/AzCore/AzCore/DOM/DomVisitor.h @@ -233,9 +233,9 @@ namespace AZ::Dom //! Helper method, constructs a failure \ref Result with the specified code and supplemental info specified by a format string //! and its arguments. template - static Result VisitorFailure(VisitorErrorCode code, TArgs... formatArgs) + static Result VisitorFailure(VisitorErrorCode code, const char* formatString, TArgs... formatArgs) { - return AZ::Failure(VisitorError(code, AZStd::string::format(formatArgs...))); + return AZ::Failure(VisitorError(code, AZStd::string::format(formatString, formatArgs...))); } //! Helper method, constructs a success \ref Result. From 016e261f368121f445faf99ee5ad3b53b4ff3668 Mon Sep 17 00:00:00 2001 From: Nicholas Van Sickle Date: Fri, 3 Dec 2021 11:25:15 -0800 Subject: [PATCH 12/18] Add DomUtils Signed-off-by: Nicholas Van Sickle --- Code/Framework/AzCore/AzCore/DOM/DomUtils.cpp | 30 +++++++++++++++++++ Code/Framework/AzCore/AzCore/DOM/DomUtils.h | 19 ++++++++++++ 2 files changed, 49 insertions(+) create mode 100644 Code/Framework/AzCore/AzCore/DOM/DomUtils.cpp create mode 100644 Code/Framework/AzCore/AzCore/DOM/DomUtils.h diff --git a/Code/Framework/AzCore/AzCore/DOM/DomUtils.cpp b/Code/Framework/AzCore/AzCore/DOM/DomUtils.cpp new file mode 100644 index 0000000000..3c5d4eb668 --- /dev/null +++ b/Code/Framework/AzCore/AzCore/DOM/DomUtils.cpp @@ -0,0 +1,30 @@ +/* + * Copyright (c) Contributors to the Open 3D Engine Project. + * For complete copyright and license terms please see the LICENSE at the root of this distribution. + * + * SPDX-License-Identifier: Apache-2.0 OR MIT + * + */ + +#include + +#include + +namespace AZ::Dom::Utils +{ + Visitor::Result ReadFromString(Backend& backend, AZStd::string_view string, AZ::Dom::Lifetime lifetime, Visitor& visitor) + { + return backend.ReadFromBuffer(string.data(), string.length(), lifetime, visitor); + } + + Visitor::Result ReadFromStringInPlace(Backend& backend, AZStd::string& string, Visitor& visitor) + { + return backend.ReadFromBufferInPlace(string.data(), visitor); + } + + Visitor::Result WriteToString(Backend& backend, AZStd::string& buffer, Backend::WriteCallback writeCallback) + { + AZ::IO::ByteContainerStream stream{ &buffer }; + return backend.WriteToStream(stream, writeCallback); + } +} diff --git a/Code/Framework/AzCore/AzCore/DOM/DomUtils.h b/Code/Framework/AzCore/AzCore/DOM/DomUtils.h new file mode 100644 index 0000000000..cd64d2e107 --- /dev/null +++ b/Code/Framework/AzCore/AzCore/DOM/DomUtils.h @@ -0,0 +1,19 @@ +/* + * Copyright (c) Contributors to the Open 3D Engine Project. + * For complete copyright and license terms please see the LICENSE at the root of this distribution. + * + * SPDX-License-Identifier: Apache-2.0 OR MIT + * + */ + +#pragma once + +#include + +namespace AZ::Dom::Utils +{ + Visitor::Result ReadFromString(Backend& backend, AZStd::string_view string, AZ::Dom::Lifetime lifetime, Visitor& visitor); + Visitor::Result ReadFromStringInPlace(Backend& backend, AZStd::string& string, Visitor& visitor); + + Visitor::Result WriteToString(Backend& backend, AZStd::string& buffer, Backend::WriteCallback writeCallback); +} // namespace AZ::Dom::Utils From f2f7dad97219c63dfedfd86504c030c90b571a57 Mon Sep 17 00:00:00 2001 From: Nicholas Van Sickle Date: Fri, 3 Dec 2021 12:26:19 -0800 Subject: [PATCH 13/18] Remove FormatVisitorErrorMessage Signed-off-by: Nicholas Van Sickle --- .../DOM/Backends/JSON/JsonSerializationUtils.cpp | 14 +++++++++----- Code/Framework/AzCore/AzCore/DOM/DomVisitor.cpp | 5 +++++ Code/Framework/AzCore/AzCore/DOM/DomVisitor.h | 10 ++-------- 3 files changed, 16 insertions(+), 13 deletions(-) diff --git a/Code/Framework/AzCore/AzCore/DOM/Backends/JSON/JsonSerializationUtils.cpp b/Code/Framework/AzCore/AzCore/DOM/Backends/JSON/JsonSerializationUtils.cpp index 66851f1a40..4adf0bdd21 100644 --- a/Code/Framework/AzCore/AzCore/DOM/Backends/JSON/JsonSerializationUtils.cpp +++ b/Code/Framework/AzCore/AzCore/DOM/Backends/JSON/JsonSerializationUtils.cpp @@ -103,8 +103,10 @@ namespace AZ::Dom::Json if (m_entryStack.front().m_entryCount != attributeCount) { return VisitorFailure( - VisitorErrorCode::InternalError, "EndObject: Expected %lu attributes but received %lu attributes instead", attributeCount, - m_entryStack.front().m_entryCount); + VisitorErrorCode::InternalError, + AZStd::string::format( + "EndObject: Expected %lu attributes but received %lu attributes instead", attributeCount, + m_entryStack.front().m_entryCount)); } m_entryStack.pop_front(); @@ -155,8 +157,9 @@ namespace AZ::Dom::Json if (m_entryStack.front().m_entryCount != elementCount) { return VisitorFailure( - VisitorErrorCode::InternalError, "EndArray: Expected %lu elements but received %lu elements instead", elementCount, - m_entryStack.front().m_entryCount); + VisitorErrorCode::InternalError, + AZStd::string::format( + "EndArray: Expected %lu elements but received %lu elements instead", elementCount, m_entryStack.front().m_entryCount)); } m_entryStack.pop_front(); @@ -507,7 +510,8 @@ namespace AZ::Dom::Json for (auto it = currentValue.MemberEnd(); it != currentValue.MemberBegin(); --it) { auto entry = (it - 1); - const AZStd::string_view key(entry->name.GetString(), aznumeric_cast(entry->name.GetStringLength())); + const AZStd::string_view key( + entry->name.GetString(), aznumeric_cast(entry->name.GetStringLength())); entryStack.push(&entry->value); entryStack.push(key); } diff --git a/Code/Framework/AzCore/AzCore/DOM/DomVisitor.cpp b/Code/Framework/AzCore/AzCore/DOM/DomVisitor.cpp index d5190cbbac..ad314da385 100644 --- a/Code/Framework/AzCore/AzCore/DOM/DomVisitor.cpp +++ b/Code/Framework/AzCore/AzCore/DOM/DomVisitor.cpp @@ -60,6 +60,11 @@ namespace AZ::Dom return AZ::Failure(VisitorError(code)); } + Visitor::Result Visitor::VisitorFailure(VisitorErrorCode code, AZStd::string additionalInfo) + { + return AZ::Failure(VisitorError(code, AZStd::move(additionalInfo))); + } + Visitor::Result Visitor::VisitorFailure(VisitorError error) { return AZ::Failure(error); diff --git a/Code/Framework/AzCore/AzCore/DOM/DomVisitor.h b/Code/Framework/AzCore/AzCore/DOM/DomVisitor.h index 26fb2aa063..bbe78131c3 100644 --- a/Code/Framework/AzCore/AzCore/DOM/DomVisitor.h +++ b/Code/Framework/AzCore/AzCore/DOM/DomVisitor.h @@ -227,17 +227,11 @@ namespace AZ::Dom //! Helper method, constructs a failure \ref Result with the specified code. static Result VisitorFailure(VisitorErrorCode code); + //! Helper method, constructs a failure \ref Result with the specified code and supplemental info. + static Result VisitorFailure(VisitorErrorCode code, AZStd::string additionalInfo); //! Helper method, constructs a failure \ref Result with the specified error. static Result VisitorFailure(VisitorError error); - //! Helper method, constructs a failure \ref Result with the specified code and supplemental info specified by a format string - //! and its arguments. - template - static Result VisitorFailure(VisitorErrorCode code, const char* formatString, TArgs... formatArgs) - { - return AZ::Failure(VisitorError(code, AZStd::string::format(formatString, formatArgs...))); - } - //! Helper method, constructs a success \ref Result. static Result VisitorSuccess(); }; From 037a93406ac3d3cbd04960b7fa43b1bdb364fa5f Mon Sep 17 00:00:00 2001 From: Nicholas Van Sickle Date: Fri, 3 Dec 2021 12:40:52 -0800 Subject: [PATCH 14/18] Fix a couple format string issues Signed-off-by: Nicholas Van Sickle --- .../AzCore/DOM/Backends/JSON/JsonSerializationUtils.cpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Code/Framework/AzCore/AzCore/DOM/Backends/JSON/JsonSerializationUtils.cpp b/Code/Framework/AzCore/AzCore/DOM/Backends/JSON/JsonSerializationUtils.cpp index 4adf0bdd21..e1db5be29e 100644 --- a/Code/Framework/AzCore/AzCore/DOM/Backends/JSON/JsonSerializationUtils.cpp +++ b/Code/Framework/AzCore/AzCore/DOM/Backends/JSON/JsonSerializationUtils.cpp @@ -105,7 +105,7 @@ namespace AZ::Dom::Json return VisitorFailure( VisitorErrorCode::InternalError, AZStd::string::format( - "EndObject: Expected %lu attributes but received %lu attributes instead", attributeCount, + "EndObject: Expected %llu attributes but received %llu attributes instead", attributeCount, m_entryStack.front().m_entryCount)); } @@ -159,7 +159,7 @@ namespace AZ::Dom::Json return VisitorFailure( VisitorErrorCode::InternalError, AZStd::string::format( - "EndArray: Expected %lu elements but received %lu elements instead", elementCount, m_entryStack.front().m_entryCount)); + "EndArray: Expected %llu elements but received %llu elements instead", elementCount, m_entryStack.front().m_entryCount)); } m_entryStack.pop_front(); From 8165f54c05ee4babe5b2585e5ef6e02fb1d9cb1f Mon Sep 17 00:00:00 2001 From: Nicholas Van Sickle Date: Fri, 3 Dec 2021 13:52:16 -0800 Subject: [PATCH 15/18] Use temporary lifetimes for test Signed-off-by: Nicholas Van Sickle --- Code/Framework/AzCore/Tests/DOM/DomJsonTests.cpp | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/Code/Framework/AzCore/Tests/DOM/DomJsonTests.cpp b/Code/Framework/AzCore/Tests/DOM/DomJsonTests.cpp index a416bcf86b..b408cd2466 100644 --- a/Code/Framework/AzCore/Tests/DOM/DomJsonTests.cpp +++ b/Code/Framework/AzCore/Tests/DOM/DomJsonTests.cpp @@ -57,7 +57,7 @@ namespace AZ::Dom::Tests auto visitDocumentFn = [this](AZ::Dom::Visitor& visitor) { - return Json::VisitRapidJsonValue(*m_document, visitor, Lifetime::Persistent); + return Json::VisitRapidJsonValue(*m_document, visitor, Lifetime::Temporary); }; // Document -> Document @@ -82,7 +82,7 @@ namespace AZ::Dom::Tests [&canonicalSerializedDocument](AZ::Dom::Visitor& visitor) { JsonBackend backend; - return Dom::Utils::ReadFromString(backend, canonicalSerializedDocument, Dom::Lifetime::Persistent, visitor); + return Dom::Utils::ReadFromString(backend, canonicalSerializedDocument, Lifetime::Temporary, visitor); }); EXPECT_TRUE(result.IsSuccess()); EXPECT_EQ(AZ::JsonSerialization::Compare(*m_document, result.GetValue()), JsonSerializerCompareResult::Equal); @@ -96,7 +96,7 @@ namespace AZ::Dom::Tests backend, serializedDocument, [&backend, &canonicalSerializedDocument](AZ::Dom::Visitor& visitor) { - return Dom::Utils::ReadFromString(backend, canonicalSerializedDocument, Dom::Lifetime::Persistent, visitor); + return Dom::Utils::ReadFromString(backend, canonicalSerializedDocument, Lifetime::Temporary, visitor); }); EXPECT_TRUE(result.IsSuccess()); EXPECT_EQ(canonicalSerializedDocument, serializedDocument); From 1c42cd98cc11277a3c0a0f634e7e0a58e975409c Mon Sep 17 00:00:00 2001 From: Nicholas Van Sickle Date: Mon, 6 Dec 2021 10:31:35 -0800 Subject: [PATCH 16/18] Address a bit more review feedback Signed-off-by: Nicholas Van Sickle --- .../AzCore/DOM/Backends/JSON/JsonBackend.h | 2 +- .../Backends/JSON/JsonSerializationUtils.cpp | 44 +++++++++---------- .../Backends/JSON/JsonSerializationUtils.h | 6 +-- 3 files changed, 25 insertions(+), 27 deletions(-) diff --git a/Code/Framework/AzCore/AzCore/DOM/Backends/JSON/JsonBackend.h b/Code/Framework/AzCore/AzCore/DOM/Backends/JSON/JsonBackend.h index 1c8f4eb607..af8e4e114c 100644 --- a/Code/Framework/AzCore/AzCore/DOM/Backends/JSON/JsonBackend.h +++ b/Code/Framework/AzCore/AzCore/DOM/Backends/JSON/JsonBackend.h @@ -35,7 +35,7 @@ namespace AZ::Dom Visitor::Result WriteToStream(AZ::IO::GenericStream& stream, WriteCallback callback) override { AZStd::unique_ptr visitor = Json::CreateJsonStreamWriter(stream, WriteFormat); - return callback(*visitor.get()); + return callback(*visitor); } }; } // namespace AZ::Dom diff --git a/Code/Framework/AzCore/AzCore/DOM/Backends/JSON/JsonSerializationUtils.cpp b/Code/Framework/AzCore/AzCore/DOM/Backends/JSON/JsonSerializationUtils.cpp index e1db5be29e..17f9bfea54 100644 --- a/Code/Framework/AzCore/AzCore/DOM/Backends/JSON/JsonSerializationUtils.cpp +++ b/Code/Framework/AzCore/AzCore/DOM/Backends/JSON/JsonSerializationUtils.cpp @@ -95,18 +95,19 @@ namespace AZ::Dom::Json return VisitorFailure(VisitorErrorCode::InternalError, "EndObject called without a matching BeginObject call"); } - if (!m_entryStack.front().m_isObject) + const ValueInfo& frontEntry = m_entryStack.front(); + if (!frontEntry.m_isObject) { return VisitorFailure(VisitorErrorCode::InternalError, "Expected EndArray and received EndObject instead"); } - if (m_entryStack.front().m_entryCount != attributeCount) + if (frontEntry.m_entryCount != attributeCount) { return VisitorFailure( VisitorErrorCode::InternalError, AZStd::string::format( "EndObject: Expected %llu attributes but received %llu attributes instead", attributeCount, - m_entryStack.front().m_entryCount)); + frontEntry.m_entryCount)); } m_entryStack.pop_front(); @@ -149,17 +150,18 @@ namespace AZ::Dom::Json return VisitorFailure(VisitorErrorCode::InternalError, "EndArray called without a matching BeginArray call"); } - if (m_entryStack.front().m_isObject) + const ValueInfo& frontEntry = m_entryStack.front(); + if (frontEntry.m_isObject) { return VisitorFailure(VisitorErrorCode::InternalError, "Expected EndObject and received EndArray instead"); } - if (m_entryStack.front().m_entryCount != elementCount) + if (frontEntry.m_entryCount != elementCount) { return VisitorFailure( VisitorErrorCode::InternalError, AZStd::string::format( - "EndArray: Expected %llu elements but received %llu elements instead", elementCount, m_entryStack.front().m_entryCount)); + "EndArray: Expected %llu elements but received %llu elements instead", elementCount, frontEntry.m_entryCount)); } m_entryStack.pop_front(); @@ -176,16 +178,17 @@ namespace AZ::Dom::Json // Retrieve the top value of the stack and replace it with a null value rapidjson::Value value; m_entryStack.front().m_value.Swap(value); - ++m_entryStack.front().m_entryCount; + ValueInfo& newEntry = m_entryStack.front(); + ++newEntry.m_entryCount; - if (m_entryStack.front().m_key.IsString()) + if (newEntry.m_key.IsString()) { - m_entryStack.front().m_container.AddMember(m_entryStack.front().m_key.Move(), AZStd::move(value), m_allocator); - m_entryStack.front().m_key.SetNull(); + newEntry.m_container.AddMember(m_entryStack.front().m_key.Move(), AZStd::move(value), m_allocator); + newEntry.m_key.SetNull(); } else { - m_entryStack.front().m_container.PushBack(AZStd::move(value), m_allocator); + newEntry.m_container.PushBack(AZStd::move(value), m_allocator); } return VisitorSuccess(); @@ -358,11 +361,7 @@ namespace AZ::Dom::Json bool RapidJsonReadHandler::String(const char* str, rapidjson::SizeType length, bool copy) { - Lifetime lifetime = m_stringLifetime; - if (!copy) - { - lifetime = Lifetime::Temporary; - } + const Lifetime lifetime = copy ? m_stringLifetime : Lifetime::Temporary; return CheckResult(m_visitor->String(AZStd::string_view(str, length), lifetime)); } @@ -378,11 +377,7 @@ namespace AZ::Dom::Json { m_visitor->Key(AZ::Name(key)); } - Lifetime lifetime = m_stringLifetime; - if (!copy) - { - lifetime = Lifetime::Temporary; - } + const Lifetime lifetime = copy ? m_stringLifetime : Lifetime::Temporary; return CheckResult(m_visitor->RawKey(key, lifetime)); } @@ -408,12 +403,15 @@ namespace AZ::Dom::Json bool RapidJsonReadHandler::CheckResult(Visitor::Result result) { - if (!result.IsSuccess()) + if (result.IsSuccess()) + { + return true; + } + else { m_outcome = AZStd::move(result); return false; } - return true; } // diff --git a/Code/Framework/AzCore/AzCore/DOM/Backends/JSON/JsonSerializationUtils.h b/Code/Framework/AzCore/AzCore/DOM/Backends/JSON/JsonSerializationUtils.h index db43395475..af0955dcfe 100644 --- a/Code/Framework/AzCore/AzCore/DOM/Backends/JSON/JsonSerializationUtils.h +++ b/Code/Framework/AzCore/AzCore/DOM/Backends/JSON/JsonSerializationUtils.h @@ -233,12 +233,12 @@ namespace AZ::Dom::Json if (buffer.data()[buffer.size()] == '\0') { NullDelimitedStringStream stream(buffer); - reader.Parse(parseFlags)>(stream, handler); + reader.Parse(parseFlags)>(stream, handler); } else { rapidjson::MemoryStream stream(buffer.data(), buffer.size()); - reader.Parse(parseFlags)>(stream, handler); + reader.Parse(parseFlags)>(stream, handler); } return handler.TakeOutcome(); } @@ -250,7 +250,7 @@ namespace AZ::Dom::Json NullDelimitedStringStream stream(buffer); RapidJsonReadHandler handler(&visitor, Lifetime::Persistent); - reader.Parse(parseFlags) | rapidjson::kParseInsituFlag>(stream, handler); + reader.Parse(parseFlags) | rapidjson::kParseInsituFlag>(stream, handler); return handler.TakeOutcome(); } } // namespace AZ::Dom::Json From ea9d7c9d82b400acc292947680a4ad744e7e0ab6 Mon Sep 17 00:00:00 2001 From: Nicholas Van Sickle Date: Mon, 6 Dec 2021 10:45:45 -0800 Subject: [PATCH 17/18] Add optional size parameter to ReadFromBufferInPlace Signed-off-by: Nicholas Van Sickle --- Code/Framework/AzCore/AzCore/DOM/Backends/JSON/JsonBackend.h | 2 +- Code/Framework/AzCore/AzCore/DOM/DomBackend.cpp | 4 ++-- Code/Framework/AzCore/AzCore/DOM/DomBackend.h | 3 ++- Code/Framework/AzCore/AzCore/DOM/DomUtils.cpp | 2 +- 4 files changed, 6 insertions(+), 5 deletions(-) diff --git a/Code/Framework/AzCore/AzCore/DOM/Backends/JSON/JsonBackend.h b/Code/Framework/AzCore/AzCore/DOM/Backends/JSON/JsonBackend.h index af8e4e114c..45825ff92d 100644 --- a/Code/Framework/AzCore/AzCore/DOM/Backends/JSON/JsonBackend.h +++ b/Code/Framework/AzCore/AzCore/DOM/Backends/JSON/JsonBackend.h @@ -27,7 +27,7 @@ namespace AZ::Dom return Json::VisitSerializedJson({ buffer, size }, lifetime, visitor); } - Visitor::Result ReadFromBufferInPlace(char* buffer, Visitor& visitor) override + Visitor::Result ReadFromBufferInPlace(char* buffer, [[maybe_unused]] AZStd::optional size, Visitor& visitor) override { return Json::VisitSerializedJsonInPlace(buffer, visitor); } diff --git a/Code/Framework/AzCore/AzCore/DOM/DomBackend.cpp b/Code/Framework/AzCore/AzCore/DOM/DomBackend.cpp index 8ff0e8a0bb..b29c0be231 100644 --- a/Code/Framework/AzCore/AzCore/DOM/DomBackend.cpp +++ b/Code/Framework/AzCore/AzCore/DOM/DomBackend.cpp @@ -10,8 +10,8 @@ namespace AZ::Dom { - Visitor::Result Backend::ReadFromBufferInPlace(char* buffer, Visitor& visitor) + Visitor::Result Backend::ReadFromBufferInPlace(char* buffer, AZStd::optional size, Visitor& visitor) { - return ReadFromBuffer(buffer, strlen(buffer), AZ::Dom::Lifetime::Persistent, visitor); + return ReadFromBuffer(buffer, size.value_or(strlen(buffer)), AZ::Dom::Lifetime::Persistent, visitor); } } // namespace AZ::Dom diff --git a/Code/Framework/AzCore/AzCore/DOM/DomBackend.h b/Code/Framework/AzCore/AzCore/DOM/DomBackend.h index c70af72652..6c9c40ef0e 100644 --- a/Code/Framework/AzCore/AzCore/DOM/DomBackend.h +++ b/Code/Framework/AzCore/AzCore/DOM/DomBackend.h @@ -12,6 +12,7 @@ #include #include #include +#include #include namespace AZ::Dom @@ -32,7 +33,7 @@ namespace AZ::Dom //! - The string won't be deallocated until the visitor no longer needs the values and //! - The string is safe to modify in place. //! The base implementation simply calls ReadFromBuffer. - virtual Visitor::Result ReadFromBufferInPlace(char* buffer, Visitor& visitor); + virtual Visitor::Result ReadFromBufferInPlace(char* buffer, AZStd::optional size, Visitor& visitor); //! A callback that accepts a Visitor, making DOM calls to inform the serializer, and returns an //! aggregate error code to indicate whether or not the operation succeeded. diff --git a/Code/Framework/AzCore/AzCore/DOM/DomUtils.cpp b/Code/Framework/AzCore/AzCore/DOM/DomUtils.cpp index 3c5d4eb668..ca27b177bd 100644 --- a/Code/Framework/AzCore/AzCore/DOM/DomUtils.cpp +++ b/Code/Framework/AzCore/AzCore/DOM/DomUtils.cpp @@ -19,7 +19,7 @@ namespace AZ::Dom::Utils Visitor::Result ReadFromStringInPlace(Backend& backend, AZStd::string& string, Visitor& visitor) { - return backend.ReadFromBufferInPlace(string.data(), visitor); + return backend.ReadFromBufferInPlace(string.data(), string.size(), visitor); } Visitor::Result WriteToString(Backend& backend, AZStd::string& buffer, Backend::WriteCallback writeCallback) From 6cf805256bebb03d726d7fe018cb890c03d04da7 Mon Sep 17 00:00:00 2001 From: Nicholas Van Sickle Date: Mon, 6 Dec 2021 15:55:35 -0800 Subject: [PATCH 18/18] Make backend write API string-based Signed-off-by: Nicholas Van Sickle --- .../Framework/AzCore/AzCore/DOM/Backends/JSON/JsonBackend.h | 4 +++- Code/Framework/AzCore/AzCore/DOM/DomBackend.h | 4 ++-- Code/Framework/AzCore/AzCore/DOM/DomUtils.cpp | 6 ------ Code/Framework/AzCore/AzCore/DOM/DomUtils.h | 2 -- Code/Framework/AzCore/Tests/DOM/DomJsonTests.cpp | 6 +++--- 5 files changed, 8 insertions(+), 14 deletions(-) diff --git a/Code/Framework/AzCore/AzCore/DOM/Backends/JSON/JsonBackend.h b/Code/Framework/AzCore/AzCore/DOM/Backends/JSON/JsonBackend.h index 45825ff92d..1023796f61 100644 --- a/Code/Framework/AzCore/AzCore/DOM/Backends/JSON/JsonBackend.h +++ b/Code/Framework/AzCore/AzCore/DOM/Backends/JSON/JsonBackend.h @@ -10,6 +10,7 @@ #include #include +#include namespace AZ::Dom { @@ -32,8 +33,9 @@ namespace AZ::Dom return Json::VisitSerializedJsonInPlace(buffer, visitor); } - Visitor::Result WriteToStream(AZ::IO::GenericStream& stream, WriteCallback callback) override + Visitor::Result WriteToBuffer(AZStd::string& buffer, WriteCallback callback) { + AZ::IO::ByteContainerStream stream{ &buffer }; AZStd::unique_ptr visitor = Json::CreateJsonStreamWriter(stream, WriteFormat); return callback(*visitor); } diff --git a/Code/Framework/AzCore/AzCore/DOM/DomBackend.h b/Code/Framework/AzCore/AzCore/DOM/DomBackend.h index 6c9c40ef0e..2e4ccf8f3e 100644 --- a/Code/Framework/AzCore/AzCore/DOM/DomBackend.h +++ b/Code/Framework/AzCore/AzCore/DOM/DomBackend.h @@ -38,7 +38,7 @@ namespace AZ::Dom //! A callback that accepts a Visitor, making DOM calls to inform the serializer, and returns an //! aggregate error code to indicate whether or not the operation succeeded. using WriteCallback = AZStd::function; - //! Attempt to write a value to a stream using a write callback. - virtual Visitor::Result WriteToStream(AZ::IO::GenericStream& stream, WriteCallback callback) = 0; + //! Attempt to write a value to the specified string using a write callback. + virtual Visitor::Result WriteToBuffer(AZStd::string& buffer, WriteCallback callback) = 0; }; } // namespace AZ::Dom diff --git a/Code/Framework/AzCore/AzCore/DOM/DomUtils.cpp b/Code/Framework/AzCore/AzCore/DOM/DomUtils.cpp index ca27b177bd..9751b9cecc 100644 --- a/Code/Framework/AzCore/AzCore/DOM/DomUtils.cpp +++ b/Code/Framework/AzCore/AzCore/DOM/DomUtils.cpp @@ -21,10 +21,4 @@ namespace AZ::Dom::Utils { return backend.ReadFromBufferInPlace(string.data(), string.size(), visitor); } - - Visitor::Result WriteToString(Backend& backend, AZStd::string& buffer, Backend::WriteCallback writeCallback) - { - AZ::IO::ByteContainerStream stream{ &buffer }; - return backend.WriteToStream(stream, writeCallback); - } } diff --git a/Code/Framework/AzCore/AzCore/DOM/DomUtils.h b/Code/Framework/AzCore/AzCore/DOM/DomUtils.h index cd64d2e107..84a6eb5687 100644 --- a/Code/Framework/AzCore/AzCore/DOM/DomUtils.h +++ b/Code/Framework/AzCore/AzCore/DOM/DomUtils.h @@ -14,6 +14,4 @@ namespace AZ::Dom::Utils { Visitor::Result ReadFromString(Backend& backend, AZStd::string_view string, AZ::Dom::Lifetime lifetime, Visitor& visitor); Visitor::Result ReadFromStringInPlace(Backend& backend, AZStd::string& string, Visitor& visitor); - - Visitor::Result WriteToString(Backend& backend, AZStd::string& buffer, Backend::WriteCallback writeCallback); } // namespace AZ::Dom::Utils diff --git a/Code/Framework/AzCore/Tests/DOM/DomJsonTests.cpp b/Code/Framework/AzCore/Tests/DOM/DomJsonTests.cpp index b408cd2466..c7af6438cf 100644 --- a/Code/Framework/AzCore/Tests/DOM/DomJsonTests.cpp +++ b/Code/Framework/AzCore/Tests/DOM/DomJsonTests.cpp @@ -71,7 +71,7 @@ namespace AZ::Dom::Tests { AZStd::string serializedDocument; JsonBackend backend; - auto result = Dom::Utils::WriteToString(backend, serializedDocument, visitDocumentFn); + auto result = backend.WriteToBuffer(serializedDocument, visitDocumentFn); EXPECT_TRUE(result.IsSuccess()); EXPECT_EQ(canonicalSerializedDocument, serializedDocument); } @@ -92,8 +92,8 @@ namespace AZ::Dom::Tests { AZStd::string serializedDocument; JsonBackend backend; - auto result = Dom::Utils::WriteToString( - backend, serializedDocument, + auto result = backend.WriteToBuffer( + serializedDocument, [&backend, &canonicalSerializedDocument](AZ::Dom::Visitor& visitor) { return Dom::Utils::ReadFromString(backend, canonicalSerializedDocument, Lifetime::Temporary, visitor);