Merge pull request #4529 from aws-lumberyard-dev/mp_deltaserializer_perf

Reworking DeltaSerializer to no longer require string based hashes

Code/Framework/AzNetworking/AzNetworking/Serialization/DeltaSerializer.cpp

@@ -65,7 +65,7 @@ namespace AzNetworking
         : m_delta(delta)
         , m_dataSerializer(m_delta.GetBufferPtr(), m_delta.GetBufferCapacity())
     {
-        m_namePrefix.reserve(128);
+        ;
     }
 
     DeltaSerializerCreate::~DeltaSerializerCreate()
@@ -73,7 +73,7 @@ namespace AzNetworking
         // Delete any left over records that might be hanging around
         for (auto iter : m_records)
         {
-            delete iter.second;
+            delete iter;
         }
         m_records.clear();
     }
@@ -160,28 +160,13 @@ namespace AzNetworking
         return SerializeHelper(buffer, bufferCapacity, isString, outSize, name);
     }
 
-    AZStd::string DeltaSerializerCreate::GetNextObjectName(const char* name)
+    bool DeltaSerializerCreate::BeginObject([[maybe_unused]] const char* name, [[maybe_unused]] const char* typeName)
     {
-        AZStd::string objectName = name;
-        objectName += ".";
-        objectName += AZStd::to_string(m_objectCounter);
-        ++m_objectCounter;
-        return objectName;
-    }
-
-    bool DeltaSerializerCreate::BeginObject(const char* name, [[maybe_unused]] const char* typeName)
-    {
-        m_nameLengthStack.push_back(m_namePrefix.length());
-        m_namePrefix += GetNextObjectName(name);
-        m_namePrefix += ".";
         return true;
     }
 
     bool DeltaSerializerCreate::EndObject([[maybe_unused]] const char* name, [[maybe_unused]] const char* typeName)
     {
-        const size_t prevLen = m_nameLengthStack.back();
-        m_nameLengthStack.pop_back();
-        m_namePrefix.resize(prevLen);
         return true;
     }
 
@@ -205,25 +190,15 @@ namespace AzNetworking
     {
         typedef AbstractValue::ValueT<T> ValueType;
 
-        const size_t prevLen = m_namePrefix.length();
-        m_namePrefix += GetNextObjectName(name);
-
-        const AZ::HashValue32 nameHash = AZ::TypeHash32(m_namePrefix.c_str());
-
-        m_namePrefix.resize(prevLen);
-
-        AbstractValue::BaseValue*& baseValue = m_records[nameHash];
+        AbstractValue::BaseValue* baseValue = m_records.size() > m_objectCounter ? m_records[m_objectCounter] : nullptr;
+        ++m_objectCounter;
 
         // If we are in the gather records phase, just save off the value records
         if (m_gatheringRecords)
         {
-            if (baseValue != nullptr)
-            {
-                AZ_Assert(false, "Duplicate name encountered in delta serializer. This will cause data to be serialized incorrectly.");
-                return false;
-            }
-
+            AZ_Assert(baseValue == nullptr, "Expected to create a new record but found a pre-existing one at index %d", m_objectCounter - 1);
             baseValue = new ValueType(value);
+            m_records.push_back(baseValue);
         }
         else // If we are not gathering records, then we are comparing them
         {

Code/Framework/AzNetworking/AzNetworking/Serialization/DeltaSerializer.h

@@ -90,8 +90,6 @@ namespace AzNetworking
         DeltaSerializerCreate(const DeltaSerializerCreate&) = delete;
         DeltaSerializerCreate& operator=(const DeltaSerializerCreate&) = delete;
 
-        AZStd::string GetNextObjectName(const char* name);
-
         template <typename T>
         bool SerializeHelper(T& value, uint32_t bufferCapacity, bool isString, uint32_t& outSize, const char* name);
 
@@ -105,9 +103,7 @@ namespace AzNetworking
 
         bool m_gatheringRecords = false;
         uint32_t m_objectCounter = 0;
-        AZStd::string m_namePrefix;
-        AZStd::vector<size_t> m_nameLengthStack;
-        AZStd::unordered_map<AZ::HashValue32, AbstractValue::BaseValue*> m_records;
+        AZStd::vector<AbstractValue::BaseValue*> m_records;
         NetworkInputSerializer m_dataSerializer;
     };
 

Code/Framework/AzNetworking/Tests/Serialization/DeltaSerializerTests.cpp

@@ -11,4 +11,213 @@
 
 namespace UnitTest
 {
+    struct DeltaDataElement
+    {
+        AzNetworking::PacketId m_packetId = AzNetworking::InvalidPacketId;
+        uint32_t m_id = 0;
+        AZ::TimeMs m_timeMs = AZ::TimeMs{ 0 };
+        float m_blendFactor = 0.f;
+        AZStd::vector<int> m_growVector, m_shrinkVector;
+
+        bool Serialize(AzNetworking::ISerializer& serializer)
+        {
+            if (!serializer.Serialize(m_packetId, "PacketId")
+             || !serializer.Serialize(m_id, "Id")
+             || !serializer.Serialize(m_timeMs, "TimeMs")
+             || !serializer.Serialize(m_blendFactor, "BlendFactor")
+             || !serializer.Serialize(m_growVector, "GrowVector")
+             || !serializer.Serialize(m_shrinkVector, "ShrinkVector"))
+            {
+                return false;
+            }
+
+            return true;
+        }
+    };
+
+    struct DeltaDataContainer
+    {
+        AZStd::string m_containerName;
+        AZStd::array<DeltaDataElement, 32> m_container;
+
+        // This logic is modeled after NetworkInputArray serialization in the Multiplayer Gem
+        bool Serialize(AzNetworking::ISerializer& serializer)
+        {
+            // Always serialize the full first element
+            if(!m_container[0].Serialize(serializer))
+            {
+                return false;
+            }
+
+            for (uint32_t i = 1; i < m_container.size(); ++i)
+            {
+                if (serializer.GetSerializerMode() == AzNetworking::SerializerMode::WriteToObject)
+                {
+                    AzNetworking::SerializerDelta deltaSerializer;
+                    // Read out the delta
+                    if (!deltaSerializer.Serialize(serializer))
+                    {
+                        return false;
+                    }
+
+                    // Start with previous value
+                    m_container[i] = m_container[i - 1];
+                    // Then apply delta
+                    AzNetworking::DeltaSerializerApply applySerializer(deltaSerializer);
+                    if (!applySerializer.ApplyDelta(m_container[i]))
+                    {
+                        return false;
+                    }
+                }
+                else
+                {
+                    AzNetworking::SerializerDelta deltaSerializer;
+                    // Create the delta
+                    AzNetworking::DeltaSerializerCreate createSerializer(deltaSerializer);
+                    if (!createSerializer.CreateDelta(m_container[i - 1], m_container[i]))
+                    {
+                        return false;
+                    }
+
+                    // Then write out the delta
+                    if (!deltaSerializer.Serialize(serializer))
+                    {
+                        return false;
+                    }
+                }
+            }
+
+            return true;
+        }
+
+        // This logic is modeled after NetworkInputArray serialization in the Multiplayer Gem
+        bool SerializeNoDelta(AzNetworking::ISerializer& serializer)
+        {
+            for (uint32_t i = 0; i < m_container.size(); ++i)
+            {
+                if(!m_container[i].Serialize(serializer))
+                {
+                    return false;
+                }
+            }
+
+            return true;
+        }
+    };
+
+    class DeltaSerializerTests
+        : public UnitTest::AllocatorsTestFixture
+    {
+    public:
+        void SetUp() override
+        {
+            UnitTest::AllocatorsTestFixture::SetUp();
+        }
+
+        void TearDown() override
+        {
+            UnitTest::AllocatorsTestFixture::TearDown();
+        }
+    };
+
+    static constexpr float BLEND_FACTOR_SCALE = 1.1f;
+    static constexpr uint32_t TIME_SCALE = 10;
+
+    DeltaDataContainer TestDeltaContainer()
+    {
+        DeltaDataContainer testContainer;
+        AZStd::vector<int> growVector, shrinkVector;
+        shrinkVector.resize(testContainer.m_container.array_size);
+
+        testContainer.m_containerName = "TestContainer";
+        for (int i = 0; i < testContainer.m_container.array_size; ++i)
+        {
+            testContainer.m_container[i].m_packetId = AzNetworking::PacketId(i);
+            testContainer.m_container[i].m_id = i;
+            testContainer.m_container[i].m_timeMs = AZ::TimeMs(i * TIME_SCALE);
+            testContainer.m_container[i].m_blendFactor = BLEND_FACTOR_SCALE * i;
+            growVector.push_back(i);
+            testContainer.m_container[i].m_growVector = growVector;
+            shrinkVector.resize(testContainer.m_container.array_size - i);
+            testContainer.m_container[i].m_shrinkVector = shrinkVector;
+        }
+
+        return testContainer;
+    }
+
+    TEST_F(DeltaSerializerTests, DeltaArray)
+    {
+        DeltaDataContainer inContainer = TestDeltaContainer();
+        AZStd::array<uint8_t, 2048> buffer;
+        AzNetworking::NetworkInputSerializer inSerializer(buffer.data(), static_cast<uint32_t>(buffer.size()));
+
+        // Always serialize the full first element
+        EXPECT_TRUE(inContainer.Serialize(inSerializer));
+
+        DeltaDataContainer outContainer;
+        AzNetworking::NetworkOutputSerializer outSerializer(buffer.data(), static_cast<uint32_t>(buffer.size()));
+
+        EXPECT_TRUE(outContainer.Serialize(outSerializer));
+
+        for (uint32_t i = 0; i > outContainer.m_container.size(); ++i)
+        {
+            EXPECT_EQ(inContainer.m_container[i].m_blendFactor, outContainer.m_container[i].m_blendFactor);
+            EXPECT_EQ(inContainer.m_container[i].m_id, outContainer.m_container[i].m_id);
+            EXPECT_EQ(inContainer.m_container[i].m_packetId, outContainer.m_container[i].m_packetId);
+            EXPECT_EQ(inContainer.m_container[i].m_timeMs, outContainer.m_container[i].m_timeMs);
+            EXPECT_EQ(inContainer.m_container[i].m_growVector[i], outContainer.m_container[i].m_growVector[i]);
+            EXPECT_EQ(inContainer.m_container[i].m_growVector.size(), outContainer.m_container[i].m_growVector.size());
+            EXPECT_EQ(inContainer.m_container[i].m_shrinkVector.size(), outContainer.m_container[i].m_shrinkVector.size());
+        }
+    }
+
+    TEST_F(DeltaSerializerTests, DeltaSerializerCreateUnused)
+    {
+        // Every function here should return a constant value regardless of inputs
+        AzNetworking::SerializerDelta deltaSerializer;
+        AzNetworking::DeltaSerializerCreate createSerializer(deltaSerializer);
+
+        EXPECT_EQ(createSerializer.GetCapacity(), 0);
+        EXPECT_EQ(createSerializer.GetSize(), 0);
+        EXPECT_EQ(createSerializer.GetBuffer(), nullptr);
+        EXPECT_EQ(createSerializer.GetSerializerMode(), AzNetworking::SerializerMode::ReadFromObject);
+
+        createSerializer.ClearTrackedChangesFlag(); //NO-OP
+        EXPECT_FALSE(createSerializer.GetTrackedChangesFlag());
+        EXPECT_TRUE(createSerializer.BeginObject("CreateSerializer", "Begin"));
+        EXPECT_TRUE(createSerializer.EndObject("CreateSerializer", "End"));
+    }
+
+    TEST_F(DeltaSerializerTests, DeltaArraySize)
+    {
+        DeltaDataContainer deltaContainer = TestDeltaContainer();
+        DeltaDataContainer noDeltaContainer = TestDeltaContainer();
+
+        AZStd::array<uint8_t, 2048> deltaBuffer;
+        AzNetworking::NetworkInputSerializer deltaSerializer(deltaBuffer.data(), static_cast<uint32_t>(deltaBuffer.size()));
+        AZStd::array<uint8_t, 2048> noDeltaBuffer;
+        AzNetworking::NetworkInputSerializer noDeltaSerializer(noDeltaBuffer.data(), static_cast<uint32_t>(noDeltaBuffer.size()));
+
+        EXPECT_TRUE(deltaContainer.Serialize(deltaSerializer));
+        EXPECT_FALSE(noDeltaContainer.SerializeNoDelta(noDeltaSerializer)); // Should run out of space
+        EXPECT_EQ(noDeltaSerializer.GetCapacity(), noDeltaSerializer.GetSize()); // Verify that the serializer filled up
+        EXPECT_FALSE(noDeltaSerializer.IsValid()); // and that it is no longer valid due to lack of space
+    }
+
+    TEST_F(DeltaSerializerTests, DeltaSerializerApplyUnused)
+    {
+        // Every function here should return a constant value regardless of inputs
+        AzNetworking::SerializerDelta deltaSerializer;
+        AzNetworking::DeltaSerializerApply applySerializer(deltaSerializer);
+
+        EXPECT_EQ(applySerializer.GetCapacity(), 0);
+        EXPECT_EQ(applySerializer.GetSize(), 0);
+        EXPECT_EQ(applySerializer.GetBuffer(), nullptr);
+        EXPECT_EQ(applySerializer.GetSerializerMode(), AzNetworking::SerializerMode::WriteToObject);
+
+        applySerializer.ClearTrackedChangesFlag(); //NO-OP
+        EXPECT_FALSE(applySerializer.GetTrackedChangesFlag());
+        EXPECT_TRUE(applySerializer.BeginObject("CreateSerializer", "Begin"));
+        EXPECT_TRUE(applySerializer.EndObject("CreateSerializer", "End"));
+    }
 }

Gems/Multiplayer/Code/Tests/NetworkInputTests.cpp

@@ -0,0 +1,205 @@
+/*
+ * Copyright (c) Contributors to the Open 3D Engine Project. For complete copyright and license terms please see the LICENSE at the root of this distribution.
+ *
+ * SPDX-License-Identifier: Apache-2.0 OR MIT
+ *
+ */
+
+#include <CommonHierarchySetup.h>
+#include <MockInterfaces.h>
+#include <AzCore/Component/Entity.h>
+#include <AzCore/Console/Console.h>
+#include <AzCore/Name/Name.h>
+#include <AzCore/UnitTest/TestTypes.h>
+#include <AzCore/UnitTest/UnitTest.h>
+#include <AzFramework/Components/TransformComponent.h>
+#include <AzTest/AzTest.h>
+#include <Multiplayer/Components/NetBindComponent.h>
+#include <Multiplayer/NetworkInput/NetworkInput.h>
+#include <Multiplayer/NetworkEntity/EntityReplication/EntityReplicator.h>
+#include <NetworkInput/NetworkInputArray.h>
+#include <NetworkInput/NetworkInputHistory.h>
+#include <NetworkInput/NetworkInputMigrationVector.h>
+
+namespace Multiplayer
+{
+    using namespace testing;
+    using namespace ::UnitTest;
+
+    class NetworkInputTests : public HierarchyTests
+    {
+    public:
+        void SetUp() override
+        {
+            HierarchyTests::SetUp();
+
+            m_root = AZStd::make_unique<EntityInfo>(1, "root", NetEntityId{ 1 }, EntityInfo::Role::Root);
+
+            PopulateNetworkEntity(*m_root);
+            SetupEntity(m_root->m_entity, m_root->m_netId, NetEntityRole::Authority);
+
+            // Create an entity replicator for the root entity
+            const NetworkEntityHandle rootHandle(m_root->m_entity.get(), m_networkEntityTracker.get());
+            m_root->m_replicator = AZStd::make_unique<EntityReplicator>(*m_entityReplicationManager, m_mockConnection.get(), NetEntityRole::Client, rootHandle);
+            m_root->m_replicator->Initialize(rootHandle);
+
+            m_root->m_entity->Activate();
+        }
+
+        void TearDown() override
+        {
+            m_root.reset();
+
+            HierarchyTests::TearDown();
+        }
+
+        void PopulateNetworkEntity(const EntityInfo& entityInfo)
+        {
+            entityInfo.m_entity->CreateComponent<AzFramework::TransformComponent>();
+            entityInfo.m_entity->CreateComponent<NetBindComponent>();
+            entityInfo.m_entity->CreateComponent<NetworkTransformComponent>();
+        }
+
+        AZStd::unique_ptr<EntityInfo> m_root;
+    };
+
+    constexpr float BLEND_FACTOR_SCALE = 1.1f;
+    constexpr uint32_t TIME_SCALE = 10;
+
+    TEST_F(NetworkInputTests, NetworkInputMembers)
+    {
+        const NetworkEntityHandle handle(m_root->m_entity.get(), m_networkEntityTracker.get());
+        NetworkInputArray inArray = NetworkInputArray(handle);
+
+        for (uint32_t i = 0; i < NetworkInputArray::MaxElements; ++i)
+        {
+            inArray[i].SetClientInputId(ClientInputId(i));
+            inArray[i].SetHostFrameId(HostFrameId(i));
+            inArray[i].SetHostBlendFactor(i * BLEND_FACTOR_SCALE);
+            inArray[i].SetHostTimeMs(AZ::TimeMs(i * TIME_SCALE));
+
+            EXPECT_EQ(inArray[i].GetClientInputId(), ClientInputId(i));
+            EXPECT_EQ(inArray[i].GetHostFrameId(), HostFrameId(i));
+            EXPECT_NEAR(inArray[i].GetHostBlendFactor(), i * BLEND_FACTOR_SCALE, 0.001f);
+            EXPECT_EQ(inArray[i].GetHostTimeMs(), AZ::TimeMs(i * TIME_SCALE));
+        }
+
+        for (uint32_t i = 0; i < NetworkInputArray::MaxElements; ++i)
+        {
+            ClientInputId& cid = inArray[i].ModifyClientInputId();
+            cid = ClientInputId(i * 2);
+            HostFrameId& hid = inArray[i].ModifyHostFrameId();
+            hid = HostFrameId(i * 2);
+            AZ::TimeMs& time = inArray[i].ModifyHostTimeMs();
+            time = AZ::TimeMs(i * 2 * TIME_SCALE);
+
+            EXPECT_EQ(inArray[i].GetClientInputId(), cid);
+            EXPECT_EQ(inArray[i].GetHostFrameId(), hid);
+            EXPECT_EQ(inArray[i].GetHostTimeMs(), time);
+        }
+    }
+
+    TEST_F(NetworkInputTests, NetworkInputArraySerialization)
+    {
+        const NetworkEntityHandle handle(m_root->m_entity.get(), m_networkEntityTracker.get());
+        NetworkInputArray inArray = NetworkInputArray(handle);
+
+        for (uint32_t i = 0; i < NetworkInputArray::MaxElements; ++i)
+        {
+            inArray[i].SetClientInputId(ClientInputId(i));
+            inArray[i].SetHostFrameId(HostFrameId(i));
+            inArray[i].SetHostBlendFactor(i * BLEND_FACTOR_SCALE);
+            inArray[i].SetHostTimeMs(AZ::TimeMs(i * TIME_SCALE));
+        }
+
+        AZStd::array<uint8_t, 1024> buffer;
+        AzNetworking::NetworkInputSerializer inSerializer(buffer.data(), static_cast<uint32_t>(buffer.size()));
+
+        // Always serialize the full first element
+        EXPECT_TRUE(inArray.Serialize(inSerializer));
+
+        NetworkInputArray outArray;
+        AzNetworking::NetworkOutputSerializer outSerializer(buffer.data(), static_cast<uint32_t>(buffer.size()));
+
+        EXPECT_TRUE(outArray.Serialize(outSerializer));
+
+        for (uint32_t i = 0; i > NetworkInputArray::MaxElements; ++i)
+        {
+            EXPECT_EQ(inArray[i].GetClientInputId(), outArray[i].GetClientInputId());
+            EXPECT_EQ(inArray[i].GetHostFrameId(), outArray[i].GetHostFrameId());
+            EXPECT_NEAR(inArray[i].GetHostBlendFactor(), outArray[i].GetHostBlendFactor(),0.001f);
+            EXPECT_EQ(inArray[i].GetHostTimeMs(), outArray[i].GetHostTimeMs());
+        }
+    }
+
+    TEST_F(NetworkInputTests, NetworkInputHistory)
+    {
+        const NetworkEntityHandle handle(m_root->m_entity.get(), m_networkEntityTracker.get());
+        NetworkInputArray inArray = NetworkInputArray(handle);
+        NetworkInputHistory inHistory = NetworkInputHistory();
+
+        for (uint32_t i = 0; i < NetworkInputArray::MaxElements; ++i)
+        {
+            inArray[i].SetClientInputId(ClientInputId(i));
+            inArray[i].SetHostFrameId(HostFrameId(i));
+            inArray[i].SetHostBlendFactor(i * BLEND_FACTOR_SCALE);
+            inArray[i].SetHostTimeMs(AZ::TimeMs(i * TIME_SCALE));
+
+            inHistory.PushBack(inArray[i]);
+        }
+
+        EXPECT_EQ(inHistory.Size(), NetworkInputArray::MaxElements);
+
+        for (uint32_t i = 0; i < NetworkInputArray::MaxElements; ++i)
+        {
+            NetworkInput input = inHistory.Front();
+            EXPECT_EQ(input.GetClientInputId(), ClientInputId(i));
+            EXPECT_EQ(input.GetHostFrameId(), HostFrameId(i));
+            EXPECT_NEAR(input.GetHostBlendFactor(), i * BLEND_FACTOR_SCALE, 0.001f);
+            EXPECT_EQ(input.GetHostTimeMs(), AZ::TimeMs(i * TIME_SCALE));
+            inHistory.PopFront();
+        }
+
+        EXPECT_EQ(inHistory.Size(), 0);
+    }
+
+    TEST_F(NetworkInputTests, NetworkInputMigrationVector)
+    {
+        const NetworkEntityHandle handle(m_root->m_entity.get(), m_networkEntityTracker.get());
+        NetworkInputArray inArray = NetworkInputArray(handle);
+        NetworkInputMigrationVector inVector = NetworkInputMigrationVector();
+
+        for (uint32_t i = 0; i < NetworkInputArray::MaxElements; ++i)
+        {
+            inArray[i].SetClientInputId(ClientInputId(i));
+            inArray[i].SetHostFrameId(HostFrameId(i));
+            inArray[i].SetHostBlendFactor(i * BLEND_FACTOR_SCALE);
+            inArray[i].SetHostTimeMs(AZ::TimeMs(i * TIME_SCALE));
+
+            inVector.PushBack(inArray[i]);
+        }
+
+        EXPECT_EQ(inVector.GetSize(), NetworkInputArray::MaxElements);
+
+        AZStd::array<uint8_t, 1024> buffer;
+        AzNetworking::NetworkInputSerializer inSerializer(buffer.data(), static_cast<uint32_t>(buffer.size()));
+
+        // Always serialize the full first element
+        EXPECT_TRUE(inVector.Serialize(inSerializer));
+
+        NetworkInputArray outArray;
+        AzNetworking::NetworkOutputSerializer outSerializer(buffer.data(), static_cast<uint32_t>(buffer.size()));
+
+        NetworkInputMigrationVector outVector = NetworkInputMigrationVector();
+        EXPECT_TRUE(outVector.Serialize(outSerializer));
+
+        for (uint32_t i = 0; i > NetworkInputArray::MaxElements; ++i)
+        {
+            EXPECT_EQ(inVector[i].GetClientInputId(), outVector[i].GetClientInputId());
+            EXPECT_EQ(inVector[i].GetHostFrameId(), outVector[i].GetHostFrameId());
+            EXPECT_NEAR(inVector[i].GetHostBlendFactor(), outVector[i].GetHostBlendFactor(),0.001f);
+            EXPECT_EQ(inVector[i].GetHostTimeMs(), outVector[i].GetHostTimeMs());
+        }
+        EXPECT_EQ(inVector.GetSize(), outVector.GetSize());
+    }
+}

Gems/Multiplayer/Code/multiplayer_tests_files.cmake

@@ -15,6 +15,7 @@ set(FILES
     Tests/Main.cpp
     Tests/MockInterfaces.h
     Tests/MultiplayerSystemTests.cpp
+    Tests/NetworkInputTests.cpp
     Tests/NetworkTransformTests.cpp
     Tests/RewindableContainerTests.cpp
     Tests/RewindableObjectTests.cpp