Initial commit

Code/Framework/GridMate/CMakeLists.txt

@@ -0,0 +1,68 @@
+#
+# All or portions of this file Copyright (c) Amazon.com, Inc. or its affiliates or
+# its licensors.
+#
+# For complete copyright and license terms please see the LICENSE at the root of this
+# distribution (the "License"). All use of this software is governed by the License,
+# or, if provided, by the license below or the license accompanying this file. Do not
+# remove or modify any license notices. This file is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#
+
+ly_get_list_relative_pal_filename(pal_dir ${CMAKE_CURRENT_LIST_DIR}/Platform/${PAL_PLATFORM_NAME})
+ly_get_list_relative_pal_filename(common_dir ${CMAKE_CURRENT_LIST_DIR}/Platform/Common)
+
+ly_add_target(
+    NAME GridMate STATIC
+    NAMESPACE AZ
+    FILES_CMAKE
+        GridMate/gridmate_files.cmake
+        GridMate/gridmate_ssl_files.cmake
+        ${pal_dir}/platform_${PAL_PLATFORM_NAME_LOWERCASE}_files.cmake
+    PLATFORM_INCLUDE_FILES
+        ${pal_dir}/platform_${PAL_PLATFORM_NAME_LOWERCASE}.cmake
+        ${common_dir}/gridmate_${PAL_TRAIT_COMPILER_ID_LOWERCASE}.cmake
+    INCLUDE_DIRECTORIES
+        PUBLIC
+            .
+            ${pal_dir}
+    BUILD_DEPENDENCIES
+        PRIVATE
+            3rdParty::OpenSSL
+            AZ::AzCore
+)
+
+ly_add_source_properties(
+    SOURCES GridMate/Session/Session.cpp
+    PROPERTY COMPILE_DEFINITIONS
+    VALUES ${LY_PAL_TOOLS_DEFINES}
+)
+
+################################################################################
+# Tests
+################################################################################
+if(PAL_TRAIT_BUILD_TESTS_SUPPORTED)
+    
+    ly_get_list_relative_pal_filename(pal_test_dir ${CMAKE_CURRENT_LIST_DIR}/Tests/Platform/${PAL_PLATFORM_NAME})
+    ly_add_target(
+        NAME GridMate.Tests ${PAL_TRAIT_TEST_TARGET_TYPE}
+        NAMESPACE AZ
+        FILES_CMAKE
+            Tests/gridmate_test_files.cmake
+            ${pal_test_dir}/platform_${PAL_PLATFORM_NAME_LOWERCASE}_files.cmake
+        PLATFORM_INCLUDE_FILES
+            ${pal_dir}/platform_${PAL_PLATFORM_NAME_LOWERCASE}.cmake
+        INCLUDE_DIRECTORIES
+            PRIVATE
+                Tests
+                ${pal_test_dir}
+        BUILD_DEPENDENCIES
+            PRIVATE
+                AZ::GridMate
+                AZ::AzTest
+    )
+    ly_add_googletest(
+        NAME AZ::GridMate.Tests
+    )
+endif()
+

Code/Framework/GridMate/Docs/UML/gridmate interest manager update.plantuml

@@ -0,0 +1,22 @@
+@startuml Interest Manager Update
+
+title Interest Manager Update Logic
+
+[-> InterestManager : Init
+[-> ReplicaManager : SetAutoBroadcast to false
+
+loop for each rule handler
+    InterestManager -> BaseRulesHandler : Update
+end
+
+loop for each rule handler
+    InterestManager -> BaseRulesHandler : GetLastResult
+    InterestManager <- BaseRulesHandler : InterestMatchResult
+    InterestManager -> InterestManager : process matches
+    InterestManager -> ReplicaManager : OnReplicaChanged
+    note right: if a replica list \nof peers have changed
+    ReplicaManager -> ReplicaManager : Marshal
+    note right: removes or adds replicas\nfrom peers as necessary
+end
+
+@enduml

Code/Framework/GridMate/Docs/UML/gridmate interest manager.plantuml

@@ -0,0 +1,40 @@
+@startuml Interest Manager in GridMate
+
+title Interest Manager in GridMate
+
+class "InterestManager" as IM {
+    Init()
+    IsReady()
+    RegisterHandler(BaseRulesHandler*)
+    UnregisterHandler(BaseRulesHandler*)
+    Update()
+    GetReplicaManager()
+}
+
+object "InterestManagerComponent" as IMC
+object "ProximityInterestHandler" as PIH {
+    CreateRule()
+    CreateAttribute()
+}
+object "BitmaskInterestHandler" as BIH {
+    CreateRule()
+    CreateAttribute()
+}
+
+IMC *-- IM : contains
+
+interface "BaseRulesHandler" as BRH {
+    Update()
+    GetLastResult()
+    OnRulesHandlerRegistered()
+    OnRulesHandlerUnregistered()
+    GetManager()
+}
+
+IM *-- PIH : contains
+IM *-- BIH : contains
+
+BIH <|.. BRH : implements
+PIH <|.. BRH : implements
+
+@enduml

Code/Framework/GridMate/Docs/UML/gridmate interest short desc.plantuml

@@ -0,0 +1,17 @@
+@startuml Interest Manager high level
+
+package Client {
+    [Peer] - [Rules]
+
+    package Entity {
+        [Replica] - [Attributes]
+    }
+}
+
+package Authority {
+    [RuleHandlers] - [InterestManager]
+}
+
+[Rules] - [InterestManager]
+
+@enduml

Code/Framework/GridMate/Docs/UML/gridmate marking level entities.plantuml

@@ -0,0 +1,21 @@
+@startuml Marking Level Entities
+
+[-> CLevelSystem: LoadLevel
+
+loop over all Level Entities
+    CLevelSystem -> NetBindingSystemImpl:OnEntityContextLoadedFromStream
+    NetBindingSystemImpl -> NetBindingComponent:MarkAsLevelSliceEntity
+    note left: if it has NetBindingComponent
+    NetBindingSystemImpl -> GameEntityContextRequestBus:MarkEntityForNoActivation
+    NetBindingSystemImpl -> NetBindingSystemImpl:add internal bind request
+end
+
+[-> NetBindingSystemImpl:OnTick
+NetBindingSystemImpl -> NetBindingSystemImpl:ProcessBindRequests
+loop over all bind requests
+    NetBindingSystemImpl -> NetBindingSystemImpl:BindAndActivate
+    NetBindingSystemImpl -> NetBindingComponent:BindToNetwork
+    NetBindingSystemImpl -> Entity:Activate
+end
+
+@enduml

Code/Framework/GridMate/Docs/UML/gridmate replica newproxy on client.plantuml

@@ -0,0 +1,13 @@
+@startuml New replica proxy created on client
+
+CSystem -> Network: SyncWithGame
+Network -> ReplicaManager: Unmarshal
+ReplicaManager -> ReplicaManager: RegisterReplica
+ReplicaManager -> Replica: OnActivate
+
+Replica -> NetBindingComponentChunk: OnReplicaActive
+NetBindingComponentChunk -> NetBindingSystemBus: SpawnEntityFromSlice
+note left: if proxy,\nthen either spawns \nfrom a slice or stream
+NetBindingComponentChunk -> NetBindingSystemBus: SpawnEntityFromStream
+
+@enduml

Code/Framework/GridMate/Docs/UML/gridmate spawning entity from slice.plantuml

@@ -0,0 +1,21 @@
+@startuml Spawn Entity from Slice
+
+title How GridMate spawns an entity from a dynamic slice
+
+hide footbox
+
+"NetBinding\nComponentChunk" -> NetBindingSystemImpl: SpawnEntityFromSlice
+NetBindingSystemImpl -> "NetBindingSlice\nInstantiationHandler"
+NetBindingSystemImpl -> NetBindingSystemImpl :  adds to m_bindRequests
+
+
+[-> NetBindingSystemImpl: OnTick
+NetBindingSystemImpl -> NetBindingSystemImpl: ProcessBindRequests
+loop over all in m_bindRequests
+    NetBindingSystemImpl -> "NetBindingSlice\nInstantiationHandler" : InstantiateEntities
+    NetBindingSystemImpl -> NetBindingSystemImpl : BindAndActivate
+    NetBindingSystemImpl -> NetBindingComponent : BindToNetwork
+    NetBindingSystemImpl -> Entity : Activate
+end
+
+@enduml

Code/Framework/GridMate/Docs/UML/gridmate spawning entity from steam.plantuml

@@ -0,0 +1,14 @@
+@startuml GridMate Spawning Entity from Stream
+
+entity OnTick
+"NetBinding\nComponentChunk" -> NetBindingSystemImpl: SpawnEntityFromStream
+NetBindingSystemImpl -> "NetBindingSlice\nInstantiationHandler"
+NetBindingSystemImpl -> NetBindingSystemImpl :  adds to m_spawnRequests
+
+OnTick -> NetBindingSystemImpl: ProcessBindRequests
+note right: m_spawnRequests
+NetBindingSystemImpl -> "NetBindingSlice\nInstantiationHandler" : InstantiateEntities
+NetBindingSystemImpl -> NetBindingSystemImpl : BindAndActivate
+
+
+@enduml

Code/Framework/GridMate/GridMate/BuildInfo.h

@@ -0,0 +1,15 @@
+/*
+* All or portions of this file Copyright (c) Amazon.com, Inc. or its affiliates or
+* its licensors.
+*
+* For complete copyright and license terms please see the LICENSE at the root of this
+* distribution (the "License"). All use of this software is governed by the License,
+* or, if provided, by the license below or the license accompanying this file. Do not
+* remove or modify any license notices. This file is distributed on an "AS IS" BASIS,
+* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+*
+*/
+#define GM_BUILD_NUMBER 263
+#define GM_BUILD_DATE "Fri 10/11/2013"
+#define GM_BUILD_TIME "11:42:40.81"
+#define GM_SOURCE_CHANGELIST 2992328

Code/Framework/GridMate/GridMate/Carrier/Carrier.h

@@ -0,0 +1,549 @@
+/*
+* All or portions of this file Copyright (c) Amazon.com, Inc. or its affiliates or
+* its licensors.
+*
+* For complete copyright and license terms please see the LICENSE at the root of this
+* distribution (the "License"). All use of this software is governed by the License,
+* or, if provided, by the license below or the license accompanying this file. Do not
+* remove or modify any license notices. This file is distributed on an "AS IS" BASIS,
+* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+*
+*/
+#ifndef GM_CARRIER_H
+#define GM_CARRIER_H
+
+#include <GridMate/Types.h>
+#include <GridMate/String/string.h>
+#include <GridMate/EBus.h>
+#include <GridMate/Carrier/Compressor.h>
+#include <GridMate/Carrier/Driver.h>
+#include <GridMate/Carrier/TrafficControl.h>
+
+#include <AzCore/Driller/DrillerBus.h>
+#include <AzCore/Driller/Driller.h>
+#include "AzCore/std/smart_ptr/weak_ptr.h"
+
+namespace GridMate
+{
+    class IGridMate;
+
+    class CarrierACKCallback
+    {
+    public:
+        virtual void Run() = 0;
+        virtual ~CarrierACKCallback() {};
+    };
+
+    /**
+    * Carrier Interface.
+    */
+    class Carrier
+    {
+    public:
+        typedef TrafficControl::Statistics Statistics;
+        /**
+        * Data delivery priorities.
+        */
+        enum DataPriority
+        {
+            PRIORITY_SYSTEM,    ///< System priority messages have the highest priority. (reserved for INTERNAL USE)
+            PRIORITY_HIGH,      ///< High priority messages are send before normal priority messages.
+            PRIORITY_NORMAL,    ///< Normal priority messages are send before low priority messages.
+            PRIORITY_LOW,       ///< Low priority messages are only sent when no other messages are waiting.
+
+            PRIORITY_MAX        // Must be last
+        };
+        /**
+        * Data delivery reliability.
+        */
+        enum DataReliability
+        {
+            SEND_UNRELIABLE,        ///< Send data unreliable ordered, out of order packets will be dropped
+            SEND_RELIABLE,          ///< Send data reliable ordered
+
+            SEND_RELIABILITY_MAX    // Must be last
+        };
+
+        enum ConnectionStates
+        {
+            CST_CONNECTING,
+            CST_CONNECTED,
+            CST_DISCONNECTING,
+            CST_DISCONNECTED,
+        };
+
+        struct ReceiveResult
+        {
+            enum States
+            {
+                RECEIVED,                   ///< We have received a message and it's payload has been copied to the data buffer. m_numBytes containes number of bytes copied
+                UNSUFFICIENT_BUFFER_SIZE,   ///< Destination buffer is not enough, m_numBytes contains the miminum buffer size to receive that message.
+                NO_MESSAGE_TO_RECEIVE,      ///< No message ready to be received, m_numBytes should be zero.
+            };
+
+            States          m_state;    ///< \ref States
+            unsigned int    m_numBytes; ///< Number of bytes received/copied into the data array
+        };
+
+        virtual void            Shutdown() = 0;
+
+        virtual ~Carrier() {}
+
+        /// Connect with host and port. This is ASync operation, the connection is active after OnConnectionEstablished is called.
+        virtual ConnectionID    Connect(const char* hostAddress, unsigned int port) = 0;
+        /// Connect with internal address format. This is ASync operation, the connection is active after OnConnectionEstablished is called.
+        virtual ConnectionID    Connect(const string& address) = 0;
+        /// Request a disconnect procedure. This is ASync operation, the connection is closed after OnDisconnect is called.
+        virtual void            Disconnect(ConnectionID id) = 0;
+
+        virtual unsigned int    GetPort() const = 0;
+
+        /// Returns the maximum message size that will fit in one datagram
+        virtual unsigned int GetMessageMTU() = 0;
+
+        /// Returns maximum message size (with splitting or without). Splitting will make your message reliable, which might not be optimal for Unreliable messages. It's better to send two unreliable.
+        //virtual unsigned int  GetMaxMessageSize(bool withSplitting = true) = 0;
+
+        virtual string          ConnectionToAddress(ConnectionID id) = 0;
+
+        /**
+        * Sends buffer with an ACK callback. When the transport layer recieves an ACK it will run the callback.
+        * The carrier runs in the main game thread, so if the callback executes a function in another thread it is the
+        * responsibility of the callback creator to add thread safety.
+        *
+        * This adds reasonable overhead to the carrier data handling, and so should only be used when a callback is essential to operations.
+         *
+        * Note: ACK callback is not supported with broadcast targets and will assert.
+         */
+        virtual void            SendWithCallback(const char* data, unsigned int dataSize, AZStd::unique_ptr<CarrierACKCallback> ackCallback, ConnectionID target = AllConnections, DataReliability reliability = SEND_RELIABLE, DataPriority priority = PRIORITY_NORMAL, unsigned char channel = 0) = 0;
+        /**
+        * Sends a buffer to the target with the parameterized reliability, priority and channel.
+        *
+        * Note: Unreliable sends with buffers larger than the MTU will get upgraded to reliable.
+        */
+        virtual void            Send(const char* data, unsigned int dataSize, ConnectionID target = AllConnections, DataReliability reliability = SEND_RELIABLE, DataPriority priority = PRIORITY_NORMAL, unsigned char channel = 0) = 0;
+        /**
+         * Receive the data for the specific connection.
+         * \note Internal buffers are used make sure you periodically receive data for all connections,
+         * otherwise you might cause buffer overflow error.
+         */
+        virtual ReceiveResult   Receive(char* data, unsigned int maxDataSize, ConnectionID from, unsigned char channel = 0) = 0;
+
+        /**
+         * Query the next received message (which can the retreived with receive) maximum size.
+         * \note This is NOT always the actually message size, but a big enough buffer (rounded the nearest internal max datagram size)
+         * to hold that message.
+         */
+        unsigned int            QueryNextReceiveMessageMaxSize(ConnectionID from, unsigned char channel = 0)
+        {
+            return Receive(nullptr, 0, from, channel).m_numBytes;
+        }
+
+        // Add new connection pool function or a callback ?
+
+        /**
+         * Update must be called once per frame. In processes system messages and callback data from the carrier thread.
+         */
+        virtual void                Update() = 0;
+
+        virtual unsigned int        GetNumConnections() const = 0;
+
+        //virtual ConnectionStates  GetConnectionState(unsigned int index) const = 0;
+        //virtual ConnectionStates  GetConnectionState(ConnectionID id) const = 0;
+
+        struct FlowInformation
+        {
+            size_t      m_numToSendMessages;
+            size_t      m_numToReceiveMessages;
+
+            size_t      m_dataInTransfer;           ///< Current data in transfer (out of the ToSendQueue but NOT confirmed - reliable only)
+            size_t      m_congestionWindow;         ///<
+        };
+
+        /**
+         * Stores connection statistics, it's ok to pass NULL for any of the statistics
+         * \param id Connection ID
+         * \param lastSecond last second statistics for all data
+         * \param lifetime lifetime statistics for all data
+         * \param effectiveLastSecond last second statistics for effective data (actual data - carrier overhead excluded)
+         * \param effectiveLifetime lifetime statistics for effective data (actual data - carrier overhead excluded)
+         * \returns connection state
+         */
+        virtual ConnectionStates    QueryStatistics(ConnectionID id, TrafficControl::Statistics* lastSecond = nullptr, TrafficControl::Statistics* lifetime = nullptr,
+            TrafficControl::Statistics* effectiveLastSecond = nullptr, TrafficControl::Statistics* effectiveLifetime = nullptr,
+            FlowInformation* flowInformation = nullptr) = 0;
+
+        /**
+         * Debug function, prints the connection status report to the stdout.
+         */
+        virtual void            DebugStatusReport(ConnectionID id, unsigned char channel = 0)  { (void)id; (void)channel; }
+        virtual void            DebugDeleteConnection(ConnectionID id)                          { (void)id; }
+        // @{ Debug functions that control detect disconnect at runtime. IMPORTANT: This will override the default setting in the CarrierDescriptor!
+        virtual void            DebugEnableDisconnectDetection(bool isEnabled)                  { (void)isEnabled; }
+        virtual bool            DebugIsEnableDisconnectDetection() const                        { return false; }
+        // @}
+        virtual ConnectionID    DebugGetConnectionId(unsigned int index) const = 0;
+
+        //////////////////////////////////////////////////////////////////////////
+        // Synchronized clock in milliseconds. It will wrap around ~49.7 days.
+        /**
+         * Enables sync of the clock every syncInterval milliseconds.
+         * Only one peer in the connected grid can have the clock sync enabled, all the others will sync to it,
+         * if you enable it on two or more an assert will occur.
+         * \note syncInterval is just so the clocks stay is sync, GetTime will adjust the time using the last received sync value (or the creation of the carrier).
+         * keep the synIterval high >= 1 sec. Systems will use their internal timer to adjust anyway, this will minimize the bandwidth usage.
+         * We adjust the time with the RTT so the accuracy will depend on the RTT and it's fluctuations. On average you can expect accuracy <250 ms (implantation dependent)
+         * in practice we can reduce this to <100 but this will require a lot more complex clock scheme.
+         */
+        virtual void            StartClockSync(unsigned int syncInterval = 1000, bool isReset = false) = 0;
+        virtual void            StopClockSync() = 0;
+        /**
+         * Returns current carrier time in milliseconds. If nobody syncs the clock the time will be relative
+         * to the carrier creation.
+         */
+        virtual AZ::u32         GetTime() = 0;
+        //////////////////////////////////////////////////////////////////////////
+
+        /// Returns the max frequency we will grab messages from the queues and send in milliseconds.
+        unsigned int            GetMaxSendRate() const      { return m_maxSendRateMS; }
+
+        /// Return the owning instance of the gridmate.
+        AZ_FORCE_INLINE IGridMate*       GetGridMate() const         { return m_gridMate; }
+    protected:
+        explicit Carrier(IGridMate* gridMate)
+            : m_gridMate(gridMate)  {}
+        Carrier(const Carrier& rhs);
+        Carrier& operator=(const Carrier& rhs);
+
+        IGridMate*   m_gridMate;                ///< Pointer to the owning gridmate instance.
+        unsigned int m_maxSendRateMS;           ///< Maximum send rate in milliseconds.
+        unsigned int m_connectionRetryIntervalBase;
+        unsigned int m_connectionRetryIntervalMax;
+        unsigned int m_batchPacketCount;        ///< Number of packets queued to force send (rather than wait for m_maxSendRateMS expiration)
+    };
+
+    /**
+    * Carrier descriptor, required structure when we create a carrier (so
+    * we know how to set up all parameters)
+    */
+    struct CarrierDesc
+    {
+        CarrierDesc()
+            : m_driver(nullptr)
+            , m_trafficControl(nullptr)
+            , m_handshake(nullptr)
+            , m_simulator(nullptr)
+            , m_compressionFactory(nullptr)
+            , m_familyType(0)
+            , m_address(nullptr)
+            , m_port(0)
+            , m_driverReceiveBufferSize(0)
+            , m_driverSendBufferSize(0)
+            , m_driverIsFullPackets(false)
+            , m_driverIsCrossPlatform(false)
+            , m_version(1)
+            , m_securityData(nullptr)
+            , m_enableDisconnectDetection(true)
+            , m_connectionTimeoutMS(5000)
+            , m_disconnectDetectionRttThreshold(500.0f)
+            , m_disconnectDetectionPacketLossThreshold(0.3f)
+            , m_connectionEvaluationThreshold(0.5f)
+            , m_threadCpuID(-1)
+            , m_threadPriority(-100000)
+            , m_threadUpdateTimeMS(30)
+            , m_threadInstantResponse(true) //Default true to prevent packet loss
+            , m_recvPacketsLimit(0)
+            , m_maxConnections(~0u)
+            , m_connectionRetryIntervalBase(10)
+            , m_connectionRetryIntervalMax(1000)
+            , m_sendBatchPacketCount(0)         //0 = instant send; N = wait for N full packets or m_threadUpdateTimeMS timeout
+        {}
+
+        // connection params, driver interfaces, status callbacks
+        class Driver*                   m_driver;
+        class TrafficControl*           m_trafficControl;
+        class Handshake*                m_handshake;
+        class Simulator*                m_simulator;
+
+        AZStd::shared_ptr<CompressionFactory>       m_compressionFactory;       ///< Abstract factory to provide carrier with compression implementation
+
+        int                             m_familyType;               ///< Family type (this is driver specific value) for default family use 0.
+        const char*                     m_address;                  ///< Communication address, when 0 we use any address otherwise we bind a specific one.
+        unsigned int                    m_port;                     ///< Communication port. When 0 is implicit port (assigned by the system) or a value for explicit port.
+        unsigned int                    m_driverReceiveBufferSize;  ///< Driver receive buffer size (0 uses default buffer size). Used only if m_driver == null.
+        unsigned int                    m_driverSendBufferSize;     ///< Driver send buffer size (0 uses default buffer size). Used only if m_driver == null.
+        bool                            m_driverIsFullPackets;      ///< Used only for sockets drivers and LAN. Normally an internet packet is ~1500 bytes. With full packets you will enable big packets (64 KB or less) packets (which will fail on internet, but usually ok locally).
+        bool                            m_driverIsCrossPlatform;    ///< True if we will need communicate across platforms (need to make sure we use common platform features).
+
+        VersionType                     m_version;                  ///< Carriers with mismatching version numbers are not allowed to connect to each other. Default is 1.
+
+        const char*                     m_securityData;             ///< Pointer to string with security data
+
+        bool                            m_enableDisconnectDetection; ///< Enable/Disable disconnect detection. (should be set to false ONLY for debug purpose)
+        unsigned int                    m_connectionTimeoutMS;      ///< Connection timeout in milliseconds
+        float                           m_disconnectDetectionRttThreshold; ///< Rtt threshold in milliseconds, connection will be dropped once actual rtt is bigger than this value
+        float                           m_disconnectDetectionPacketLossThreshold; ///< Packet loss percentage threshold (0.0..1.0, 1.0 is 100%), connection will be dropped once actual packet loss exceeds this value
+
+        /**
+         * When a disconnect condition is detected (packet loss, connection timeout, high RTT, etc.) all other connections will be evaluated
+         * What we want to achieve is to disconnect connections in groups, as big as possible. To do so, we use a factor.This factor is percentage (0.00 to 1.00)
+         * of the disconnect conditions to be used to determine if a connection is bad. Default factor 0.5.
+         * Example: Let's say our connection timeout is 10 sec. We have N connections. We detected a disconnection from connection X reaching a 10 sec limit.
+         * if the connectionEvaluationThreshold is let's say 0.5 (default) all connections that we have not heard for 10 * 0.5 = 5 sec we will be disconnected
+         * on the spot.
+         */
+        float                           m_connectionEvaluationThreshold;
+
+        // thread processing
+        int  m_threadCpuID;         ///< -1 for no thread use, otherwise the number depends on the platform. \ref AZStd::thread_desc \ref AZ::JobManagerThreadDesc
+        int  m_threadPriority;      ///< depends on the platform, value of -100000 means it will inherit calling thead priority.
+        // NOTE: May we should group when m_threadUpdateTimeMS is less than 10 ms we switch to m_threadFastResponse = true mode and remove m_threadFastResponse flag.
+        int  m_threadUpdateTimeMS;  ///< Thread update time in milliseconds [0,100]. This time in general should be higher than 10 milliseconds. Otherwise it will be more efficient to set m_threadFastResponse
+        /**
+         * This flag is used to instruct carrier thread to react instantaneously when a data needs to be send/received.
+         *
+         * By default this flag is false as we would like to process network data every m_threadUpdateTimeMS
+         * otherwise to achieve this instant response time (0 latency) we will use more bandwidth because
+         * messages will be grouped less efficiently (especially true when you have small messages).
+         */
+        bool m_threadInstantResponse;
+
+        unsigned int m_recvPacketsLimit; ///< Maximum packets per second allowed to be received from an existing connection
+        unsigned int m_maxConnections; ///< maximum number of connections
+
+        unsigned int m_connectionRetryIntervalBase; ///< Base for expotential backoff of connection request retries (ie. if it's 30, will retry connection request with 30, 60, 120, 240 msec, ... delays)
+        unsigned int m_connectionRetryIntervalMax; ///< Cap for interval between connection requests
+        unsigned int m_sendBatchPacketCount;            ///< Number of packets queued to force send (rather than wait for m_maxSendRateMS expiration)
+    };
+
+    /**
+     * Default carrier implementation
+     */
+    class DefaultCarrier
+    {
+    public:
+        static Carrier* Create(const CarrierDesc& desc, IGridMate* gridMate);
+        static void     Destroy(Carrier* carrier)           { delete carrier; }
+    };
+
+    /**
+    * Carrier error codes
+    */
+    enum class CarrierErrorCode : int
+    {
+        EC_DRIVER = 0,  ///< Driver layer error.
+        EC_SECURITY,    ///< Carrier layer Security error
+    };
+
+    /**
+    * Driver error
+    */
+    struct DriverError
+    {
+        Driver::ErrorCodes m_errorCode; ///< Driver error code, including platform specific error codes.
+        //TODO: Session will be closed only if this error is critical.
+        //TODO: bool IsCriticalError() const { return true; }
+    };
+
+    /**
+    * Security error
+    */
+    struct SecurityError
+    {
+        enum
+        {
+            EC_OK = 0,
+
+            EC_UPDATE_TIMEOUT,                  ///< Carrier should be Updated/Ticked in time( the connection timeout value for now)
+            EC_BUFFER_READ_OUT_OF_BOUND,        ///< Out of bounds buffer reads
+            EC_CHANNEL_ID_OUT_OF_BOUND,         ///< Out of bounds channel id
+            EC_MESSAGE_TYPE_NOT_SUPPORTED,      ///< Unsupported message type
+            EC_SEQUENCE_NUMBER_OUT_OF_BOUND,    ///< Seq number is far from expected range
+            EC_SEQUENCE_NUMBER_DUPLICATED,      ///< Duplicate seq number
+            EC_PACKET_RATE_TOO_HIGH,            ///< Packet rate is too high
+            EC_DATA_RATE_TOO_HIGH,              ///< Data rate is too high
+            EC_INVALID_SOURCE_ADDRESS,          ///< Invalid source address
+            EC_DATAGRAM_TOO_LARGE,              ///< datagram exceeds max size
+            EC_BAD_PACKET,                      ///< datagram exceeds max size
+
+            // EC_MAX must be last
+            EC_MAX                              ///< Max # of types
+        } m_errorCode;                          ///< Security error code
+    };
+
+    /**
+    * Reasons for a disconnect callback to be called.
+    */
+    enum class CarrierDisconnectReason : AZ::u8
+    {
+        DISCONNECT_USER_REQUESTED = 0,          ///< The user requested to close the connection.
+        DISCONNECT_BAD_CONNECTION,              ///< Traffic conditions are bad to maintain a connection.
+        DISCONNECT_BAD_PACKETS,                 ///< We received invalid data packets.
+        DISCONNECT_DRIVER_ERROR,
+        DISCONNECT_HANDSHAKE_REJECTED,
+        DISCONNECT_HANDSHAKE_TIMEOUT,
+        /** A connection was initiated while the previous was never closed (properly).
+        * As a result the connection will be closed on both sides to synchronize.
+        * If you initiated this connection, you should make sure you closed the previous properly
+        * and if so you can retry to connect, which will most likely succeed. Since both sides are in sync.
+        */
+        DISCONNECT_WAS_ALREADY_CONNECTED,
+        DISCONNECT_SHUTTING_DOWN,               ///< Carrier is shutting down. You should not have connection at this time to begin with.
+        DISCONNECT_DEBUG_DELETE_CONNECTION,
+
+        DISCONNECT_VERSION_MISMATCH,            ///< Attempting to connect to a different application version.
+
+        DISCONNECT_MAX,                         ///< Must be last for internal reasons
+    };
+
+    /**
+     * Base class for carrier events
+     */
+
+    class CarrierEventsBase
+    {
+    public:
+        virtual ~CarrierEventsBase() {}
+
+        string ReasonToString(CarrierDisconnectReason reason);
+    };
+
+    class CarrierEvents
+        : public CarrierEventsBase
+        , public GridMateEBusTraits
+    {
+    public:
+        virtual void OnIncomingConnection(Carrier* carrier, ConnectionID id)
+        {
+            (void)carrier;
+            (void)id;
+        }
+
+        virtual void OnFailedToConnect(Carrier* carrier, ConnectionID id, CarrierDisconnectReason reason)
+        {
+            (void)carrier;
+            (void)id;
+            (void)reason;
+        }
+
+        virtual void OnConnectionEstablished(Carrier* carrier, ConnectionID id)
+        {
+            (void)carrier;
+            (void)id;
+        }
+
+        virtual void OnDisconnect(Carrier* carrier, ConnectionID id, CarrierDisconnectReason reason)
+        {
+            (void)carrier;
+            (void)id;
+            (void)reason;
+        }
+
+        /// Report all carrier and driver errors! id == InvalidConnectionID if the error is not connection related!
+        virtual void OnDriverError(Carrier* carrier, ConnectionID id, const DriverError& error)
+        {
+            (void)carrier;
+            (void)id;
+            (void)error;
+        }
+        virtual void OnSecurityError(Carrier* carrier, ConnectionID id, const SecurityError& error)
+        {
+            (void)carrier;
+            (void)id;
+            (void)error;
+        }
+
+        /**
+        * Notifies of data rate change
+        *
+        * The carrier has detected a peer rate change (likely from congestion). Listeners should
+        * decrease/increase generation rate to match. This version sends the rate for the lowest
+        * rate peer connection, but future versions will update per-peer and add a ConnectionID
+        * param.
+        *
+        * carrier ptr to carrier
+        * id connection with rate change
+        * sendLimitBytesPerSec new send rate in _bytes_ per second
+        */
+        virtual void OnRateChange(Carrier* carrier, ConnectionID id, AZ::u32 sendLimitBytesPerSec)
+        {
+            (void)carrier;
+            (void)id;
+            (void)sendLimitBytesPerSec;
+        };
+
+        /**
+         * Notifies of message arrival
+         *
+         * Note: as with all EBUS functions the callee must add thread safety if required
+         *
+         * carrier ptr to carrier
+         * id connection with new message
+         * channel channel receiving message
+         */
+        virtual void OnReceive(Carrier* carrier, ConnectionID id, unsigned char channel)
+        {
+            (void)carrier;
+            (void)id;
+            (void)channel;
+        };
+    };
+
+    typedef AZ::EBus<CarrierEvents> CarrierEventBus;
+
+    namespace Debug
+    {
+        class CarrierDrillerEvents
+            : public CarrierEventsBase
+            , public AZ::Debug::DrillerEBusTraits
+        {
+        public:
+            virtual void OnIncomingConnection(Carrier* carrier, ConnectionID id) = 0;
+
+            virtual void OnFailedToConnect(Carrier* carrier, ConnectionID id, CarrierDisconnectReason reason) = 0;
+
+            virtual void OnConnectionEstablished(Carrier* carrier, ConnectionID id) = 0;
+
+            virtual void OnDisconnect(Carrier* carrier, ConnectionID id, CarrierDisconnectReason reason) = 0;
+
+            /// Report all carrier and driver errors! id == InvalidConnectionID if the error is not connection related!
+            virtual void OnDriverError(Carrier* carrier, ConnectionID id, const DriverError& error) = 0;
+            virtual void OnSecurityError(Carrier* carrier, ConnectionID id, const SecurityError& error) = 0;
+
+            //////////////////////////////////////////////////////////////////////////
+            // Executed from NETWORK thread
+
+            // Driver
+            /// SendTo
+            /// ReceiveFrom
+            /// Errors
+
+            // Traffic control
+
+            /// Called every second when you update last second statistics
+            virtual void OnUpdateStatistics(const GridMate::string& address, const TrafficControl::Statistics& lastSecond, const TrafficControl::Statistics& lifeTime, const TrafficControl::Statistics& effectiveLastSecond, const TrafficControl::Statistics& effectiveLifeTime) = 0;
+
+            // Simulator
+            /// Enable/Disable
+            /// Change Simulator parameters
+
+            // Carrier
+            virtual void OnConnectionStateChanged(Carrier* carrier, ConnectionID id, Carrier::ConnectionStates newState) = 0;
+
+            //////////////////////////////////////////////////////////////////////////
+
+            //////////////////////////////////////////////////////////////////////////
+            // Executed from GAME/MAIN thread
+
+            // Handshake low level (we drill the handshake on session level too)
+
+            // Carrier - in addition to carrier events
+
+            //////////////////////////////////////////////////////////////////////////
+        };
+
+        typedef AZ::EBus<CarrierDrillerEvents> CarrierDrillerBus;
+    }
+}
+
+#endif // GM_CARRIER_H

Code/Framework/GridMate/GridMate/Carrier/Compressor.h

@@ -0,0 +1,106 @@
+/*
+* All or portions of this file Copyright (c) Amazon.com, Inc. or its affiliates or
+* its licensors.
+*
+* For complete copyright and license terms please see the LICENSE at the root of this
+* distribution (the "License"). All use of this software is governed by the License,
+* or, if provided, by the license below or the license accompanying this file. Do not
+* remove or modify any license notices. This file is distributed on an "AS IS" BASIS,
+* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+*
+*/
+#ifndef GM_COMPRESSOR_INTERFACE_H
+#define GM_COMPRESSOR_INTERFACE_H
+
+#include <AzCore/PlatformDef.h>
+#include <AzCore/std/functional.h>
+
+namespace GridMate
+{
+    /**
+    * Collection of compression related error codes
+    */
+    enum class CompressorError
+    {
+        Ok,                   ///< No error, operation finished successfully
+        InsufficientBuffer,   ///< Buffer size is insufficient for the operation to complete, increase the size and try again
+        CorruptData           ///< Malformed or hacked packet, potentially security issue
+    };
+
+    /*
+    * Unique identifier of a given compressor
+    */
+    using CompressorType = AZ::u32;
+
+    /**
+     * Packet data compressor interface
+     */
+    class Compressor
+    {
+    public:
+        virtual ~Compressor() = default;
+
+        /*
+        * Initialize compressor
+        */
+        virtual bool Init() = 0;
+
+        /*
+        * Unique identifier of a given compressor
+        */
+        virtual CompressorType GetType() const = 0;
+
+        /*
+        * Returns max possible size of uncompressed data chunk needed to fit compressed data in maxCompSize bytes
+        */
+        virtual size_t GetMaxChunkSize(size_t maxCompSize) const = 0;
+
+        /*
+        * Returns size of compressed buffer needed to uncompress uncompSize of bytes
+        */
+        virtual size_t GetMaxCompressedBufferSize(size_t uncompSize) const = 0;
+
+        /*
+        * Finalizes the stream, and returns composed packet
+        * \param uncompData - buffer to compress
+        * \param uncompSize - length of data to compress from uncompData
+        * \param compData - should be able to fit at least GetMaxCompressedBufferSize(uncompSize) bytes
+        * \param compDataSize - size of compData buffer
+        * \param compSize - length of compressed data written into compData
+        *
+        * Chunk based compressors should loop internally in Compress() to compress all chunks of uncompData.
+        */
+        virtual CompressorError Compress(const void* uncompData, size_t uncompSize, void* compData, size_t compDataSize, size_t& compSize) = 0;
+
+        /*
+        * Decompress packet
+        * \param compData - buffer to decompress
+        * \param compSize - length of data to deccompress from compData
+        * \param uncompData - should be able to fit at least GetDecompressedBufferSize(compressedDataSize)
+        * \param uncompDataSize - size of uncompData buffer.
+        * \param consumedSize - the number of bytes processed out of compData. (previously named chunkSize)
+        * \param uncompSize - length of decompressed data written into uncompData
+        *
+        * Chunk based decompressors should loop internally in Decompress() to decompress all chunks of compData.
+        */
+        virtual CompressorError Decompress(const void* compData, size_t compDataSize, void* uncompData, size_t uncompDataSize, size_t& consumedSize, size_t& uncompSize) = 0;
+    };
+
+    /**
+    * Abstract factory to instantiate compressors
+    * Used by carrier to create compressor
+    */
+    class CompressionFactory
+    {
+    public:
+        virtual ~CompressionFactory() = default;
+
+        /*
+        * Instantiate new compressor
+        */
+        virtual AZStd::shared_ptr<Compressor> CreateCompressor() = 0;
+    };
+}
+
+#endif // GM_COMPRESSOR_INTERFACE_H
+

Code/Framework/GridMate/GridMate/Carrier/Cripter.h

@@ -0,0 +1,29 @@
+/*
+* All or portions of this file Copyright (c) Amazon.com, Inc. or its affiliates or
+* its licensors.
+*
+* For complete copyright and license terms please see the LICENSE at the root of this
+* distribution (the "License"). All use of this software is governed by the License,
+* or, if provided, by the license below or the license accompanying this file. Do not
+* remove or modify any license notices. This file is distributed on an "AS IS" BASIS,
+* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+*
+*/
+#ifndef GM_CRIPTER_INTERFACE_H
+#define GM_CRIPTER_INTERFACE_H
+
+#include <GridMate/Types.h>
+
+namespace GridMate
+{
+    /**
+     * Traffic control interface
+     */
+    class Cripter
+    {
+    public:
+    };
+}
+
+#endif // GM_CRIPTER_INTERFACE_H
+

Code/Framework/GridMate/GridMate/Carrier/DefaultHandshake.cpp

@@ -0,0 +1,119 @@
+/*
+* All or portions of this file Copyright (c) Amazon.com, Inc. or its affiliates or
+* its licensors.
+*
+* For complete copyright and license terms please see the LICENSE at the root of this
+* distribution (the "License"). All use of this software is governed by the License,
+* or, if provided, by the license below or the license accompanying this file. Do not
+* remove or modify any license notices. This file is distributed on an "AS IS" BASIS,
+* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+*
+*/
+
+#include <GridMate/Carrier/DefaultHandshake.h>
+#include <GridMate/Serialize/UtilityMarshal.h>
+
+using namespace GridMate;
+
+//=========================================================================
+// DefaultHandshake
+// [11/5/2010]
+//=========================================================================
+DefaultHandshake::DefaultHandshake(unsigned int timeOut, VersionType version)
+{
+    m_handshakeTimeOutMS = timeOut;
+    m_version = version;
+}
+
+//=========================================================================
+// ~DefaultHandshake
+// [11/5/2010]
+//=========================================================================
+DefaultHandshake::~DefaultHandshake()
+{
+}
+
+//=========================================================================
+// GetWelcomeData
+// [11/5/2010]
+//=========================================================================
+void
+DefaultHandshake::OnInitiate(ConnectionID id, WriteBuffer& wb)
+{
+    (void)id;
+    wb.Write(m_version);
+}
+
+//=========================================================================
+// OnReceiveRequest
+// [11/5/2010]
+//=========================================================================
+HandshakeErrorCode
+DefaultHandshake::OnReceiveRequest(ConnectionID id, ReadBuffer& rb, WriteBuffer& wb)
+{
+    (void)id;
+    OnInitiate(id, wb); // send back the version string
+
+    VersionType version;
+    if (rb.Read(version) && version == m_version)
+    {
+        return HandshakeErrorCode::OK;
+    }
+    else
+    {
+        return HandshakeErrorCode::VERSION_MISMATCH;
+    }
+}
+
+//=========================================================================
+// OnConfirmRequest
+// [2/10/2011]
+//=========================================================================
+bool
+DefaultHandshake::OnConfirmRequest(ConnectionID id, ReadBuffer& rb)
+{
+    return OnReceiveAck(id, rb);
+}
+
+//=========================================================================
+// OnReceiveAck
+// [2/2/2011]
+//=========================================================================
+bool
+DefaultHandshake::OnReceiveAck(ConnectionID id, ReadBuffer& rb)
+{
+    (void)id;
+    (void)rb;
+    return true;
+}
+
+//=========================================================================
+// OnConfirmAck
+// [2/10/2011]
+//=========================================================================
+bool
+DefaultHandshake::OnConfirmAck(ConnectionID id, ReadBuffer& rb)
+{
+    return OnReceiveAck(id, rb);
+}
+
+//=========================================================================
+// OnNewConnection
+// [11/5/2010]
+//=========================================================================
+bool
+DefaultHandshake::OnNewConnection(const string& address)
+{
+    (void)address;
+    return true; /// We don't have a ban list yet
+}
+
+//=========================================================================
+// OnDisconnect
+// [11/5/2010]
+//=========================================================================
+void
+DefaultHandshake::OnDisconnect(ConnectionID id)
+{
+    (void)id;
+}

Code/Framework/GridMate/GridMate/Carrier/DefaultHandshake.h

@@ -0,0 +1,66 @@
+/*
+* All or portions of this file Copyright (c) Amazon.com, Inc. or its affiliates or
+* its licensors.
+*
+* For complete copyright and license terms please see the LICENSE at the root of this
+* distribution (the "License"). All use of this software is governed by the License,
+* or, if provided, by the license below or the license accompanying this file. Do not
+* remove or modify any license notices. This file is distributed on an "AS IS" BASIS,
+* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+*
+*/
+#ifndef GM_DEFAULT_HANDSHAKE_H
+#define GM_DEFAULT_HANDSHAKE_H
+
+#include <GridMate/Carrier/Handshake.h>
+
+namespace GridMate
+{
+    /**
+     * Default handshake interface.
+     */
+    class DefaultHandshake
+        : public Handshake
+    {
+    public:
+        GM_CLASS_ALLOCATOR(DefaultHandshake);
+
+        DefaultHandshake(unsigned int timeOut, VersionType version);
+        virtual ~DefaultHandshake();
+        /// Called from the system to write initial handshake data.
+        virtual void                OnInitiate(ConnectionID id, WriteBuffer& wb);
+        /**
+        * Called when a system receives a handshake initiation from another system.
+        * You can write a reply in the WriteBuffer.
+        * return true if you accept this connection and false if you reject it.
+        */
+        virtual HandshakeErrorCode  OnReceiveRequest(ConnectionID id, ReadBuffer& rb, WriteBuffer& wb);
+        /**
+        * If we already have a valid connection and we receive another connection request, the system will
+        * call this function to verify the state of the connection.
+        */
+        virtual bool                OnConfirmRequest(ConnectionID id, ReadBuffer& rb);
+        /**
+        * Called when we receive Ack from the other system on our initial data \ref OnInitiate.
+        * return true to accept the ack or false to reject the handshake.
+        */
+        virtual bool                OnReceiveAck(ConnectionID id, ReadBuffer& rb);
+        /**
+        * Called when we receive Ack from the other system while we were connected. This callback is called
+        * so we can just confirm that our connection is valid!
+        */
+        virtual bool                OnConfirmAck(ConnectionID id, ReadBuffer& rb);
+        /// Return true if you want to reject early reject a connection.
+        virtual bool                OnNewConnection(const string& address);
+        /// Called when we close a connection.
+        virtual void                OnDisconnect(ConnectionID id);
+        /// Return timeout in milliseconds of the handshake procedure.
+        virtual unsigned int        GetHandshakeTimeOutMS() const       { return m_handshakeTimeOutMS; }
+    private:
+        unsigned int m_handshakeTimeOutMS;
+        VersionType m_version;
+    };
+}
+
+#endif // GM_DEFAULT_HANDSHAKE_H
+

Code/Framework/GridMate/GridMate/Carrier/DefaultSimulator.cpp

@@ -0,0 +1,722 @@
+/*
+* All or portions of this file Copyright (c) Amazon.com, Inc. or its affiliates or
+* its licensors.
+*
+* For complete copyright and license terms please see the LICENSE at the root of this
+* distribution (the "License"). All use of this software is governed by the License,
+* or, if provided, by the license below or the license accompanying this file. Do not
+* remove or modify any license notices. This file is distributed on an "AS IS" BASIS,
+* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+*
+*/
+
+#include <GridMate/Carrier/DefaultSimulator.h>
+#include <GridMate/Carrier/Driver.h>
+
+#include <AzCore/std/parallel/lock.h>
+
+#include <AzCore/Math/Crc.h>
+
+#include <stdlib.h>
+
+using namespace GridMate;
+
+//=========================================================================
+// DefaultSimulator
+// [10/15/2010]
+//=========================================================================
+DefaultSimulator::DefaultSimulator()
+{
+    // Latency in milliseconds
+    m_minOutLatency = 0;
+    m_maxOutLatency = 0;
+    m_minInLatency = 0;
+    m_maxInLatency = 0;
+
+    // Packet loss in %
+    m_minOutPacketLoss = 0;
+    m_maxOutPacketLoss = 0;
+    m_minInPacketLoss = 0;
+    m_maxInPacketLoss = 0;
+    m_isOutPacketLoss = false;
+    m_isInPacketLoss = false;
+
+    m_numOutPacketsTillDrop = 0;
+    m_numInPacketsTillDrop = 0;
+
+    // Packet loss in periods
+    m_minInPacketDropInterval = 0;
+    m_maxInPacketDropInterval = 0;
+    m_minInPacketDropPeriod = 0;
+    m_maxInPacketDropPeriod = 0;
+    m_minOutPacketDropInterval = 0;
+    m_maxOutPacketDropInterval = 0;
+    m_minOutPacketDropPeriod = 0;
+    m_maxOutPacketDropPeriod = 0;
+
+    m_inPacketDropInterval = 0;
+    m_inPacketDropPeriod = 0;
+    m_outPacketDropInterval = 0;
+    m_outPacketDropPeriod = 0;
+
+    // Bandwidth in Kbps
+    m_minOutBandwidth = 0;
+    m_maxOutBandwidth = 0;
+    m_minInBandwidth = 0;
+    m_maxInBandwidth = 0;
+
+    m_currentDataOut = 0;
+    m_currentDataOutMax = 0;
+    m_currentDataIn = 0;
+    m_currentDataInMax = 0;
+
+    m_dataLimiterTimeout = 0;
+
+    m_outReorder = false;
+    m_inReorder = false;
+
+    m_enable = false;
+
+    m_driver = nullptr;
+}
+
+//=========================================================================
+// ~DefaultSimulator
+// [10/15/2010]
+//=========================================================================
+DefaultSimulator::~DefaultSimulator()
+{
+    FreeAllData();
+}
+
+void DefaultSimulator::BindDriver(class Driver* driver)
+{
+    m_driver = driver;
+}
+
+void DefaultSimulator::UnbindDriver()
+{
+    FreeAllData();
+    m_driver = nullptr;
+}
+
+//=========================================================================
+// OnConnect
+// [10/15/2010]
+//=========================================================================
+void
+DefaultSimulator::OnConnect(const AZStd::intrusive_ptr<DriverAddress>& address)
+{
+    (void)address;
+}
+
+//=========================================================================
+// OnDisconnect
+// [10/15/2010]
+//=========================================================================
+void
+DefaultSimulator::OnDisconnect(const AZStd::intrusive_ptr<DriverAddress>& address)
+{
+    AZStd::lock_guard<AZStd::mutex> l(m_lock);
+
+    for (PacketListType::iterator pakIter = m_outgoing.begin(); pakIter != m_outgoing.end(); )
+    {
+        Packet& pak = *pakIter;
+        if (pak.m_address == address)
+        {
+            azfree(pak.m_data, GridMateAllocatorMP);
+            pakIter = m_outgoing.erase(pakIter);
+        }
+        else
+        {
+            ++pakIter;
+        }
+    }
+    for (PacketListType::iterator pakIter = m_incoming.begin(); pakIter != m_incoming.end(); )
+    {
+        Packet& pak = *pakIter;
+        if (pak.m_address == address)
+        {
+            azfree(pak.m_data, GridMateAllocatorMP);
+            pakIter = m_incoming.erase(pakIter);
+        }
+        else
+        {
+            ++pakIter;
+        }
+    }
+}
+
+//=========================================================================
+// OnSend
+// [10/15/2010]
+//=========================================================================
+bool
+DefaultSimulator::OnSend(const AZStd::intrusive_ptr<DriverAddress>& to, const void* data, unsigned int dataSize)
+{
+    if (!m_enable)
+    {
+        return false;
+    }
+
+    {
+        AZStd::lock_guard<AZStd::mutex> l(m_lock);
+
+        if (m_isOutPacketLoss)
+        {
+            if (m_numOutPacketsTillDrop == 0)
+            {
+                m_numOutPacketsTillDrop = static_cast<unsigned int>((double)rand() / ((double)RAND_MAX + 1) * (m_maxOutPacketLoss - m_minOutPacketLoss) + m_minOutPacketLoss);
+                return true; // we handle the packet and do nothing ;)
+            }
+            else
+            {
+                m_numOutPacketsTillDrop--;
+            }
+        }
+
+        if (m_outPacketDropInterval != 0)
+        {
+            return true; // If we are dropping outbound packets due to time restriction just drop the packet
+        }
+
+        if (m_currentDataOutMax != 0)
+        {
+            m_currentDataOut += dataSize;
+            if (m_currentDataOut > m_currentDataOutMax)
+            {
+                return true; // we are over the limit drop the packet
+            }
+        }
+
+        if (m_maxOutLatency > 0)
+        {
+            int latency = static_cast<int>(static_cast<double>(rand()) / (static_cast<double>(RAND_MAX) + 1) * (m_maxOutLatency - m_minOutLatency) + m_minOutLatency);
+            if (latency > 0)
+            {
+                Packet pak;
+                pak.m_data = reinterpret_cast<char*>(azmalloc(dataSize, 1, GridMateAllocatorMP));
+                pak.m_dataSize = dataSize;
+                memcpy(pak.m_data, data, dataSize);
+                pak.m_latency = latency;
+                pak.m_address = to;
+                pak.m_startTime = AZStd::chrono::system_clock::now();
+
+                if (m_outReorder)
+                {
+                    // insert at random place
+                    PacketListType::iterator pakIter = m_outgoing.begin();
+                    int pos = rand() % (m_outgoing.size() + 1);
+                    for (; pos > 0; --pos)
+                    {
+                        ++pakIter;
+                    }
+                    m_outgoing.insert(pakIter, pak);
+                }
+                else
+                {
+                    m_outgoing.push_back(pak);
+                }
+                return true;
+            }
+        }
+    }
+
+    return false;
+}
+
+//=========================================================================
+// OnReceive
+// [10/15/2010]
+//=========================================================================
+bool
+DefaultSimulator::OnReceive(const AZStd::intrusive_ptr<DriverAddress>& from, const void* data, unsigned int dataSize)
+{
+    if (!m_enable)
+    {
+        return false;
+    }
+
+    {
+        AZStd::lock_guard<AZStd::mutex> l(m_lock);
+        if (m_isInPacketLoss)
+        {
+            if (m_numInPacketsTillDrop == 0)
+            {
+                m_numInPacketsTillDrop = static_cast<unsigned int>((double)rand() / ((double)RAND_MAX + 1) * (m_maxInPacketLoss - m_minInPacketLoss)    + m_minInPacketLoss);
+                return true; // we handle the packet and do nothing ;)
+            }
+            else
+            {
+                m_numInPacketsTillDrop--;
+            }
+        }
+
+        if (m_inPacketDropInterval != 0)
+        {
+            return true; // If we are dropping outbound packets due to time restriction just drop the packet
+        }
+
+        if (m_currentDataInMax != 0)
+        {
+            m_currentDataIn += dataSize;
+            if (m_currentDataIn > m_currentDataInMax)
+            {
+                return true; // we are over the limit drop the packet
+            }
+        }
+
+        if (m_maxInLatency > 0)
+        {
+            int latency = static_cast<int>(static_cast<double>(rand()) / (static_cast<double>(RAND_MAX) + 1) * (m_maxInLatency - m_minInLatency)    + m_minInLatency);
+            if (latency > 0)
+            {
+                Packet pak;
+                pak.m_data =  reinterpret_cast<char*>(azmalloc(dataSize, 1, GridMateAllocatorMP));
+                pak.m_dataSize = dataSize;
+                memcpy(pak.m_data, data, dataSize);
+                pak.m_latency = latency;
+                pak.m_address = from;
+                pak.m_startTime = AZStd::chrono::system_clock::now();
+
+                if (m_inReorder)
+                {
+                    // insert at random place
+                    PacketListType::iterator pakIter = m_incoming.begin();
+                    int pos = rand() % (m_incoming.size() + 1);
+                    for (; pos > 0; --pos)
+                    {
+                        ++pakIter;
+                    }
+                    m_incoming.insert(pakIter, pak);
+                }
+                else
+                {
+                    m_incoming.push_back(pak);
+                }
+                return true;
+            }
+        }
+
+        if (m_maxOutBandwidth > 0)
+        {
+        }
+    }
+
+    return false;
+}
+
+//=========================================================================
+// ReceiveDataFrom
+// [10/15/2010]
+//=========================================================================
+unsigned int
+DefaultSimulator::ReceiveDataFrom(AZStd::intrusive_ptr<DriverAddress>& from, char* data, unsigned int maxDataSize)
+{
+    (void)maxDataSize;
+    if (!m_enable && m_incoming.empty())
+    {
+        return 0;
+    }
+
+    unsigned int dataSize = 0;
+    {
+        AZStd::lock_guard<AZStd::mutex> l(m_lock);
+
+        if (!m_incoming.empty())
+        {
+            Packet& pak = m_incoming.front();
+            AZStd::chrono::milliseconds elapsed = AZStd::chrono::system_clock::now() - pak.m_startTime;
+            if (!m_enable || elapsed.count() >= pak.m_latency)
+            {
+                from = pak.m_address;
+                AZ_Assert(maxDataSize >= pak.m_dataSize, "Buffer to receive data is too small");
+                memcpy(data, pak.m_data, pak.m_dataSize);
+                azfree(pak.m_data, GridMateAllocatorMP);
+
+                dataSize = pak.m_dataSize;
+
+                m_incoming.pop_front();
+            }
+        }
+    }
+    return dataSize;
+}
+
+//=========================================================================
+// Update
+// [10/15/2010]
+//=========================================================================
+void
+DefaultSimulator::Update()
+{
+    if (!m_enable && m_outgoing.empty())
+    {
+        return;
+    }
+    {
+        //
+        TimeStamp now = AZStd::chrono::system_clock::now();
+        //////////////////////////////////////////////////////////////////////////
+        // Or we can deliver this from the engine
+        unsigned int deltaTime = static_cast<unsigned int>(AZStd::chrono::milliseconds(now - m_currentTime).count());
+        deltaTime = AZStd::GetMin<unsigned int>(deltaTime, 100);
+        //////////////////////////////////////////////////////////////////////////
+        m_currentTime = now;
+
+        AZStd::lock_guard<AZStd::mutex> l(m_lock);
+        for (PacketListType::iterator pakIter = m_outgoing.begin(); pakIter != m_outgoing.end(); )
+        {
+            Packet& pak = *pakIter;
+            AZStd::chrono::milliseconds elapsed = now - pak.m_startTime;
+            if (!m_enable || elapsed.count() >= pak.m_latency)
+            {
+                if (m_driver->Send(pak.m_address, pak.m_data, pak.m_dataSize) == Driver::EC_OK)
+                {
+                    azfree(pak.m_data, GridMateAllocatorMP);
+                    pakIter = m_outgoing.erase(pakIter);
+                    continue;
+                }
+            }
+            // we don't want to send the next packet (or anything after it),
+            // so break out of the loop.
+            break;
+        }
+
+        if (m_maxInPacketDropPeriod != 0 || m_maxOutPacketDropPeriod != 0)
+        {
+            if (m_inPacketDropPeriod > deltaTime)
+            {
+                m_inPacketDropPeriod -= deltaTime;
+                if (m_inPacketDropInterval > deltaTime)
+                {
+                    m_inPacketDropInterval -= deltaTime;
+                }
+                else
+                {
+                    m_inPacketDropInterval = 0;
+                }
+            }
+            else if (m_maxInPacketDropPeriod != 0 && m_maxInPacketDropInterval != 0)
+            {
+                // period has expired compute a new period
+                m_inPacketDropPeriod = static_cast<unsigned int>((double)rand() / ((double)RAND_MAX + 1) * (m_maxInPacketDropPeriod - m_minInPacketDropPeriod) + m_minInPacketDropPeriod);
+                m_inPacketDropInterval = static_cast<unsigned int>((double)rand() / ((double)RAND_MAX + 1) * (m_maxInPacketDropInterval - m_minInPacketDropInterval) + m_minInPacketDropInterval);
+                if (m_inPacketDropInterval > m_inPacketDropPeriod)
+                {
+                    m_inPacketDropInterval = m_inPacketDropPeriod; // at worst we can drop all the packets for the period
+                }
+            }
+
+            if (m_outPacketDropPeriod > deltaTime)
+            {
+                m_outPacketDropPeriod -= deltaTime;
+                if (m_outPacketDropInterval > deltaTime)
+                {
+                    m_outPacketDropInterval -= deltaTime;
+                }
+                else
+                {
+                    m_outPacketDropInterval = 0;
+                }
+            }
+            else if (m_maxOutPacketDropPeriod != 0 && m_maxOutPacketDropInterval != 0)
+            {
+                // period has expired compute a new period
+                m_outPacketDropPeriod = static_cast<unsigned int>((double)rand() / ((double)RAND_MAX + 1) * (m_maxOutPacketDropPeriod - m_minOutPacketDropPeriod) + m_minOutPacketDropPeriod);
+                m_outPacketDropInterval = static_cast<unsigned int>((double)rand() / ((double)RAND_MAX + 1) * (m_maxOutPacketDropInterval - m_minOutPacketDropInterval) + m_minOutPacketDropInterval);
+                if (m_outPacketDropInterval > m_outPacketDropPeriod)
+                {
+                    m_outPacketDropInterval = m_outPacketDropPeriod; // at worst we can drop all the packets for the period
+                }
+            }
+        }
+
+        if (m_maxOutBandwidth != 0 || m_maxInBandwidth != 0)
+        {
+            m_dataLimiterTimeout += deltaTime;
+
+            if (m_dataLimiterTimeout > 1000)  // every second
+            {
+                m_dataLimiterTimeout -= 1000;
+
+                m_currentDataOut = 0;
+                m_currentDataIn = 0;
+                if (m_maxOutBandwidth != 0)
+                {
+                    m_currentDataOutMax = static_cast<unsigned int>((double)rand() / ((double)RAND_MAX + 1) * (m_maxOutBandwidth - m_minOutBandwidth) + m_minOutBandwidth);
+                    m_currentDataOutMax = m_currentDataOutMax / 8 * 1024; // Convert to bytes per second
+                }
+                if (m_maxInBandwidth)
+                {
+                    m_currentDataInMax = static_cast<unsigned int>((double)rand() / ((double)RAND_MAX + 1) * (m_maxInBandwidth - m_minInBandwidth) + m_minInBandwidth);
+                    m_currentDataInMax = m_currentDataInMax / 8 * 1024; // Convert to bytes per second
+                }
+            }
+        }
+    }
+}
+
+//=========================================================================
+// FreeAllData
+// [10/15/2010]
+//=========================================================================
+void
+DefaultSimulator::FreeAllData()
+{
+    AZStd::lock_guard<AZStd::mutex> l(m_lock);
+    while (!m_outgoing.empty())
+    {
+        Packet& pak = m_outgoing.front();
+        azfree(pak.m_data, GridMateAllocatorMP);
+        m_outgoing.pop_front();
+    }
+    while (!m_incoming.empty())
+    {
+        Packet& pak = m_incoming.front();
+        azfree(pak.m_data, GridMateAllocatorMP);
+        m_incoming.pop_front();
+    }
+}
+
+//=========================================================================
+// SetOutgoingLatency
+// [10/18/2010]
+//=========================================================================
+void
+DefaultSimulator::SetOutgoingLatency(unsigned int minDelayMS, unsigned int maxDelayMS)
+{
+    AZStd::lock_guard<AZStd::mutex> l(m_lock);
+    m_minOutLatency = minDelayMS;
+    m_maxOutLatency = maxDelayMS;
+}
+//=========================================================================
+// SetIncomingLatency
+// [10/18/2010]
+//=========================================================================
+void
+DefaultSimulator::SetIncomingLatency(unsigned int minDelayMS, unsigned int maxDelayMS)
+{
+    AZStd::lock_guard<AZStd::mutex> l(m_lock);
+    m_minInLatency = minDelayMS;
+    m_maxInLatency = maxDelayMS;
+}
+
+//=========================================================================
+// GetOutgoingLatency
+// [10/18/2010]
+//=========================================================================
+void
+DefaultSimulator::GetOutgoingLatency(unsigned int& minDelayMS, unsigned int& maxDelayMS)
+{
+    AZStd::lock_guard<AZStd::mutex> l(m_lock);
+    minDelayMS = m_minOutLatency;
+    maxDelayMS = m_maxOutLatency;
+}
+//=========================================================================
+// GetIncomingLatency
+// [10/18/2010]
+//=========================================================================
+void
+DefaultSimulator::GetIncomingLatency(unsigned int& minDelayMS, unsigned int& maxDelayMS)
+{
+    AZStd::lock_guard<AZStd::mutex> l(m_lock);
+    minDelayMS = m_minInLatency;
+    maxDelayMS = m_maxInLatency;
+}
+
+//=========================================================================
+// SetOutgoingPacketLoss
+// [10/18/2010]
+//=========================================================================
+void
+DefaultSimulator::SetOutgoingPacketLoss(unsigned int minInterval, unsigned int maxInterval)
+{
+    AZStd::lock_guard<AZStd::mutex> l(m_lock);
+
+    if (minInterval > 0)
+    {
+        m_minOutPacketLoss = minInterval - 1;
+    }
+
+    if (maxInterval > 0)
+    {
+        m_maxOutPacketLoss = maxInterval - 1;
+        m_isOutPacketLoss = true;
+    }
+    else
+    {
+        m_isOutPacketLoss = false;
+    }
+}
+
+//=========================================================================
+// SetIncomingPacketLoss
+// [10/18/2010]
+//=========================================================================
+void
+DefaultSimulator::SetIncomingPacketLoss(unsigned int minInterval, unsigned int maxInterval)
+{
+    AZStd::lock_guard<AZStd::mutex> l(m_lock);
+
+    if (minInterval > 0)
+    {
+        m_minInPacketLoss = minInterval - 1;
+    }
+
+    if (maxInterval > 0)
+    {
+        m_maxInPacketLoss = maxInterval - 1;
+        m_isInPacketLoss = true;
+    }
+    else
+    {
+        m_isInPacketLoss = false;
+    }
+}
+
+//=========================================================================
+// GetOutgoingPacketLoss
+// [5/25/2011]
+//=========================================================================
+void
+DefaultSimulator::GetOutgoingPacketLoss(unsigned int& minInterval, unsigned int& maxInterval)
+{
+    if (m_isOutPacketLoss)
+    {
+        AZStd::lock_guard<AZStd::mutex> l(m_lock);
+        minInterval = m_minOutPacketLoss + 1;
+        maxInterval = m_maxOutPacketLoss + 1;
+    }
+    else
+    {
+        minInterval = 0;
+        maxInterval = 0;
+    }
+}
+
+//=========================================================================
+// GetIncomingPacketLoss
+// [5/25/2011]
+//=========================================================================
+void
+DefaultSimulator::GetIncomingPacketLoss(unsigned int& minInterval, unsigned int& maxInterval)
+{
+    if (m_isInPacketLoss)
+    {
+        AZStd::lock_guard<AZStd::mutex> l(m_lock);
+        minInterval = m_minInPacketLoss + 1;
+        maxInterval = m_maxInPacketLoss + 1;
+    }
+    else
+    {
+        minInterval = 0;
+        maxInterval = 0;
+    }
+}
+
+//=========================================================================
+// SetOutgoingBandwidth
+// [10/18/2010]
+//=========================================================================
+void
+DefaultSimulator::SetOutgoingBandwidth(unsigned int minBandwidthKbps, unsigned int maxBandwidthKbps)
+{
+    AZStd::lock_guard<AZStd::mutex> l(m_lock);
+    m_minOutBandwidth = minBandwidthKbps;
+    m_maxOutBandwidth = maxBandwidthKbps;
+}
+//=========================================================================
+// SetIncomingBandwidth
+// [10/18/2010]
+//=========================================================================
+void
+DefaultSimulator::SetIncomingBandwidth(unsigned int minBandwidthKbps, unsigned int maxBandwidthKbps)
+{
+    AZStd::lock_guard<AZStd::mutex> l(m_lock);
+    m_minInBandwidth = minBandwidthKbps;
+    m_maxInBandwidth = maxBandwidthKbps;
+}
+
+//=========================================================================
+// GetOutgoingBandwidth
+// [10/18/2010]
+//=========================================================================
+void
+DefaultSimulator::GetOutgoingBandwidth(unsigned int& minBandwidthKbps, unsigned int& maxBandwidthKbps)
+{
+    AZStd::lock_guard<AZStd::mutex> l(m_lock);
+    minBandwidthKbps = m_minOutBandwidth;
+    maxBandwidthKbps = m_maxOutBandwidth;
+}
+//=========================================================================
+// GetIncomingBandwidth
+// [10/18/2010]
+//=========================================================================
+void
+DefaultSimulator::GetIncomingBandwidth(unsigned int& minBandwidthKbps, unsigned int& maxBandwidthKbps)
+{
+    AZStd::lock_guard<AZStd::mutex> l(m_lock);
+    minBandwidthKbps = m_minInBandwidth;
+    maxBandwidthKbps = m_maxInBandwidth;
+}
+
+//=========================================================================
+// SetOutgoingPacketDrop
+// [10/21/2013]
+//=========================================================================
+void
+DefaultSimulator::SetOutgoingPacketDrop(unsigned int minDropInterval, unsigned int maxDropInterval, unsigned int minDropPeriod, unsigned int maxDropPeriod)
+{
+    AZStd::lock_guard<AZStd::mutex> l(m_lock);
+    m_minOutPacketDropInterval = minDropInterval;
+    m_maxOutPacketDropInterval = maxDropInterval;
+    m_minOutPacketDropPeriod = minDropPeriod;
+    m_maxOutPacketDropPeriod = maxDropPeriod;
+
+    m_outPacketDropInterval = 0;
+    m_outPacketDropPeriod = 0;
+}
+
+//=========================================================================
+// SetIncomingPacketDrop
+// [10/21/2013]
+//=========================================================================
+void
+DefaultSimulator::SetIncomingPacketDrop(unsigned int minDropInterval, unsigned int maxDropInterval, unsigned int minDropPeriod, unsigned int maxDropPeriod)
+{
+    AZStd::lock_guard<AZStd::mutex> l(m_lock);
+    m_minInPacketDropInterval = minDropInterval;
+    m_maxInPacketDropInterval = maxDropInterval;
+    m_minInPacketDropPeriod = minDropPeriod;
+    m_maxInPacketDropPeriod = maxDropPeriod;
+
+    m_inPacketDropInterval = 0;
+    m_inPacketDropPeriod = 0;
+}
+
+//=========================================================================
+// GetOutgoingPacketDrop
+// [10/21/2013]
+//=========================================================================
+void DefaultSimulator::GetOutgoingPacketDrop(unsigned int& minDropInterval, unsigned int& maxDropInterval, unsigned int& minDropPeriod, unsigned int& maxDropPeriod)
+{
+    AZStd::lock_guard<AZStd::mutex> l(m_lock);
+    minDropInterval = m_minOutPacketDropInterval;
+    maxDropInterval = m_maxOutPacketDropInterval;
+    minDropPeriod = m_minOutPacketDropPeriod;
+    maxDropPeriod = m_maxOutPacketDropPeriod;
+}
+
+//=========================================================================
+// GetIncomingPacketDrop
+// [10/21/2013]
+//=========================================================================
+void DefaultSimulator::GetIncomingPacketDrop(unsigned int& minDropInterval, unsigned int& maxDropInterval, unsigned int& minDropPeriod, unsigned int& maxDropPeriod)
+{
+    AZStd::lock_guard<AZStd::mutex> l(m_lock);
+    minDropInterval = m_minInPacketDropInterval;
+    maxDropInterval = m_maxInPacketDropInterval;
+    minDropPeriod = m_minInPacketDropPeriod;
+    maxDropPeriod = m_maxInPacketDropPeriod;
+}

Code/Framework/GridMate/GridMate/Carrier/DefaultSimulator.h

@@ -0,0 +1,163 @@
+/*
+* All or portions of this file Copyright (c) Amazon.com, Inc. or its affiliates or
+* its licensors.
+*
+* For complete copyright and license terms please see the LICENSE at the root of this
+* distribution (the "License"). All use of this software is governed by the License,
+* or, if provided, by the license below or the license accompanying this file. Do not
+* remove or modify any license notices. This file is distributed on an "AS IS" BASIS,
+* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+*
+*/
+#ifndef GM_DEFAULT_SIMULATOR_H
+#define GM_DEFAULT_SIMULATOR_H
+
+#include <GridMate/Carrier/Simulator.h>
+#include <GridMate/Containers/list.h>
+#include <AzCore/std/parallel/mutex.h>
+
+namespace GridMate
+{
+    /**
+    * Simulator default implementation. It will run on the network thread so all user functions are thread safe.
+    */
+    class DefaultSimulator
+        : public Simulator
+    {
+        friend class CarrierThread;
+
+        /// Called from Carrier, so simulator can use the low level driver directly.
+        virtual void BindDriver(Driver* driver);
+        /// Called from Carrier when driver can no longer be used(ie. will be destroyed)
+        virtual void UnbindDriver();
+        /// Called when Carrier has established a new connection.
+        virtual void OnConnect(const AZStd::intrusive_ptr<DriverAddress>& address);
+        /// Called when Carrier has lost a connection.
+        virtual void OnDisconnect(const AZStd::intrusive_ptr<DriverAddress>& address);
+        /// Called when Carrier has send a package.
+        virtual bool OnSend(const AZStd::intrusive_ptr<DriverAddress>& to, const void* data, unsigned int dataSize);
+        /// Called when Carrier receives package receive.
+        virtual bool OnReceive(const AZStd::intrusive_ptr<DriverAddress>& from, const void* data, unsigned int dataSize);
+        /// Called from Carrier when no more data has arrived and you can supply you data (with latency, out of order, etc.
+        virtual unsigned int ReceiveDataFrom(AZStd::intrusive_ptr<DriverAddress>& from, char* data, unsigned int maxDataSize);
+
+        virtual void Update();
+    public:
+        GM_CLASS_ALLOCATOR(DefaultSimulator);
+
+        DefaultSimulator();
+        virtual ~DefaultSimulator();
+
+        void Enable()           { m_enable = true; }
+        void Disable()          { m_enable = false; }
+        bool IsEnabled() const  { return m_enable; }
+
+        void SetOutgoingLatency(unsigned int minDelayMS, unsigned int maxDelayMS);
+        void SetIncomingLatency(unsigned int minDelayMS, unsigned int maxDelayMS);
+
+        void GetOutgoingLatency(unsigned int& minDelayMS, unsigned int& maxDelayMS);
+        void GetIncomingLatency(unsigned int& minDelayMS, unsigned int& maxDelayMS);
+
+        /// Lose one packet every interval
+        void SetOutgoingPacketLoss(unsigned int minInterval, unsigned int maxInterval);
+        void SetIncomingPacketLoss(unsigned int minInterval, unsigned int maxInterval);
+
+        void GetOutgoingPacketLoss(unsigned int& minInterval, unsigned int& maxInterval);
+        void GetIncomingPacketLoss(unsigned int& minInterval, unsigned int& maxInterval);
+
+        void SetOutgoingBandwidth(unsigned int minBandwidthKbps, unsigned int maxBandwidthKbps);
+        void SetIncomingBandwidth(unsigned int minBandwidthKbps, unsigned int maxBandwidthKbps);
+
+        void GetOutgoingBandwidth(unsigned int& minBandwidthKbps, unsigned int& maxBandwidthKbps);
+        void GetIncomingBandwidth(unsigned int& minBandwidthKbps, unsigned int& maxBandwidthKbps);
+
+        void SetOutgoingPacketDrop(unsigned int minDropInterval, unsigned int maxDropInterval, unsigned int minDropPeriod, unsigned int maxDropPeriod);
+        void SetIncomingPacketDrop(unsigned int minDropInterval, unsigned int maxDropInterval, unsigned int minDropPeriod, unsigned int maxDropPeriod);
+
+        void GetOutgoingPacketDrop(unsigned int& minDropInterval, unsigned int& maxDropInterval, unsigned int& minDropPeriod, unsigned int& maxDropPeriod);
+        void GetIncomingPacketDrop(unsigned int& minDropInterval, unsigned int& maxDropInterval, unsigned int& minDropPeriod, unsigned int& maxDropPeriod);
+
+        // Enable packet reordering you need to enable latency to reorder packets.
+        void SetOutgoingReorder(bool enable)    { m_outReorder = enable; }
+        void SetIncomingReorder(bool enable)    { m_inReorder = enable; }
+        bool IsOutgoingReorder() const          { return m_outReorder; }
+        bool IsIncomingReorder() const          { return m_inReorder; }
+
+    protected:
+        struct Packet
+        {
+            char*                   m_data;
+            unsigned int            m_dataSize;
+
+            AZStd::intrusive_ptr<DriverAddress> m_address;  // To or From address
+            TimeStamp               m_startTime;
+            unsigned int            m_latency;  /// time to action from startTime mark in milliseconds.
+        };
+
+        AZStd::mutex    m_lock;
+
+        /// Free all internal data
+        void FreeAllData();
+
+        volatile bool   m_enable;
+
+        // Latency in milliseconds
+        unsigned int    m_minOutLatency;
+        unsigned int    m_maxOutLatency;
+        unsigned int    m_minInLatency;
+        unsigned int    m_maxInLatency;
+
+        // Packet loss 1 every X packets
+        unsigned int    m_minOutPacketLoss;
+        unsigned int    m_maxOutPacketLoss;
+        volatile bool   m_isOutPacketLoss;
+        unsigned int    m_minInPacketLoss;
+        unsigned int    m_maxInPacketLoss;
+        volatile bool   m_isInPacketLoss;
+
+        unsigned int    m_numOutPacketsTillDrop;
+        unsigned int    m_numInPacketsTillDrop;
+
+        // Packet loss drop for X ms every Y ms
+        unsigned int    m_minInPacketDropInterval;  ///< Min interval to drop packets in milliseconds for inbound packets.
+        unsigned int    m_maxInPacketDropInterval;  ///< Max interval to drop packets in milliseconds for inbound packets.
+        unsigned int    m_minInPacketDropPeriod;    ///< Min period for packet drop in milliseconds for inbound packets. We drop packets for DropInterval every DropPeriod.
+        unsigned int    m_maxInPacketDropPeriod;    ///< Max -- " --
+        unsigned int    m_minOutPacketDropInterval; ///< Min interval to drop packets in milliseconds for outbound packets.
+        unsigned int    m_maxOutPacketDropInterval; ///< Max -- " --
+        unsigned int    m_minOutPacketDropPeriod;   ///< Min period for packet drop in milliseconds for inbound packets. We drop packets for DropInterval every DropPeriod.
+        unsigned int    m_maxOutPacketDropPeriod;   ///< Max -- " --
+
+        unsigned int    m_inPacketDropInterval;     ///< If interval is != 0 we are currently dropping packets. Interval value is in milliseconds.
+        unsigned int    m_inPacketDropPeriod;       ///< Milliseconds left until net simulation interval in milliseconds inbound.
+        unsigned int    m_outPacketDropInterval;    ///< If interval is != 0 we are currently dropping packets. Interval value is in milliseconds.
+        unsigned int    m_outPacketDropPeriod;      ///< Milliseconds left until net simulation interval in milliseconds outbound.
+
+        // Bandwidth in Kbps
+        unsigned int    m_minOutBandwidth;
+        unsigned int    m_maxOutBandwidth;
+        unsigned int    m_minInBandwidth;
+        unsigned int    m_maxInBandwidth;
+
+        unsigned int    m_currentDataOut;           ///< How much data have we send since m_dataLimiterTimeout was reset in bytes.
+        unsigned int    m_currentDataOutMax;        ///< What is the current output data limit till m_dataLimiterTimeout is reset in bytes.
+        unsigned int    m_currentDataIn;            ///< How much data have we received since m_dataLimiterTimeout was reset in bytes.
+        unsigned int    m_currentDataInMax;         ///< What is the current incoming data limit till m_dataLimiterTimeout is reset in bytes.
+
+        unsigned int    m_dataLimiterTimeout;       ///< Time since we counting currentDataXXX limits.
+
+        volatile bool   m_outReorder;
+        volatile bool   m_inReorder;
+
+        typedef list<Packet> PacketListType;
+
+        PacketListType  m_outgoing;
+        PacketListType  m_incoming;
+
+        TimeStamp       m_currentTime;              ///< Current time. (replace this with a global clock when possible.)
+
+        class Driver*   m_driver;
+    };
+}
+
+#endif // GM_DEFAULT_SIMULATOR_H

Code/Framework/GridMate/GridMate/Carrier/DefaultTrafficControl.cpp

@@ -0,0 +1,720 @@
+/*
+* All or portions of this file Copyright (c) Amazon.com, Inc. or its affiliates or
+* its licensors.
+*
+* For complete copyright and license terms please see the LICENSE at the root of this
+* distribution (the "License"). All use of this software is governed by the License,
+* or, if provided, by the license below or the license accompanying this file. Do not
+* remove or modify any license notices. This file is distributed on an "AS IS" BASIS,
+* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+*
+*/
+
+#include <GridMate/Carrier/DefaultTrafficControl.h>
+#include <GridMate/Carrier/Carrier.h>
+#include <GridMate/Carrier/Driver.h>
+
+#include <AzCore/Math/Crc.h>
+#include <AzCore/Math/MathUtils.h>
+
+using namespace GridMate;
+
+//#define GRIDMATE_FIXED_RATE_BYTES 1000000
+//#define VERBOSE_DISCONNECT_DEBUGGING
+
+//////////////////////////////////////////////////////////////////////////
+// TCP Cubic parameters
+const double DefaultTrafficControl::ConnectionData::k_CubicBeta     = 0.2;                                  ///< beta (backoff rate) for congestion calculations
+const double DefaultTrafficControl::ConnectionData::k_CubicAlpha    = 3 * k_CubicBeta / (2 - k_CubicBeta);  ///< alpha for TCP friendly window estimation
+const double DefaultTrafficControl::ConnectionData::k_CubicScaleC   = 0.4;                                  ///< Scaling factor. COTS default (0.4)
+//////////////////////////////////////////////////////////////////////////
+
+//=========================================================================
+// DefaultTrafficControl
+// [10/5/2010]
+//=========================================================================
+DefaultTrafficControl::DefaultTrafficControl(unsigned int maxSystemPacketSize, float rttConnectionThreshold, float packetLossThreshold, unsigned int maxRecvPackets)
+    : m_maxPacketSize(maxSystemPacketSize)
+    , m_rttConnectionThreshold(rttConnectionThreshold)  /// in milliseconds
+    , m_packetLossThreshold(packetLossThreshold)        /// percent 1.0 is 100%
+    , m_lastStatDataReset(0.0f)
+    , m_lostPacketTimeoutMS(1000)
+    , m_defaultMaxCongestionWindowSize(1000000)
+    , m_maxRecvPackets(maxRecvPackets)
+{
+    AZ_Assert(maxSystemPacketSize >= 256, "Maximum system packet is too small!");
+    m_currentTime = AZStd::chrono::system_clock::now();
+}
+
+//=========================================================================
+// DefaultTrafficControl
+// [10/5/2010]
+//=========================================================================
+DefaultTrafficControl::~DefaultTrafficControl()
+{
+}
+
+//=========================================================================
+// OnConnect
+// [10/5/2010]
+//=========================================================================
+void
+DefaultTrafficControl::OnConnect(TrafficControlConnectionId id, const AZStd::intrusive_ptr<DriverAddress>& address)
+{
+    AZ_Assert(id->m_trafficData == NULL, "We have already assigned traffic data to this connection!");
+    if (id->m_trafficData != NULL)
+    {
+        return;
+    }
+
+    ConnectionData cd;
+
+    //cd.mode = ConnectionData::Bad;
+    //cd.penaltyTime = 4.0f;
+
+    // to avoid measuring stats while handshake is done, get a stamp in the future (after any possible handshake)
+    cd.m_handshakeDone = m_currentTime + AZStd::chrono::minutes(60);
+    cd.m_isReceivedDataAfterLastSend = false;
+    cd.m_address = address->ToAddress();
+    cd.m_recvPacketAllowance = m_maxRecvPackets;
+    cd.m_canReceiveData = true;
+
+    cd.m_lastAckSend = m_currentTime;
+
+    //////////////////////////////////////////////////////////////////////////
+    // Slow start traffic control
+    cd.m_lastWindowSizeIncrease = m_currentTime;
+    cd.m_lastWindowSizeDecrease = m_currentTime;
+    cd.m_inTransfer = 0;
+    cd.m_slowStartThreshold = 0;
+    cd.m_congestionWindow = m_maxPacketSize;
+    cd.m_maxCongestionWindow = m_defaultMaxCongestionWindowSize;
+    //////////////////////////////////////////////////////////////////////////
+
+    m_connections.push_back(cd);
+    id->m_trafficData = &m_connections.back();
+}
+
+//=========================================================================
+// OnDisconnect
+// [10/12/2010]
+//=========================================================================
+void
+DefaultTrafficControl::OnDisconnect(TrafficControlConnectionId id)
+{
+    ConnectionData* cd = reinterpret_cast<ConnectionData*>(id->m_trafficData);
+    id->m_trafficData = NULL;
+    bool isFound = false;
+    for (ConnectionListType::iterator i = m_connections.begin(); i != m_connections.end(); ++i)
+    {
+        if (&*i == cd)
+        {
+            m_connections.erase(i);
+            isFound = true;
+            break;
+        }
+    }
+    (void)isFound;
+    AZ_Assert(isFound, "Traffic control data is NOT in the list!");
+}
+
+//=========================================================================
+// OnHandshakeComplete
+// [2/22/2011]
+//=========================================================================
+void
+DefaultTrafficControl::OnHandshakeComplete(TrafficControlConnectionId id)
+{
+    // because the NAT punch can stall the connection, out packed loss
+    // can be really high after the connection is established. This can
+    // produce an bad connection situation.
+    // for now we just reset the stats. Another option will be to just not update stats at all.
+    ConnectionData* cd = reinterpret_cast<ConnectionData*>(id->m_trafficData);
+    cd->m_handshakeDone = m_currentTime;
+}
+
+//=========================================================================
+// OnSend
+// [10/5/2010]
+//=========================================================================
+void
+DefaultTrafficControl::OnSend(TrafficControlConnectionId id, DataGramControlData& info)
+{
+    info.m_time = m_currentTime;
+
+    ConnectionData* cd = reinterpret_cast<ConnectionData*>(id->m_trafficData);
+
+    cd->m_sdCurrentSecond.m_dataSend += info.m_size;
+    cd->m_sdCurrentSecond.m_packetSend++;
+    if (info.m_effectiveSize)
+    {
+        cd->m_sdEffectiveCurrentSecond.m_dataSend += info.m_effectiveSize;
+        cd->m_sdEffectiveCurrentSecond.m_packetSend++;
+    }
+    cd->m_lastPacketSend = info.m_time;
+    cd->m_isReceivedDataAfterLastSend = false;
+
+    cd->m_inTransfer += info.m_size;
+
+#if defined(AZ_DEBUG_BUILD)
+    //{
+    //    static GridMate::TimeStamp last_time = m_currentTime;
+    //    if( (m_currentTime - last_time).count() > 1000000) //1000ms
+    //    {
+    //        last_time = m_currentTime;
+    //        AZ_TracePrintf("GridMate", "Sent: CWND %u sent %u rcvd %u\n",
+    //           cd->m_congestionWindow, cd->m_sdLifetime.m_packetSend, cd->m_sdLifetime.m_packetReceived);
+    //    }
+    //}
+#endif
+}
+
+//=========================================================================
+// OnSendAck
+// [8/1/2012]
+//=========================================================================
+void
+DefaultTrafficControl::OnSendAck(TrafficControlConnectionId id)
+{
+    ConnectionData* cd = reinterpret_cast<ConnectionData*>(id->m_trafficData);
+    cd->m_lastAckSend = m_currentTime;
+}
+
+//=========================================================================
+// OnAck
+// [10/5/2010]
+//=========================================================================
+void
+DefaultTrafficControl::OnAck(TrafficControlConnectionId id, DataGramControlData& info, bool& windowChanged)
+{
+    ConnectionData* cd = reinterpret_cast<ConnectionData*>(id->m_trafficData);
+    cd->m_sdCurrentSecond.m_dataAcked += info.m_size;
+    cd->m_sdCurrentSecond.m_packetAcked++;
+    if (info.m_effectiveSize)
+    {
+        cd->m_sdEffectiveCurrentSecond.m_dataAcked += info.m_effectiveSize;
+        cd->m_sdEffectiveCurrentSecond.m_packetAcked++;
+    }
+
+    if (info.m_time >= cd->m_handshakeDone)   // we measure after the handshake is done \ref OnHandshakeComplete
+    {
+        // compute the packet rtt
+        AZStd::chrono::microseconds rtt =  m_currentTime - info.m_time;
+        cd->m_sdCurrentSecond.m_rtt = (cd->m_sdCurrentSecond.m_rtt + .001f * rtt.count()) * 0.5f;
+        if (info.m_effectiveSize)
+        {
+            cd->m_sdEffectiveCurrentSecond.m_rtt = (cd->m_sdEffectiveCurrentSecond.m_rtt + .001f * rtt.count()) * 0.5f;
+        }
+    }
+
+    AZ_Assert(cd->m_inTransfer >= info.m_size, "Invalid data size");
+    cd->m_inTransfer -= info.m_size;
+
+    // Traffic control - packets are getting acked, increase rate
+    if (m_currentTime != cd->m_lastWindowSizeIncrease)
+    {
+        cd->m_lastWindowSizeIncrease = m_currentTime;
+
+        bool isSlowStart = cd->m_congestionWindow <= cd->m_slowStartThreshold || cd->m_slowStartThreshold == 0;
+        if (isSlowStart)
+        {
+            cd->m_congestionWindow = AZStd::GetMin(cd->m_congestionWindow * 2, cd->m_maxCongestionWindow);
+            if (cd->m_congestionWindow > cd->m_slowStartThreshold && cd->m_slowStartThreshold != 0)
+            {
+                cd->m_congestionWindow = cd->m_slowStartThreshold;
+                if (cd->k_enableCubic)
+                {
+                    cd->m_congestionWindow += m_maxPacketSize;
+                    cd->TCPCubicExitSlowStart(m_currentTime, m_maxPacketSize);
+                }
+                else
+                {
+                    cd->m_congestionWindow += m_maxPacketSize * m_maxPacketSize / cd->m_congestionWindow;
+                }
+            }
+        }
+        else
+        {
+            if (cd->k_enableCubic)
+            {
+                cd->m_congestionWindow = cd->TCPCubicWindow(m_currentTime, m_maxPacketSize);
+            }
+            else
+            {
+                cd->m_congestionWindow += m_maxPacketSize * m_maxPacketSize / cd->m_congestionWindow;
+            }
+        }
+
+        cd->m_congestionWindow = AZStd::GetMin(cd->m_congestionWindow, cd->m_maxCongestionWindow);
+
+#if defined(GRIDMATE_FIXED_RATE_BYTES) && GRIDMATE_FIXED_RATE_BYTES > 0
+        //const float rtt = cd->m_sdLifetime.m_rtt > 1.f ? cd->m_sdLifetime.m_rtt : 100.f;
+        cd->m_sdLifetime.m_rtt = 100;
+        cd->m_congestionWindow = static_cast<unsigned int>(GRIDMATE_FIXED_RATE_BYTES / 10);
+#endif
+        windowChanged = true;
+    }
+    else
+    {
+        windowChanged = false;
+    }
+}
+
+//=========================================================================
+// OnNAck
+// [7/30/2012]
+//=========================================================================
+void
+DefaultTrafficControl::OnNAck(TrafficControlConnectionId id, DataGramControlData& info)
+{
+    (void)id;
+    // If we get N number of NAck consider that packet lost. N should be not 1 to allow for packet
+    // reordering.
+    const unsigned int N = 3;
+    info.m_time -= AZStd::chrono::milliseconds(m_lostPacketTimeoutMS / N); // decrease the time for each packet, so after N NACKs we consider the packet lost.
+}
+
+//=========================================================================
+// OnReceived
+// [11/15/2010]
+//=========================================================================
+void
+DefaultTrafficControl::OnReceived(TrafficControlConnectionId id, DataGramControlData& info)
+{
+    (void)info;
+    ConnectionData* cd = reinterpret_cast<ConnectionData*>(id->m_trafficData);
+
+    cd->m_sdCurrentSecond.m_dataReceived += info.m_size;
+    cd->m_sdCurrentSecond.m_packetReceived++;
+    if (info.m_effectiveSize)
+    {
+        cd->m_sdEffectiveCurrentSecond.m_dataReceived += info.m_effectiveSize;
+        cd->m_sdEffectiveCurrentSecond.m_packetReceived++;
+        cd->m_isReceivedDataAfterLastSend = true;
+    }
+
+    if (m_maxRecvPackets != 0)
+    {
+        --cd->m_recvPacketAllowance;
+        if (cd->m_recvPacketAllowance == 0) // hit the limit -> let's blacklist connection
+        {
+            cd->m_canReceiveData = false;
+        }
+    }
+
+#if defined(AZ_DEBUG_BUILD)
+    //{
+    //    static GridMate::TimeStamp last_time = m_currentTime;
+    //    if ((m_currentTime - last_time).count() > 1000000) //1000ms
+    //    {
+    //        last_time = m_currentTime;
+    //        AZ_TracePrintf("GridMate", "Rcvd: CWND %u sent %u rcvd %u\n",
+    //            cd->m_congestionWindow, cd->m_sdLifetime.m_packetSend, cd->m_sdLifetime.m_packetReceived);
+    //    }
+    //}
+#endif
+}
+
+//=========================================================================
+// IsSend
+// [10/5/2010]
+//=========================================================================
+bool
+DefaultTrafficControl::IsSend(TrafficControlConnectionId id)
+{
+    ConnectionData* cd = reinterpret_cast<ConnectionData*>(id->m_trafficData);
+
+    return cd->m_inTransfer <= cd->m_congestionWindow;
+}
+
+//=========================================================================
+// IsSendAck
+// [8/1/2012]
+//=========================================================================
+bool
+DefaultTrafficControl::IsSendAck(TrafficControlConnectionId id)
+{
+    ConnectionData* cd = reinterpret_cast<ConnectionData*>(id->m_trafficData);
+    if (cd->m_lastAckSend != m_currentTime)
+    {
+        return true;
+    }
+    return false;
+}
+
+//=========================================================================
+// GetAvailableWindowSize
+// [10/6/2010]
+//=========================================================================
+unsigned int
+DefaultTrafficControl::GetAvailableWindowSize(TrafficControlConnectionId id) const
+{
+    ConnectionData* cd = reinterpret_cast<ConnectionData*>(id->m_trafficData);
+    return cd->m_congestionWindow - cd->m_inTransfer;
+}
+
+TimeStamp
+DefaultTrafficControl::GetResendTime(TrafficControlConnectionId id, const DataGramControlData& info)
+{
+    (void)id;
+    return info.m_time + AZStd::chrono::milliseconds(m_lostPacketTimeoutMS + 1);
+}
+//=========================================================================
+// IsResend
+// [10/5/2010]
+//=========================================================================
+bool
+DefaultTrafficControl::IsResend(TrafficControlConnectionId id, const DataGramControlData& info, unsigned int resendDataSize)
+{
+    AZStd::chrono::milliseconds timeElapsed = m_currentTime - info.m_time;
+
+    // Consider a packet lost after certain amount of time.
+    if (timeElapsed.count() > m_lostPacketTimeoutMS)
+    {
+        ConnectionData* cd = reinterpret_cast<ConnectionData*>(id->m_trafficData);
+        if (info.m_time >= cd->m_handshakeDone)   // we measure after the handshake is done \ref OnHandshakeComplete
+        {
+            cd->m_sdCurrentSecond.m_packetLost++;
+            // how should we affect the rtt when a packet is lost ?
+            //cd->sdCurrentSecond.rtt = (cd->sdCurrentSecond.rtt + static_cast<float>(timeElapsed.count())) * 0.5f;
+            if (resendDataSize)
+            {
+                cd->m_sdEffectiveCurrentSecond.m_packetLost++;
+            }
+
+            if (cd->k_enableCubic)
+            {
+                cd->TCPCubicPacketLost(m_currentTime, m_maxPacketSize);
+            }
+            else
+            {
+                // Traffic control - we lost a packet, send less data
+                if (m_currentTime != cd->m_lastWindowSizeDecrease)
+                {
+                    cd->m_lastWindowSizeDecrease = m_currentTime;
+                    //cd->m_lastWindowSizeIncrease = m_currentTime; // don't allow increase
+                    //cd->m_slowStartThreshold = AZStd::GetMax(cd->m_congestionWindow/2,m_maxPacketSize);
+                    //cd->m_congestionWindow = m_maxPacketSize;
+                }
+            }
+        }
+        cd->m_inTransfer -= info.m_size;
+
+#if defined(GRIDMATE_FIXED_RATE_BYTES) && GRIDMATE_FIXED_RATE_BYTES > 0
+        //const float rtt = cd->m_sdLifetime.m_rtt > 1.f ? cd->m_sdLifetime.m_rtt : 100.f;
+        cd->m_sdLifetime.m_rtt = 100;
+        cd->m_congestionWindow = static_cast<unsigned int>(GRIDMATE_FIXED_RATE_BYTES / 10);
+#endif
+
+        return true;
+    }
+
+    return false;
+}
+
+//=========================================================================
+// OnResend
+// [10/5/2010]
+//=========================================================================
+void
+DefaultTrafficControl::OnReSend(TrafficControlConnectionId id, DataGramControlData& info, unsigned int resendDataSize)
+{
+    (void)info;
+    ConnectionData* cd = reinterpret_cast<ConnectionData*>(id->m_trafficData);
+    cd->m_sdCurrentSecond.m_dataResend += resendDataSize;
+    cd->m_sdEffectiveCurrentSecond.m_dataResend += resendDataSize;
+}
+
+//=========================================================================
+// IsDisconnect
+// [11/15/2010]
+//=========================================================================
+bool
+DefaultTrafficControl::IsDisconnect(TrafficControlConnectionId id, float conditionThreshold)
+{
+    AZ_Assert(conditionThreshold >= 0.0f && conditionThreshold <= 1.0f, "Invalid condition threshold!");
+    ConnectionData* cd = reinterpret_cast<ConnectionData*>(id->m_trafficData);
+
+    if (cd->m_sdLifetime.m_connectionFactor >= conditionThreshold)
+    {
+#ifdef VERBOSE_DISCONNECT_DEBUGGING
+        AZ_TracePrintf("GridMate", "Connection %p rtt %.2f ms (max. %.2f) and packetLoss %.2f (max. %.2f).\n", id, cd->m_sdLifetime.m_rtt, m_rttConnectionThreshold * conditionThreshold, cd->m_sdLifetime.m_packetLoss, m_packetLossThreshold * conditionThreshold);
+#endif
+        return true;
+    }
+
+    return false;
+}
+
+bool DefaultTrafficControl::IsCanReceiveData(TrafficControlConnectionId id) const
+{
+    ConnectionData* cd = reinterpret_cast<ConnectionData*>(id->m_trafficData);
+    return cd->m_canReceiveData;
+}
+
+//=========================================================================
+// IsSendACKOnly
+// [6/5/2012]
+//=========================================================================
+bool
+DefaultTrafficControl::IsSendACKOnly(TrafficControlConnectionId id) const
+{
+    ConnectionData* cd = reinterpret_cast<ConnectionData*>(id->m_trafficData);
+
+    // if we have received any data after the last send (which contains an ACK)
+    // we need to send an ACK only packet to confirm receiving the data.
+    if (cd->m_isReceivedDataAfterLastSend || m_currentTime - cd->m_lastAckSend > AZStd::chrono::milliseconds(m_lostPacketTimeoutMS / 10))
+    {
+        return true;
+    }
+
+    return false;
+}
+
+
+//=========================================================================
+// Update
+// [10/12/2010]
+//=========================================================================
+bool
+DefaultTrafficControl::Update()
+{
+    TimeStamp now = AZStd::chrono::system_clock::now();
+    //////////////////////////////////////////////////////////////////////////
+    // Or we can deliver this from the engine
+    float deltaTime = AZStd::chrono::duration<float>(now - m_currentTime).count();
+    //////////////////////////////////////////////////////////////////////////
+    m_currentTime = now;
+
+    m_lastStatDataReset += deltaTime;
+
+    if (m_lastStatDataReset >= 1.0f)
+    {
+        m_lastStatDataReset -= 1.0f;
+    }
+    else
+    {
+        return false; ///< We update our mode once per second.
+    }
+
+    for (ConnectionListType::iterator iConn = m_connections.begin(); iConn != m_connections.end(); ++iConn)
+    {
+        ConnectionData& cd = *iConn;
+
+        cd.m_recvPacketAllowance = m_maxRecvPackets; // adding new allowance of recv'ed packets
+
+        //////////////////////////////////////////////////////////////////////////
+        // all data
+        cd.m_sdLastSecond = cd.m_sdCurrentSecond;
+
+        // update the lifetime stats
+        cd.m_sdLifetime.m_dataSend += cd.m_sdLastSecond.m_dataSend;
+        cd.m_sdLifetime.m_dataReceived += cd.m_sdLastSecond.m_dataReceived;
+
+        cd.m_sdLifetime.m_dataAcked += cd.m_sdLastSecond.m_dataAcked;
+        cd.m_sdLifetime.m_dataResend += cd.m_sdLastSecond.m_dataResend;
+
+        cd.m_sdLifetime.m_packetSend += cd.m_sdLastSecond.m_packetSend;
+        cd.m_sdLifetime.m_packetReceived += cd.m_sdLastSecond.m_packetReceived;
+
+        cd.m_sdLifetime.m_packetAcked += cd.m_sdLastSecond.m_packetAcked;
+        cd.m_sdLifetime.m_packetLost += cd.m_sdLastSecond.m_packetLost;
+
+        // smooth the packet loss
+        cd.m_sdLifetime.m_avgPacketSend += (static_cast<float>(cd.m_sdLastSecond.m_packetSend) - cd.m_sdLifetime.m_avgPacketSend) * 0.1f;
+        cd.m_sdLifetime.m_avgPacketLost += (static_cast<float>(cd.m_sdLastSecond.m_packetLost) - cd.m_sdLifetime.m_avgPacketLost) * 0.1f;
+        cd.m_sdLastSecond.m_packetLoss = cd.m_sdLifetime.m_avgPacketLost / (cd.m_sdLifetime.m_avgPacketSend + 0.00001f /*eps*/);
+        cd.m_sdLastSecond.m_packetLoss = AZStd::min AZ_PREVENT_MACRO_SUBSTITUTION (cd.m_sdLastSecond.m_packetLoss, 1.0f);
+
+        // RTT value will be incorrect (from the previous interval) if we did not received any acks.
+        if (cd.m_sdLastSecond.m_packetAcked == 0)
+        {
+            if (cd.m_sdLastSecond.m_packetLost == 0)
+            {
+                cd.m_sdLastSecond.m_rtt = 0.0f;  // if we did not lost any packets assume we just did not send anything
+            }
+            else
+            {
+                // we don't really know what the RTT is, it's technically infinity. But that's why we have packetLoss too, so keep it the same
+                cd.m_sdLastSecond.m_rtt = cd.m_sdLifetime.m_rtt /*blockedConnectionRTT*/;
+                //AZ_TracePrintf("GridMate", "Traffic control: We did not received any packets for the last second %llu!\n\n", AZStd::GetTimeUTCMilliSecond());
+            }
+        }
+
+        // smooth out the average rtt
+        cd.m_sdLifetime.m_rtt += (cd.m_sdLastSecond.m_rtt - cd.m_sdLifetime.m_rtt) * 0.1f;
+        // We already smooth the m_packetLost over the last 10 seconds cd.m_sdLifetime.m_packetLoss += (cd.m_sdLastSecond.m_packetLoss - cd.m_sdLifetime.m_packetLoss) * 0.1f;
+        cd.m_sdLifetime.m_packetLoss = cd.m_sdLastSecond.m_packetLoss;
+        //cd.m_sdLifetime.m_flow += (cd.m_sdLastSecond.m_flow - cd.m_sdLifetime.m_flow) * 0.1f;
+        cd.m_sdLifetime.m_connectionFactor = AZStd::max AZ_PREVENT_MACRO_SUBSTITUTION (cd.m_sdLifetime.m_rtt / m_rttConnectionThreshold, cd.m_sdLifetime.m_packetLoss / m_packetLossThreshold);
+
+        //////////////////////////////////////////////////////////////////////////
+        // effective data
+        cd.m_sdEffectiveLastSecond = cd.m_sdEffectiveCurrentSecond;
+
+        // update the lifetime stats
+        cd.m_sdEffectiveLifetime.m_dataSend += cd.m_sdEffectiveLastSecond.m_dataSend;
+        cd.m_sdEffectiveLifetime.m_dataReceived += cd.m_sdEffectiveLastSecond.m_dataReceived;
+
+        cd.m_sdEffectiveLifetime.m_dataAcked += cd.m_sdEffectiveLastSecond.m_dataAcked;
+        cd.m_sdEffectiveLifetime.m_dataResend += cd.m_sdEffectiveLastSecond.m_dataResend;
+
+        cd.m_sdEffectiveLifetime.m_packetSend += cd.m_sdEffectiveLastSecond.m_packetSend;
+        cd.m_sdEffectiveLifetime.m_packetReceived += cd.m_sdEffectiveLastSecond.m_packetReceived;
+
+        cd.m_sdEffectiveLifetime.m_packetAcked += cd.m_sdEffectiveLastSecond.m_packetAcked;
+        cd.m_sdEffectiveLifetime.m_packetLost += cd.m_sdEffectiveLastSecond.m_packetLost;
+
+        // smooth the packet loss
+        cd.m_sdEffectiveLifetime.m_avgPacketSend += (static_cast<float>(cd.m_sdEffectiveLastSecond.m_packetSend) - cd.m_sdEffectiveLifetime.m_avgPacketSend) * 0.1f;
+        cd.m_sdEffectiveLifetime.m_avgPacketLost += (static_cast<float>(cd.m_sdEffectiveLastSecond.m_packetLost) - cd.m_sdEffectiveLifetime.m_avgPacketLost) * 0.1f;
+        cd.m_sdEffectiveLastSecond.m_packetLoss = cd.m_sdEffectiveLifetime.m_avgPacketLost / (cd.m_sdEffectiveLifetime.m_avgPacketSend + 0.00001f /*eps*/);
+        cd.m_sdEffectiveLastSecond.m_packetLoss = AZStd::min AZ_PREVENT_MACRO_SUBSTITUTION (cd.m_sdEffectiveLastSecond.m_packetLoss, 1.0f);
+
+        // RTT value will be incorrect (from the previous interval) if we did not received any acks.
+        if (cd.m_sdEffectiveLastSecond.m_packetAcked == 0)
+        {
+            if (cd.m_sdEffectiveLastSecond.m_packetLost == 0)
+            {
+                cd.m_sdEffectiveLastSecond.m_rtt = 0.0f;  // if we did not lost any packets assume we just did not send anything
+            }
+            else
+            {
+                // we don't really know what the RTT is, it's technically infinity. But that's why we have packetLoss too, so keep it the same
+                cd.m_sdEffectiveLastSecond.m_rtt = cd.m_sdEffectiveLifetime.m_rtt /*blockedConnectionRTT*/;
+            }
+        }
+
+        // smooth out the average rtt
+        cd.m_sdEffectiveLifetime.m_rtt += (cd.m_sdEffectiveLastSecond.m_rtt - cd.m_sdEffectiveLifetime.m_rtt) * 0.1f;
+        // We already smooth the m_packetLost over the last 10 seconds cd.m_sdEffectiveLifetime.m_packetLoss += (cd.m_sdEffectiveLastSecond.m_packetLoss - cd.m_sdEffectiveLifetime.m_packetLoss) * 0.1f;
+        cd.m_sdEffectiveLifetime.m_packetLoss = cd.m_sdEffectiveLastSecond.m_packetLoss;
+        //////////////////////////////////////////////////////////////////////////
+
+        //AZ_TracePrintf("GridMate","Traffic control: Connection %s LifeTime(rtt %.2f packetLoss %.2f) LastSecond(rtt %.2f packetLoss %.2f)\n",
+        //  cd.m_address.c_str(),cd.m_sdLifetime.m_rtt,cd.m_sdLifetime.m_packetLoss,cd.m_sdLastSecond.m_rtt,cd.m_sdLastSecond.m_packetLoss);
+
+        // send new statistics event
+        EBUS_EVENT(Debug::CarrierDrillerBus, OnUpdateStatistics, cd.m_address, cd.m_sdLastSecond, cd.m_sdLifetime, cd.m_sdEffectiveLastSecond, cd.m_sdEffectiveLifetime);
+
+        cd.m_sdCurrentSecond.Reset();
+        cd.m_sdCurrentSecond.m_rtt = cd.m_sdLastSecond.m_rtt;
+        //cd.sdCurrentSecond.flow = 1.0f; // Good
+
+        cd.m_sdEffectiveCurrentSecond.Reset();
+        cd.m_sdEffectiveCurrentSecond.m_rtt = cd.m_sdEffectiveLastSecond.m_rtt;
+    }
+
+    return true;
+}
+
+//=========================================================================
+// QueryStatistics
+// [11/11/2010]
+//=========================================================================
+void
+DefaultTrafficControl::QueryStatistics(TrafficControlConnectionId id, Statistics* lastSecond, Statistics* lifetime, Statistics* effectiveLastSecond, Statistics* effectiveLifetime) const
+{
+    ConnectionData* cd = reinterpret_cast<ConnectionData*>(id->m_trafficData);
+
+    if (lastSecond)
+    {
+        *lastSecond = cd->m_sdLastSecond;
+    }
+    if (lifetime)
+    {
+        *lifetime = cd->m_sdLifetime;
+    }
+    if (effectiveLastSecond)
+    {
+        *effectiveLastSecond = cd->m_sdEffectiveLastSecond;
+    }
+    if (effectiveLifetime)
+    {
+        *effectiveLifetime = cd->m_sdEffectiveLifetime;
+    }
+}
+
+//=========================================================================
+// QueryStatistics
+//
+//=========================================================================
+void
+DefaultTrafficControl::QueryCongestionState(TrafficControlConnectionId id, CongestionState* congestionState) const
+{
+    ConnectionData* cd = reinterpret_cast<ConnectionData*>(id->m_trafficData);
+    AZ_Warning("GridMate", congestionState != nullptr, "Invalid congestion state, data will not be set!");
+    if (congestionState)
+    {
+        congestionState->m_dataInTransfer = cd->m_inTransfer;
+        congestionState->m_congestionWindow = cd->m_congestionWindow;
+    }
+}
+
+unsigned int
+DefaultTrafficControl::ConnectionData::TCPCubicWindow(TimeStamp& now, unsigned int packetSize) const
+{
+    // Ref: "CUBIC: a new TCP-friendly high-speed TCP variant" http://dl.acm.org/citation.cfm?id=1400105
+    const double seconds = (now - m_lastWindowSizeDecrease).count() / 1000000.0;                    // Seconds since last backoff
+
+    // For very short RTT's TCP-Reno is more aggressive, so we use TCP-Reno's window as a floor
+    const unsigned int friendlyWindow = TCPRenoWindow(seconds);
+
+    // Calculate window using TCP-Cubic formula
+    // With a floor of MAX( TCP-Reno, our configured minimum )
+    const unsigned int window = AZStd::GetMax(static_cast<unsigned int>(k_CubicScaleC*pow((seconds - m_CubicK), 3)*packetSize + m_preBackoffCongestionWindow),
+    AZStd::GetMax(friendlyWindow, k_MinCongestionWindowPackets * packetSize));
+
+    return window;
+}
+void
+DefaultTrafficControl::ConnectionData::TCPCubicCalcK(unsigned int packetSize)
+{
+    m_CubicKcube = m_preBackoffCongestionWindow / ((k_CubicScaleC / k_CubicBeta)*packetSize);   // Use window size, scaling factor and backoff rate to calc m_CubicK^3
+    m_CubicK = pow(m_CubicKcube, 1.0 / 3);                                                           // Cubic inflection point, m_CubicK, in seconds
+}
+void
+DefaultTrafficControl::ConnectionData::TCPCubicPacketLost(TimeStamp& now, unsigned int packetSize)
+{
+    using AZStd::chrono::milliseconds;
+    static const unsigned int backoffRate100 = static_cast<unsigned int>((1 - k_CubicBeta) * 100);     // Backoff Rate * 100 for integer calculations
+
+    // Wait 1 RTT before allowing another backoff, plus 10ms buffer to cover jitter
+    if (now > (m_lastWindowSizeDecrease + milliseconds(static_cast<int>(m_sdLifetime.m_rtt + 10))))
+    {
+        TCPCubicCalcK(packetSize);
+        m_lastWindowSizeDecrease = now;
+        m_lastWindowSizeIncrease = now;
+        m_preBackoffCongestionWindow = m_congestionWindow;
+        m_congestionWindow = AZStd::GetMax(backoffRate100 * m_congestionWindow / 100, k_MinCongestionWindowPackets * packetSize);   // Back off but not less than minimum
+        m_slowStartThreshold = m_congestionWindow - packetSize;                                                                     // Store for idle recovery
+        //AZ_TracePrintf("GridMate", "lost CWND old %u time %f w_max %u K %f avgPER %f\n",
+        //    m_congestionWindow / packetSize, ((now - m_lastWindowSizeDecrease).count() / 1000000.0), m_preBackoffCongestionWindow / packetSize, m_CubicK, m_sdLifetime.m_avgPacketLost);
+    }
+}
+
+void
+DefaultTrafficControl::ConnectionData::TCPCubicExitSlowStart(TimeStamp& now, unsigned int packetSize)
+{
+    TCPCubicCalcK(packetSize);
+    m_lastWindowSizeDecrease = now;
+    m_lastWindowSizeIncrease = now;
+    m_lastWindowSizeDecrease -= AZStd::chrono::milliseconds(static_cast<unsigned int>(m_CubicK * 1000));    // start at the inflection point
+    m_preBackoffCongestionWindow = m_congestionWindow;
+    m_slowStartThreshold = m_congestionWindow - packetSize;                                                 // Store for idle recovery
+    //AZ_TracePrintf("GridMate", " exSS cwnd %u time %f w_max %u K %f avgPER %f\n",
+    //    m_congestionWindow / packetSize, ((now - m_lastWindowSizeDecrease).count() / 1000000.0), m_preBackoffCongestionWindow / packetSize, m_CubicK, m_sdLifetime.m_avgPacketLost);
+}
+
+unsigned int
+DefaultTrafficControl::ConnectionData::TCPRenoWindow(double seconds) const
+{
+    const double rtt = m_sdLifetime.m_rtt;
+    const double backoffWindow = (1 - k_CubicBeta)*m_preBackoffCongestionWindow;
+    if ( ! AZ::IsNormalDouble(rtt) )
+    {
+        return static_cast<unsigned int>(backoffWindow); // Unable to predict with non-normal RTT; shouldn't happen
+    }
+    const double cwnd = (backoffWindow + k_CubicAlpha * seconds / rtt);
+    return static_cast<unsigned int>(cwnd);
+}

Code/Framework/GridMate/GridMate/Carrier/DefaultTrafficControl.h

@@ -0,0 +1,198 @@
+/*
+* All or portions of this file Copyright (c) Amazon.com, Inc. or its affiliates or
+* its licensors.
+*
+* For complete copyright and license terms please see the LICENSE at the root of this
+* distribution (the "License"). All use of this software is governed by the License,
+* or, if provided, by the license below or the license accompanying this file. Do not
+* remove or modify any license notices. This file is distributed on an "AS IS" BASIS,
+* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+*
+*/
+#ifndef GM_DEFAULT_TRAFFIC_CONTROL_H
+#define GM_DEFAULT_TRAFFIC_CONTROL_H
+
+#include <GridMate/Carrier/TrafficControl.h>
+
+#include <GridMate/Containers/list.h>
+
+namespace GridMate
+{
+    /**
+    * Traffic control default implementation.
+    */
+    class DefaultTrafficControl
+        : public TrafficControl
+    {
+    public:
+        GM_CLASS_ALLOCATOR(DefaultTrafficControl);
+
+        /// maxSystemPacketSize in bytes and datagramOverhead in bytes (used for effective statistics)
+        DefaultTrafficControl(unsigned int maxSystemPacketSize, float rttConnectionThreshold, float packetLossThreshold, unsigned int maxRecvPackets);
+        ~DefaultTrafficControl() override;
+
+        /// Called when Carrier has established a new connection.
+        void OnConnect(TrafficControlConnectionId id, const AZStd::intrusive_ptr<DriverAddress>& address) override;
+        /// Called when Carrier has lost a connection.
+        void OnDisconnect(TrafficControlConnectionId id) override;
+        /// Called when Carrier completed successful handshake. Usually NAT punch happens during the handshake, which can result is high packet loss.
+        void OnHandshakeComplete(TrafficControlConnectionId id) override;
+        /// Called when Carrier has send a package.
+        void OnSend(TrafficControlConnectionId id, DataGramControlData& info) override;
+        /// Called when Carrier has send an ACK/NACK data with the packet.
+        void OnSendAck(TrafficControlConnectionId id) override;
+        /// Called when Carrier has resend a package.
+        void OnReSend(TrafficControlConnectionId id, DataGramControlData& info, unsigned int resendDataSize) override;
+        /// Called when Carrier confirmed a package delivery.
+        void OnAck(TrafficControlConnectionId id, DataGramControlData& info, bool& windowChange) override;
+        /// Called when we receive a NAck for a package delivery.
+        void OnNAck(TrafficControlConnectionId id, DataGramControlData& info) override;
+        /// Called when Carrier receives a package.
+        void OnReceived(TrafficControlConnectionId id, DataGramControlData& info) override;
+        /// Return true if we can send a package. Otherwise false.
+        bool IsSend(TrafficControlConnectionId id) override;
+        /// Return true if you should send ACK/NACK data at this time.
+        bool IsSendAck(TrafficControlConnectionId id) override;
+        /// Return number of bytes we are allowed to send at the moment. The size can/will vary over time.
+        unsigned int GetAvailableWindowSize(TrafficControlConnectionId id) const override;
+        /**
+        * Called for every package waiting for Ack. If this function returns true the packet will be considered lost.
+        * You should resend it and call OnReSend function ASAP.
+        */
+        bool IsResend(TrafficControlConnectionId id, const DataGramControlData& info, unsigned int resendDataSize) override;
+
+        TimeStamp GetResendTime(TrafficControlConnectionId id, const DataGramControlData& info) override;
+
+        bool IsDisconnect(TrafficControlConnectionId id, float conditionThreshold) override;
+
+        bool IsCanReceiveData(TrafficControlConnectionId id) const override;
+
+        /**
+         * Returns true if you need to send a ACK only (empty datagram) due to time and/or number of received datagrams.
+         * If you already have data to send ACK will be included in the datagram anyway. This function should be checked only
+         * if you have no data to send.
+         */
+        bool IsSendACKOnly(TrafficControlConnectionId id) const override;
+
+        /// Update/Tick returns true if we have updated the statistics (which we can read by \ref QueryStatistics)
+        bool Update() override;
+        /**
+         * Stores connection statistics, it's ok to pass NULL for any of the statistics.
+         * \param id Connection ID
+         * \param lastSecond last second statistics for all data
+         * \param lifetime lifetime statistics for all data
+         * \param effectiveLastSecond last second statistics for effective data (actual data - carrier overhead excluded)
+         * \param effectiveLifetime lifetime statistics for effective data (actual data - carrier overhead excluded)
+         */
+        void QueryStatistics(TrafficControlConnectionId id, Statistics* lastSecond = nullptr, Statistics* lifetime = nullptr, Statistics* effectiveLastSecond = nullptr, Statistics* effectiveLifetime = nullptr) const override;
+
+        /**
+        * Stores current congestion state into the provided block.
+        */
+        void        QueryCongestionState(TrafficControlConnectionId id, CongestionState* congestionState) const override;
+
+    private:
+        unsigned int m_maxPacketSize;       ///< Current max packet size
+
+        float m_rttConnectionThreshold;     ///< in milliseconds
+        float m_packetLossThreshold;        ///< percent 1.0 is 100%
+
+        struct StatisticData
+            : public TrafficControl::Statistics
+        {
+            float m_avgPacketSend;  ///< Used to average packet send over the last 10-15 sec. Used for packetLoss
+            float m_avgPacketLost;  ///< Used to average packet lost over the last 10-15 sec. Used for packetLoss
+
+            StatisticData() { Reset(); }
+            void Reset()
+            {
+                m_dataSend = 0;
+                m_dataReceived = 0;
+
+                m_dataResend = 0;
+                m_dataAcked = 0;
+
+                m_packetSend = 0;
+                m_packetReceived = 0;
+
+                m_packetLost = 0;
+                m_packetAcked = 0;
+
+                m_rtt = 0;
+                m_packetLoss = 0;
+                m_connectionFactor = 0;
+                //m_flow = 0;
+
+                m_avgPacketSend = 0;
+                m_avgPacketLost = 0;
+            }
+        };
+
+        struct ConnectionData
+        {
+            //////////////////////////////////////////////////////////////////////////
+            // TCP Cubic Functions
+            // NOTE: Before modifying this section read "CUBIC: a new TCP-friendly high-speed TCP variant" 
+            //       http://dl.acm.org/citation.cfm?id=1400105
+            void TCPCubicCalcK(unsigned int packetSize);
+            void TCPCubicPacketLost(TimeStamp& now, unsigned int packetSize);
+            void TCPCubicExitSlowStart(TimeStamp& now, unsigned int packetSize);
+            unsigned int TCPCubicWindow(TimeStamp& now, unsigned int packetSize) const;
+            unsigned int TCPRenoWindow(double time) const;
+            //////////////////////////////////////////////////////////////////////////
+
+            StatisticData   m_sdLifetime;           ///< Statistic data for the lifetime of the connection.
+            StatisticData   m_sdLastSecond;     ///< Statistic data for the last second.
+            StatisticData   m_sdCurrentSecond;  ///< Current data for the elapsing second.
+
+            StatisticData   m_sdEffectiveLifetime;      ///< Lifetime statistics for effective data.
+            StatisticData   m_sdEffectiveLastSecond;        ///< Last second statistics for effective data.
+            StatisticData   m_sdEffectiveCurrentSecond; ///< Elapsing second statistics for effective data.
+
+            string          m_address;          ///< Full address for this connection. (we need for debug only)
+            unsigned int    m_recvPacketAllowance; ///< Current allowance for number of incoming packets
+            bool            m_canReceiveData; ///< Able to receive data on this connection
+
+            TimeStamp       m_lastPacketSend;
+            TimeStamp       m_lastAckSend;
+            TimeStamp       m_handshakeDone;        ///< Stamp when the handshake operation has completed.
+            bool            m_isReceivedDataAfterLastSend;  ///< Flag indicating that we have received data (not just ACK) after the last send. This can be used to send instant ACK if needed.
+
+            //////////////////////////////////////////////////////////////////////////
+            // Slow start basic TCP based congestion control
+            TimeStamp       m_lastWindowSizeIncrease;
+            TimeStamp       m_lastWindowSizeDecrease;
+            unsigned int    m_inTransfer;               ///< Number of bytes in transfer
+            unsigned int    m_slowStartThreshold;       ///< Slow start threshold (SSTresh)
+            unsigned int    m_congestionWindow;         ///< Congestion window (cwnd)
+            unsigned int    m_maxCongestionWindow;      ///< Max congestion window for this connection (TODO make the other side to advertise this window)
+            //////////////////////////////////////////////////////////////////////////
+
+            //////////////////////////////////////////////////////////////////////////
+            // TCP Cubic Data
+            // NOTE: Before modifying this section read "CUBIC: a new TCP-friendly high-speed TCP variant" 
+            //       http://dl.acm.org/citation.cfm?id=1400105
+            static const bool   k_enableCubic = false;
+            static const double k_CubicAlpha;                                    ///< alpha for TCP friendly window estimation
+            static const double k_CubicBeta;                                     ///< beta (backoff rate) for congestion calculations
+            static const double k_CubicScaleC;                                   ///< Scaling factor. COTS default (0.4)
+            static const unsigned int  k_MinCongestionWindowPackets = 10;        ///< Minimum congestion window
+            unsigned int        m_preBackoffCongestionWindow = k_MinCongestionWindowPackets;                   ///< Window before backoff
+            double              m_CubicKcube = 0.0;
+            double              m_CubicK = 0.0;
+            //////////////////////////////////////////////////////////////////////////
+        };
+
+        typedef list<ConnectionData> ConnectionListType;
+        ConnectionListType m_connections;
+
+        float           m_lastStatDataReset;        ///< The time in seconds since the last time we reset the statistic data.
+        TimeStamp       m_currentTime;              ///< Current time. (replace this with a global clock when possible.)
+        unsigned int    m_lostPacketTimeoutMS;      ///< Time in milliseconds for a packet to be considered lost.
+        unsigned int    m_defaultMaxCongestionWindowSize;
+
+        unsigned int m_maxRecvPackets;
+    };
+}
+
+#endif // GM_DEFAULT_TRAFFIC_CONTROL_H

Code/Framework/GridMate/GridMate/Carrier/Driver.h

@@ -0,0 +1,229 @@
+/*
+* All or portions of this file Copyright (c) Amazon.com, Inc. or its affiliates or
+* its licensors.
+*
+* For complete copyright and license terms please see the LICENSE at the root of this
+* distribution (the "License"). All use of this software is governed by the License,
+* or, if provided, by the license below or the license accompanying this file. Do not
+* remove or modify any license notices. This file is distributed on an "AS IS" BASIS,
+* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+*
+*/
+#ifndef GM_DRIVER_H
+#define GM_DRIVER_H
+
+#include <GridMate/Types.h>
+
+#include <GridMate/String/string.h>
+
+#include <AzCore/std/delegate/delegate.h>
+#include <AzCore/std/smart_ptr/intrusive_ptr.h>
+
+namespace GridMate
+{
+    struct ThreadConnection;
+    class CarrierThread;
+    class DriverAddress;
+
+    /**
+     * Driver interface is the interface for the lowest level of the transport layer.
+     * \note All the code is executed in a thread context! Any interaction with
+     * the outside code should be made thread safe.
+     */
+    class Driver
+    {
+        friend class DriverAddress;
+    public:
+        typedef unsigned int ResultCode;
+
+        //list of common error codes
+        enum ErrorCodes
+        {
+            EC_OK = 0,                  ///<
+
+            // Socket errors
+            EC_SOCKET_CREATE,
+            EC_SOCKET_LISTEN,
+            EC_SOCKET_CLOSE,
+            EC_SOCKET_MAKE_NONBLOCK,
+            EC_SOCKET_BIND,
+            EC_SOCKET_SOCK_OPT,
+            EC_SOCKET_CONNECT,
+            EC_SOCKET_ACCEPT,
+
+            EC_SECURE_CONFIG,   // Invalid configuration.
+            EC_SECURE_CREATE,   // Failed to create and configure the SSL context.
+            EC_SECURE_CERT,     // Failed to load the provided certificate.
+            EC_SECURE_PKEY,     // Failed to load the provided private key.
+            EC_SECURE_CA_CERT,  // Failed to load the provided CA cert or cert chain.
+
+            EC_SEND,
+            EC_SEND_ADDRESS_NOT_BOUND,  ///< We failed to send because the remote address was NOT bound.
+            EC_RECEIVE,
+
+            EC_PLATFORM = 1000, ///< use codes above 1000 for platform specific error codes
+            EC_BUFFER_TOOLARGE = 1001
+        };
+
+        /**
+        * Family types for BSD socket.
+        */
+        enum BSDSocketFamilyType
+        {
+            BSD_AF_INET = 0,
+            BSD_AF_INET6,
+            BSD_AF_UNSPEC,
+        };
+
+        Driver() :
+            m_canSend(true)
+        {}
+        virtual ~Driver() {}
+
+        virtual void Update() {}
+        virtual void ProcessIncoming() {}
+        virtual void ProcessOutgoing() {}
+
+        /// \todo Add QoS support
+
+        /**
+        * Platform specific functionality.
+        */
+        /// Return maximum number of active connections at the same time.
+        virtual unsigned int GetMaxNumConnections() const = 0;
+        /// Return maximum data size we can send/receive at once in bytes, supported by the platform.
+        virtual unsigned int GetMaxSendSize() const = 0;
+        /// Return packet overhead size in bytes.
+        virtual unsigned int GetPacketOverheadSize() const { return 8 /* standard UDP*/ + 20 /* min for IPv4 */; }
+        /*
+        UDP/VDP has a 44 byte header when using port 1000 - the header is up to 4 bytes larger when using other ports
+        using voice chat over VDP adds an additional 2 bytes to the packet header
+        worst case VDP header with voice is 52 bytes - additional overhead incurred for using other ports cannot exceed 4 bytes so this is partially unaccounted for
+        */
+
+        /// Transforms an error code to string.
+        //virtual void ResultCodeToString(string& str, ResultCode resultCode) const = 0;
+
+        /**
+         * User should implement create and bind a UDP socket. This socket will be used for all communications.
+         * \param ft family type (this value depends on the platform), 0 will use the default family type (for BSD socket this is ipv4)
+         * \param address when 0 it we will assume "any address".
+         * \param port When left 0, we use implicit bind (assigned by the system). Otherwise provide a valid port number.
+         * \param receiveBufferSize socket receive buffer size in bytes, use 0 for default values.
+         * \param sendBufferSize socket send buffer size, use 0 for default values.
+         */
+        virtual ResultCode  Initialize(int familyType = 0, const char* address = nullptr, unsigned int port = 0, bool isBroadcast = false, unsigned int receiveBufferSize = 0, unsigned int sendBufferSize = 0) = 0;
+
+        /// Returns communication port (must be called after Initialize, otherwise it will return 0)
+        virtual unsigned int    GetPort() const = 0;
+
+        /// Send data to a user defined address
+        virtual ResultCode      Send(const AZStd::intrusive_ptr<DriverAddress>& to, const char* data, unsigned int dataSize) = 0;
+        /**
+         * Receives a datagram and stores the source address. maxDataSize must be >= than GetMaxSendSize(). Returns the num of of received bytes.
+         * \note If a datagram from a new connection is received, NewConnectionCB will be called. If it rejects the connection the returned from pointer
+         * will be NULL while the actual data will be returned.
+         */
+        virtual unsigned int    Receive(char* data, unsigned int maxDataSize, AZStd::intrusive_ptr<DriverAddress>& from, ResultCode* resultCode = 0) = 0;
+        /**
+         *  Wait for data to be to the ready for receive. Time out is the maximum time to wait
+         * before this function returns. If left to default value it will be in blocking mode (wait until data is ready to be received).
+         * \returns true if there is data to be received (always true if timeOut == 0), otherwise false.
+         */
+        virtual bool            WaitForData(AZStd::chrono::microseconds timeOut = AZStd::chrono::microseconds(0)) = 0;
+
+        /**
+         * When you enter wait for data mode, for many reasons you might want to stop wait for data.
+         * If you implement this function you need to make sure it's a thread safe function.
+         */
+        virtual void            StopWaitForData() = 0;
+
+        /// Return true if WaitForData was interrupted before the timeOut expired, otherwise false.
+        virtual bool            WasStopeedWaitingForData() = 0;
+
+        /// @{ Address conversion functionality. They MUST implemented thread safe. Generally this is not a problem since they just part local data.
+        ///  Create address from ip and port. If ip == NULL we will assign a broadcast address.
+        virtual string          IPPortToAddress(const char* ip, unsigned int port) const = 0;
+        virtual bool            AddressToIPPort(const string& address, string& ip, unsigned int& port) const = 0;
+        /// @}
+
+        /**
+         * Creates internal driver address to be used for send/receive calls.
+         * \note if the ip and the port are the same, the same pointer will be returned. You can use the returned pointer
+         * to compare for unique addresses.
+         * \note Driver address allocates internal resources, use it only when you intend to communicate. Otherwise operate with
+         * the string address.
+         */
+        virtual AZStd::intrusive_ptr<DriverAddress> CreateDriverAddress(const string& address) = 0;
+
+        /**
+         * Returns true if the driver can accept new data (ex, has buffer space).
+         */
+        virtual bool CanSend() const { return m_canSend; }
+
+    protected:
+        virtual void             DestroyDriverAddress(DriverAddress* address) = 0;
+
+        bool                    m_canSend;      ///< Can the driver accept more data
+    };
+
+    /**
+     * Driver address interface, for low level driver communication.
+     */
+    class DriverAddress
+    {
+        friend struct ThreadConnection;
+        friend class CarrierThread;
+    public:
+        DriverAddress(Driver* driver)
+            : m_threadConnection(NULL)
+            , m_driver(driver)
+            , m_refCount(0)
+        {
+            AZ_Assert(m_driver != NULL, "You must provide a valid driver");
+        }
+
+        DriverAddress(const DriverAddress& rhs)
+            : m_threadConnection(rhs.m_threadConnection)
+            , m_driver(rhs.m_driver)
+            , m_refCount(rhs.m_refCount)
+        {
+        }
+
+        virtual ~DriverAddress() {}
+
+        virtual string ToString() const = 0;
+
+        virtual string ToAddress() const = 0;
+
+        virtual string GetIP() const = 0;
+
+        virtual unsigned int  GetPort() const = 0;
+
+        Driver* GetDriver() const   { return m_driver; }
+
+        bool IsBoundToCarrierConnection() const { return m_threadConnection != nullptr; }
+
+        virtual const void* GetTargetAddress(unsigned int& addressSize) const { addressSize = 0; return nullptr; }
+
+        //////////////////////////////////////////////////////////////////////////
+        // for AZStd::intrusive_ptr
+        void    add_ref() { ++m_refCount;   }
+        void    release()
+        {
+            AZ_Assert(m_refCount > 0, "Reference count logic error, trying to remove reference when refcount is 0");
+            if (--m_refCount == 0)
+            {
+                m_driver->DestroyDriverAddress(this);
+            }
+        }
+        //////////////////////////////////////////////////////////////////////////
+    protected:
+        ThreadConnection*       m_threadConnection;     ///< Used by the CarrierThread/ThreadConnection.
+        Driver*                 m_driver;
+        /// reference counting
+        mutable unsigned int    m_refCount;
+    };
+}
+
+#endif // GM_DRIVER_H

Code/Framework/GridMate/GridMate/Carrier/DriverEvents.h

@@ -0,0 +1,39 @@
+/*
+* All or portions of this file Copyright (c) Amazon.com, Inc. or its affiliates or
+* its licensors.
+*
+* For complete copyright and license terms please see the LICENSE at the root of this
+* distribution (the "License"). All use of this software is governed by the License,
+* or, if provided, by the license below or the license accompanying this file. Do not
+* remove or modify any license notices. This file is distributed on an "AS IS" BASIS,
+* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+*
+*/
+#pragma once
+
+#include <GridMate/GridMate.h>
+#include <AzCore/EBus/EBus.h>
+
+namespace GridMate
+{
+    class Driver;
+    class DriverAddress;
+
+    class DriverEvents
+    {
+    public:
+        // Called when a datagram is actually sent.
+        virtual void OnDatagramSent(size_t payloadBytesSent, const AZStd::intrusive_ptr<DriverAddress>& to) = 0;
+        // Called when a datagram is received, before any filtering or processing.
+        virtual void OnDatagramReceived(size_t payloadBytesReceived, const AZStd::intrusive_ptr<DriverAddress>& from) = 0;
+    };
+
+    class DriverEBusTraits : public AZ::EBusTraits
+    {
+    public:
+        static const AZ::EBusAddressPolicy AddressPolicy = AZ::EBusAddressPolicy::ById;
+        typedef AZStd::recursive_mutex MutexType;
+        typedef Driver* BusIdType;
+    };
+    typedef AZ::EBus<DriverEvents, DriverEBusTraits> DriverEventBus;
+}

Code/Framework/GridMate/GridMate/Carrier/Handshake.h

@@ -0,0 +1,69 @@
+/*
+* All or portions of this file Copyright (c) Amazon.com, Inc. or its affiliates or
+* its licensors.
+*
+* For complete copyright and license terms please see the LICENSE at the root of this
+* distribution (the "License"). All use of this software is governed by the License,
+* or, if provided, by the license below or the license accompanying this file. Do not
+* remove or modify any license notices. This file is distributed on an "AS IS" BASIS,
+* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+*
+*/
+#ifndef GM_HANDSHAKE_INTERFACE_H
+#define GM_HANDSHAKE_INTERFACE_H
+
+#include <GridMate/Types.h>
+#include <GridMate/Serialize/Buffer.h>
+#include <GridMate/String/string.h>
+
+namespace GridMate
+{
+    enum class HandshakeErrorCode : int
+    {
+        OK = 0,
+        REJECTED,
+        PENDING,
+        VERSION_MISMATCH,
+    };
+
+    /**
+     * Handshake interface
+     */
+    class Handshake
+    {
+    public:
+        virtual ~Handshake() {}
+        /// Called from the system to write initial handshake data.
+        virtual void                OnInitiate(ConnectionID id, WriteBuffer& wb) = 0;
+        /**
+         * Called when a system receives a handshake initiation from another system.
+         * You can write a reply in the WriteBuffer.
+         * return true if you accept this connection and false if you reject it.
+         */
+        virtual HandshakeErrorCode  OnReceiveRequest(ConnectionID id, ReadBuffer& rb, WriteBuffer& wb) = 0;
+        /**
+         * If we already have a valid connection and we receive another connection request, the system will
+         * call this function to verify the state of the connection.
+         */
+        virtual bool                OnConfirmRequest(ConnectionID id, ReadBuffer& rb) = 0;
+        /**
+         * Called when we receive Ack from the other system on our initial data \ref OnInitiate.
+         * return true to accept the ack or false to reject the handshake.
+         */
+        virtual bool                OnReceiveAck(ConnectionID id, ReadBuffer& rb) = 0;
+        /**
+         * Called when we receive Ack from the other system while we were connected. This callback is called
+         * so we can just confirm that our connection is valid!
+         */
+        virtual bool                OnConfirmAck(ConnectionID id, ReadBuffer& rb) = 0;
+        /// Return true if you want to reject early reject a connection.
+        virtual bool                OnNewConnection(const string& address) = 0;
+        /// Called when we close a connection.
+        virtual void                OnDisconnect(ConnectionID id) = 0;
+        /// Return timeout in milliseconds of the handshake procedure.
+        virtual unsigned int        GetHandshakeTimeOutMS() const = 0;
+    };
+}
+
+#endif // GM_HANDSHAKE_INTERFACE_H
+

Code/Framework/GridMate/GridMate/Carrier/SecureSocketDriver.h

@@ -0,0 +1,276 @@
+/*
+* All or portions of this file Copyright (c) Amazon.com, Inc. or its affiliates or
+* its licensors.
+*
+* For complete copyright and license terms please see the LICENSE at the root of this
+* distribution (the "License"). All use of this software is governed by the License,
+* or, if provided, by the license below or the license accompanying this file. Do not
+* remove or modify any license notices. This file is distributed on an "AS IS" BASIS,
+* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+*
+*/
+#ifndef GM_SECURE_SOCKET_DRIVER_H
+#define GM_SECURE_SOCKET_DRIVER_H
+
+#include <AzCore/std/chrono/types.h>
+#include <AzCore/std/containers/queue.h>
+#include <AzCore/std/containers/unordered_map.h>
+
+#include <AzCore/State/HSM.h>
+
+#include <GridMate/Carrier/SocketDriver.h>
+
+#define AZ_DebugSecureSocket(...)
+#define AZ_DebugSecureSocketConnection(window, fmt, ...)
+
+//#define AZ_DebugUseSocketDebugLog
+//#define AZ_DebugSecureSocket AZ_TracePrintf
+//#define AZ_DebugSecureSocketConnection(window, fmt, ...) \
+//{\
+//    GridMate::string line = GridMate::string::format(fmt, __VA_ARGS__);\
+//    this->m_dbgLog += line;\
+//}
+
+#if AZ_TRAIT_GRIDMATE_SECURE_SOCKET_DRIVER_HOOK_ENABLED
+struct ssl_st;
+struct ssl_ctx_st;
+struct dh_st;
+struct bio_st;
+struct x509_store_ctx_st;
+struct evp_pkey_st;
+struct x509_st;
+struct x509_store_ct;
+
+typedef struct ssl_st SSL;
+typedef struct ssl_ctx_st SSL_CTX;
+typedef struct dh_st DH;
+typedef struct bio_st BIO;
+typedef struct x509_store_ctx_st X509_STORE_CTX;
+typedef struct x509_store_st X509_STORE;
+typedef struct evp_pkey_st EVP_PKEY;
+typedef struct x509_st X509;
+
+namespace GridMate
+{
+    static const int COOKIE_SECRET_LENGTH = 16; //128 bit key
+    static const int MAX_COOKIE_LENGTH = 255; // largest length that will fit in one byte
+
+    namespace ConnectionSecurity
+    {
+        bool IsHandshake(const char* data, AZ::u32 dataSize);
+        bool IsClientHello(const char* data, AZ::u32 dataSize);
+        bool IsChangeCipherSpec(const char* data, AZ::u32 dataSize);
+        bool IsHelloVerifyRequest(const char* data, AZ::u32 dataSize);
+        bool IsHelloRequestHandshake(const char* data, AZ::u32 dataSize);
+        const char* TypeToString(const char* data, AZ::u32 dataSize);
+    }
+
+    struct SecureSocketDesc
+    {
+        SecureSocketDesc()
+            : m_connectionTimeoutMS(5000)
+            , m_authenticateClient(false)
+            , m_maxDTLSConnectionsPerIP(~0u)
+            , m_privateKeyPEM(nullptr)
+            , m_certificatePEM(nullptr)
+            , m_certificateAuthorityPEM(nullptr) {};
+
+        AZ::u64     m_connectionTimeoutMS;
+        bool        m_authenticateClient;          // Ensure that a client must be authenticated (the server is
+                                                   // always authenticated). Only required to be set on the server!
+        unsigned int m_maxDTLSConnectionsPerIP;    // Max number of DTLS connections that can be accepted per ip
+        const char* m_privateKeyPEM;               // A base-64 encoded PEM format private key.
+        const char* m_certificatePEM;              // A base-64 encoded PEM format certificate.
+        const char* m_certificateAuthorityPEM;     // A base-64 encoded PEM format CA root certificate.
+    };
+
+    /**
+    * A driver implementation that encrypts and decrypts data sent between the application
+    * and the underlying socket. The driver depends on a socket being successfully created
+    * and bound to a port so it derives from the existing SocketDriver implementation (this
+    * approach also makes SecureSocketDriver fairly platform agnostic).
+    *
+    * In order to establish a secure channel between two peers a formal connection needs to be
+    * created and a TLS handshake performed. During this handshake a cipher is agreed upon, a
+    * shared symmetric key generated and peers authenticated.
+    *
+    * Connections are created when sending or receiving a packet from a peer for the first time
+    * and removed when explicitly disconnected or times out.
+    *
+    * The driver API is stateless so a user needn't know about the internal connections to
+    * remote peers. The user simply sends and receive datagrams as normal to endpoints on the network.
+    * All user datagrams sent during the connection handshake are queued up and sent encrypted when
+    * the connection has been successfully established.
+    */
+    class SecureSocketDriver
+        : public SocketDriver
+    {
+    public:
+        GM_CLASS_ALLOCATOR(SecureSocketDriver);
+
+        typedef AZStd::intrusive_ptr<DriverAddress> AddrPtr;
+        typedef AZStd::vector<char>                 Datagram;
+        typedef AZStd::pair<Datagram, AddrPtr>      DatagramAddr;
+
+        SecureSocketDriver(const SecureSocketDesc& desc, bool isFullPackets = false, bool crossPlatform = false, bool isHighPerformance = true);
+        virtual ~SecureSocketDriver();
+        static void apps_ssl_info_callback(const SSL *s, int where, int ret);
+
+        AZ::u32 GetMaxSendSize() const override;
+
+        Driver::ResultCode Initialize(AZ::s32 ft, const char* address, AZ::u32 port, bool isBroadcast, AZ::u32 receiveBufferSize, AZ::u32 sendBufferSize) override;
+        void               Update() override;
+        void               ProcessIncoming() override;
+        void               ProcessOutgoing() override;
+        Driver::ResultCode Receive(char* data, AZ::u32 maxDataSize, AddrPtr& from, ResultCode* resultCode) override;
+        AZ::u32            Send(const AddrPtr& to, const char* data, AZ::u32 dataSize) override;
+
+    protected:
+        enum ConnectionState
+        {
+            CS_TOP,
+                CS_ACTIVE,                              // Processing datagrams
+                    CS_SEND_HELLO_REQUEST,              // S: Waiting for client to start TLS/DTLS handshake
+                    CS_ACCEPT,                          // S: Performing TLS/DTLS handshake for an incoming connection.
+                    CS_COOKIE_EXCHANGE,                 // C: Performing cookie verification
+                    CS_CONNECT,                         // C: Performing TLS/DTLS handshake for an outgoing connection.
+                    CS_ESTABLISHED,                     // Both: SSL handshake succeeded.
+                CS_DISCONNECTED,                        // Both: Disconnected.
+            CS_MAX
+        };
+
+        /**
+        * Manage a single DTLS connection to a remote peer. It is ideally backed by two buffers
+        * that will contain ciphertext and two queues that contain plaintext.
+        *
+        * There are two flows of data:
+        *  - Plaintext is pulled from the out queue, encrypted, and ciphertext written to the out buffer.
+        *  - Ciphertext is read from the in buffer, decrypted, and plaintext added to the in queue.
+        *
+        * In practice there is two buffers of ciphertext and only one queue that contains plaintext.
+        *
+        * As an optimization, the connection does not hold it's own plaintext in queue. Since the
+        * user is going to be polling the single driver for plaintext datagrams it is better to
+        * add all decrypted datagrams to a single, shared queue for the driver to pull from.
+        *
+        * Connections have their own timeout which is set during construction. The connection will
+        * be disconnected on a timeout and no further communication will be possible.
+        */
+        class Connection
+        {
+        public:
+            GM_CLASS_ALLOCATOR(Connection);
+            Connection(const AddrPtr& addr, AZ::u32 bufferSize, AZStd::queue<DatagramAddr>* inQueue, AZ::u64 timeoutMS, int port);
+            virtual ~Connection();
+
+            bool    Initialize(SSL_CTX* sslContext, ConnectionState startState, AZ::u32 mtu);
+            void    Shutdown();
+
+            void    Update();
+            void    AddDgram(const char* data, AZ::u32 dataSize);
+            void    ProcessIncomingDTLSDgram(const char* data, AZ::u32 dataSize);
+            AZ::u32 GetDTLSDgram(char* data, AZ::u32 dataSize);
+            void    FlushOutgoingDTLSDgrams();
+            bool    IsDisconnected() const;
+
+            void ForceDTLSTimeout();
+            bool CreateSSL(SSL_CTX* sslContext);
+            bool DestroySSL();
+            const SSL* GetSSL() { return m_ssl; }
+
+        private:
+            bool OnStateActive(AZ::HSM& sm, const AZ::HSM::Event& event);
+            bool OnStateAccept(AZ::HSM& sm, const AZ::HSM::Event& event);
+            bool OnStateSendHelloRequest(AZ::HSM& sm, const AZ::HSM::Event& event);
+            bool OnStateConnect(AZ::HSM& sm, const AZ::HSM::Event& event);
+            bool OnStateCookieExchange(AZ::HSM& sm, const AZ::HSM::Event& event);
+            bool OnStateEstablished(AZ::HSM& sm, const AZ::HSM::Event& event);
+            bool OnStateDisconnected(AZ::HSM& sm, const AZ::HSM::Event& event);
+            bool HandleSSLError(AZ::s32 result);
+
+            // Read a DTLS record (datagram) from a BIO buffer. Return true if records were read and stored
+            // in outDgramList, or false if nothing was found.
+            bool ReadDgramFromBuffer(BIO* buffer, AZStd::vector<SecureSocketDriver::Datagram>& outDgramQueue);
+
+            // Queue outbound Datagrams from SSL BIO into m_outDTLSQueue
+            int QueueDatagrams();
+
+            enum ConnectionEvents
+            {
+                CE_UPDATE = 1,
+                CE_STATEFUL_HANDSHAKE,
+                CE_COOKIE_EXCHANGE_COMPLETED,
+                CE_NEW_INCOMING_DGRAM,
+                CE_NEW_OUTGOING_DGRAM,
+            };
+
+            bool m_isInitialized;
+            TimeStamp m_creationTime;    // The time the connection reached the established state.
+            AZ::u64 m_timeoutMS;
+            AZStd::queue<Datagram>          m_outboundPlainQueue;   // Outbound plaintext datagrams from application
+            BIO*                            m_outDTLSBuffer;        // Outbound DTLS serialization buffer ready for -> m_outDTLSQueue
+            AZStd::queue<Datagram>          m_outDTLSQueue;         // Outbound DTLS datagrams ready for Socket Send
+            BIO*                            m_inDTLSBuffer;         // Inbound DTLS decryption buffer ready for -> m_inboundPlaintextQueue
+            AZStd::queue<DatagramAddr>*     m_inboundPlaintextQueue;// Inbound plaintext datagrams ready for application read
+            AZ::HSM m_sm;
+            SSL* m_ssl;
+            SSL_CTX* m_sslContext;
+            AddrPtr m_addr;
+            AZ::u32 m_maxTempBufferSize;
+            AZ::s32 m_sslError;
+            AZ::u32 m_mtu;
+
+            AZStd::chrono::system_clock::time_point m_nextHelloRequestResend;
+            AZStd::chrono::milliseconds k_initialHelloRequestResendInterval = AZStd::chrono::milliseconds(100);
+            AZStd::chrono::milliseconds m_helloRequestResendInterval;
+            AZStd::chrono::system_clock::time_point m_nextHandshakeRetry;
+
+        public:
+            int m_dbgDgramsSent;
+            int m_dbgDgramsReceived;
+            int m_dbgPort;
+#ifdef AZ_DebugUseSocketDebugLog
+            GridMate::string m_dbgLog;
+#endif
+        };
+
+        bool RotateCookieSecret(bool bForce = false);
+        static int VerifyCertificate(int ok, X509_STORE_CTX* ctx);
+        int GenerateCookie(AddrPtr endpoint, unsigned char* cookie, unsigned int* cookieLen);
+        int VerifyCookie(AddrPtr endpoint, unsigned char* cookie, unsigned int cookieLen);
+
+        void FlushSocketToConnectionBuffer();
+        void UpdateConnections();
+        void FlushConnectionBuffersToSocket();
+
+        EVP_PKEY* m_privateKey;
+        X509* m_certificate;
+        SSL_CTX* m_sslContext;
+        char* m_tempSocketWriteBuffer;
+        char* m_tempSocketReadBuffer;
+
+        struct GridMateSecret
+        {
+            TimeStamp m_lastSecretGenerationTime;
+            unsigned char m_currentSecret[COOKIE_SECRET_LENGTH];
+            unsigned char m_previousSecret[COOKIE_SECRET_LENGTH];
+            bool m_isCurrentSecretValid;
+            bool m_isPreviousSecretValid;
+
+            GridMateSecret()
+                : m_isCurrentSecretValid(false)
+                , m_isPreviousSecretValid(false)
+            { }
+        } m_cookieSecret;
+
+        AZ::u32 m_maxTempBufferSize;
+        AZStd::queue<DatagramAddr> m_globalInQueue;
+        AZStd::unordered_map<SocketDriverAddress, Connection*, SocketDriverAddress::Hasher> m_connections;
+        AZStd::unordered_map<string, int> m_ipToNumConnections;
+        SecureSocketDesc m_desc;
+        AZStd::chrono::system_clock::time_point m_lastTimerCheck;       ///Time last timers were checked
+    };
+}
+
+#endif
+#endif

Code/Framework/GridMate/GridMate/Carrier/Simulator.h

@@ -0,0 +1,51 @@
+/*
+* All or portions of this file Copyright (c) Amazon.com, Inc. or its affiliates or
+* its licensors.
+*
+* For complete copyright and license terms please see the LICENSE at the root of this
+* distribution (the "License"). All use of this software is governed by the License,
+* or, if provided, by the license below or the license accompanying this file. Do not
+* remove or modify any license notices. This file is distributed on an "AS IS" BASIS,
+* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+*
+*/
+#ifndef GM_SIMULATOR_INTERFACE_H
+#define GM_SIMULATOR_INTERFACE_H
+
+#include <GridMate/Types.h>
+#include <AzCore/std/smart_ptr/intrusive_ptr.h>
+
+namespace GridMate
+{
+    class Driver;
+    class DriverAddress;
+
+    /**
+     * Simulator interface
+     */
+    class Simulator
+    {
+    public:
+        virtual ~Simulator() {}
+
+        /// Called from Carrier, so simulator can use the low level driver directly.
+        virtual void BindDriver(Driver* driver) = 0;
+        /// Called from Carrier when driver can no longer be used(ie. will be destroyed)
+        virtual void UnbindDriver() = 0;
+        /// Called when Carrier has established a new connection.
+        virtual void OnConnect(const AZStd::intrusive_ptr<DriverAddress>& address) = 0;
+        /// Called when Carrier has lost a connection.
+        virtual void OnDisconnect(const AZStd::intrusive_ptr<DriverAddress>& address) = 0;
+        /// Called when Carrier has a packet to send
+        virtual bool OnSend(const AZStd::intrusive_ptr<DriverAddress>& to, const void* data, unsigned int dataSize) = 0;
+        /// Called when Carrier receives a packet
+        virtual bool OnReceive(const AZStd::intrusive_ptr<DriverAddress>& from, const void* data, unsigned int dataSize) = 0;
+        /// Called from Carrier when no more data has arrived and can supply you with data (with latency, out of order, etc).
+        virtual unsigned int ReceiveDataFrom(AZStd::intrusive_ptr<DriverAddress>& from, char* data, unsigned int maxDataSize) = 0;
+        ///
+        virtual void Update() = 0;
+    };
+}
+
+#endif // GM_SIMULATOR_INTERFACE_H
+

Code/Framework/GridMate/GridMate/Carrier/SocketDriver.h

@@ -0,0 +1,417 @@
+/*
+* All or portions of this file Copyright (c) Amazon.com, Inc. or its affiliates or
+* its licensors.
+*
+* For complete copyright and license terms please see the LICENSE at the root of this
+* distribution (the "License"). All use of this software is governed by the License,
+* or, if provided, by the license below or the license accompanying this file. Do not
+* remove or modify any license notices. This file is distributed on an "AS IS" BASIS,
+* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+*
+*/
+#pragma once
+
+#include <GridMate/Memory.h>
+#include <AzCore/std/functional_basic.h>
+#include <AzCore/std/containers/unordered_set.h>
+#include <AzCore/PlatformIncl.h>
+#include <AzCore/std/smart_ptr/unique_ptr.h>
+#include <AzCore/std/parallel/thread.h>
+#include <AzCore/std/parallel/atomic.h>
+#include <AzCore/std/parallel/condition_variable.h>
+
+#include <GridMate_Traits_Platform.h>
+#include <GridMate/Carrier/SocketDriver_Platform.h>
+#include <GridMate/Carrier/Driver.h>
+
+#if AZ_TRAIT_GRIDMATE_SOCKET_IPV6_SUPPORT_EXTENSION
+
+namespace GridMate
+{
+    /// Emulate in6_addr, it will never be used
+    struct in6_addr
+    {
+        unsigned char   s6_addr[16];   // IPv6 address
+    };
+
+    // Emulate sockaddr_in6 structure, it will never be used
+    struct sockaddr_in6
+    {
+        unsigned short sin6_family; // AF_INET6.
+        unsigned short sin6_port;   // Transport level port number.
+        in6_addr sin6_addr;         // IPv6 address.
+    };
+
+    // Emulate addrinfo, it will be used for IPV4 loopups
+    struct addrinfo
+    {
+        int              ai_flags;
+        int              ai_family;
+        int              ai_socktype;
+        int              ai_protocol;
+        socklen_t        ai_addrlen;
+        sockaddr*        ai_addr;
+        char*            ai_canonname;
+        addrinfo*        ai_next;
+    };
+
+    static in6_addr in6addr_loopback = {
+        { 0 }
+    };
+
+    // Emulate ipv6_mreq structure, it will never be used
+    struct ipv6_mreq
+    {
+        in6_addr ipv6mr_multiaddr;          // IPv6 multicast address.
+        unsigned long ipv6mr_interface;     // Interface index.
+    };
+}
+#else
+    struct addrinfo;
+#endif
+
+namespace GridMate
+{
+    using SocketErrorBuffer = AZStd::array<char, 32>;
+
+    class SocketDriverAddress
+        : public DriverAddress
+    {
+        friend class SocketDriver;
+        SocketDriverAddress();
+    public:
+        struct Hasher
+        {
+            AZStd::size_t operator()(const SocketDriverAddress& v) const;
+        };
+
+        SocketDriverAddress(Driver* driver);
+        SocketDriverAddress(Driver* driver, const sockaddr* addr);
+        SocketDriverAddress(Driver* driver, const string& ip, unsigned int port);
+
+        bool operator==(const SocketDriverAddress& rhs) const;
+        bool operator!=(const SocketDriverAddress& rhs) const;
+
+        virtual string ToString() const;
+        virtual string ToAddress() const;
+        virtual string GetIP() const;
+        virtual unsigned int  GetPort() const;
+        virtual const void* GetTargetAddress(unsigned int& addressSize) const;
+
+        union
+        {
+            sockaddr_in     m_sockAddr;
+            sockaddr_in6    m_sockAddr6;
+        };
+    };
+
+    /**
+     * Base common class for all SocketBased drivers, you can't NOT create an instance of the SocketDriverCommon
+     * use CreateSocketDriver function for a BSD socket driver.
+     */
+    class SocketDriverCommon
+        : public Driver
+    {
+    public:
+        using SocketType = Platform::SocketType_Platform;
+        SocketDriverCommon(bool isFullPackets = false, bool isCrossPlatform = false, bool isHighPerformance = false);
+        virtual ~SocketDriverCommon();
+
+        /**
+        * Platform specific functionality.
+        */
+        /// Return maximum number of active connections at the same time.
+        virtual unsigned int GetMaxNumConnections() const       { return 32; }
+        /// Return maximum data size we can send/receive at once in bytes, supported by the platform.
+        virtual unsigned int GetMaxSendSize() const;
+        /// Return packet overhead size in bytes.
+        virtual unsigned int GetPacketOverheadSize() const;
+
+        /**
+         * User should implement create and bind a UDP socket. This socket will be used for all communications.
+         * \param ft family type, for the BSD socket it can be AFT_IPV4 or AFT_IPV6.
+         * \param address when 0 it we will assume "any address".
+         * \param port When left 0, we use implicit bind (assigned by the system). Otherwise provide a valid port number.
+         * \param isBroadcast is valid for Ipv4 only (otherwise ignored). Sets the socket to support broadcasts.
+         * \param receiveBufferSize socket receive buffer size in bytes, use 0 for default values.
+         * \param sendBufferSize socket send buffer size, use 0 for default values.
+         */
+        virtual ResultCode  Initialize(int familyType = BSD_AF_INET, const char* address = nullptr, unsigned int port = 0, bool isBroadcast = false, unsigned int receiveBufferSize = 0, unsigned int sendBufferSize = 0);
+
+        /// Returns communication port (must be called after Initialize, otherwise it will return 0)
+        virtual unsigned int GetPort() const;
+
+        /// Send data to a user defined address
+        virtual ResultCode  Send(const AZStd::intrusive_ptr<DriverAddress>& to, const char* data, unsigned int dataSize);
+        /**
+        * Receives a datagram and stores the source address. maxDataSize must be >= than GetMaxSendSize(). Returns the num of of received bytes.
+        * \note If a datagram from a new connection is received, NewConnectionCB will be called. If it rejects the connection the returned from pointer
+        * will be NULL while the actual data will be returned.
+        */
+        virtual unsigned int Receive(char* data, unsigned int maxDataSize, AZStd::intrusive_ptr<DriverAddress>& from, ResultCode* resultCode = 0);
+        /**
+         *  Wait for data to be to the ready for receive. Time out is the maximum time to wait
+         * before this function returns. If left to default value it will be in blocking mode (wait until data is ready to be received).
+         * \returns true if there is data to be received (always true if timeOut == 0), otherwise false.
+         */
+        virtual bool WaitForData(AZStd::chrono::microseconds timeOut = AZStd::chrono::microseconds(0));
+
+        /**
+         * When you enter wait for data mode, for many reasons you might want to stop wait for data.
+         * If you implement this function you need to make sure it's a thread safe function.
+         */
+        virtual void StopWaitForData();
+
+        /// Return true if WaitForData was interrupted before the timeOut expired, otherwise false.
+        virtual bool WasStopeedWaitingForData()                 { return m_isStoppedWaitForData; }
+
+        /// @{ Address conversion functionality. They MUST implemented thread safe. Generally this is not a problem since they just part local data.
+        ///  Create address from ip and port. If ip == NULL we will assign a broadcast address.
+        virtual string  IPPortToAddress(const char* ip, unsigned int port) const                        { return IPPortToAddressString(ip, port); }
+        virtual bool    AddressToIPPort(const string& address, string& ip, unsigned int& port) const    { return AddressStringToIPPort(address, ip, port); }
+        /// Create address for the socket driver from IP and port
+        static string   IPPortToAddressString(const char* ip, unsigned int port);
+        /// Decompose an address to IP and port
+        static bool     AddressStringToIPPort(const string& address, string& ip, unsigned int& port);
+        /// Return the family type of the address (AF_INET,AF_INET6 AF_UNSPEC)
+        static BSDSocketFamilyType  AddressFamilyType(const string& ip);
+        static BSDSocketFamilyType  AddressFamilyType(const char* ip)           { return AddressFamilyType(string(ip)); }
+        /// @}
+
+        virtual AZStd::intrusive_ptr<DriverAddress> CreateDriverAddress(const string& address) = 0;
+        /// Additional CreateDriverAddress function should be implemented.
+        virtual AZStd::intrusive_ptr<DriverAddress> CreateDriverAddress(const sockaddr* sockAddr) = 0;
+
+    protected:
+        /// returns result of socket(af,type,protocol)
+        virtual SocketType  CreateSocket(int af, int type, int protocol);
+        /// returns the result of bind(sockAddr)
+        virtual int         BindSocket(const sockaddr* sockAddr, size_t sockAddrLen);
+        /// set's default socket options
+        virtual ResultCode  SetSocketOptions(bool isBroadcast, unsigned int receiveBufferSize, unsigned int sendBufferSize);
+
+        class PlatformSocketDriver
+        {
+        public:
+            GM_CLASS_ALLOCATOR(PlatformSocketDriver);
+            PlatformSocketDriver(SocketDriverCommon &parent, SocketType &socket);
+            virtual ~PlatformSocketDriver();
+            virtual ResultCode Initialize(unsigned int receiveBufferSize, unsigned int sendBufferSize);
+            virtual SocketType  CreateSocket(int af, int type, int protocol);
+            virtual ResultCode Send(const sockaddr* sockAddr, unsigned int addressSize, const char* data, unsigned int dataSize);
+            virtual unsigned int Receive(char* data, unsigned maxDataSize, sockaddr* sockAddr, socklen_t sockAddrLen, ResultCode* resultCode = 0);
+            virtual bool WaitForData(AZStd::chrono::microseconds timeOut = AZStd::chrono::microseconds(0));
+            virtual void StopWaitForData();
+            static bool isSupported();
+        protected:
+            SocketDriverCommon &m_parent;
+            SocketType         &m_socket;
+
+        };
+#ifdef AZ_SOCKET_RIO_SUPPORT
+        class RIOPlatformSocketDriver final : public PlatformSocketDriver
+        {
+        public:
+            GM_CLASS_ALLOCATOR(RIOPlatformSocketDriver);
+            RIOPlatformSocketDriver(SocketDriverCommon &parent, SocketType &socket);
+            ~RIOPlatformSocketDriver();
+            static bool isSupported();
+            SocketType CreateSocket(int af, int type, int protocol) override;
+            ResultCode Initialize(unsigned int receiveBufferSize, unsigned int sendBufferSize) override;
+            void WorkerSendThread();
+            ResultCode Send(const sockaddr* sockAddr, unsigned int /*addressSize*/, const char* data, unsigned int dataSize) override;
+            unsigned int Receive(char* data, unsigned maxDataSize, sockaddr* sockAddr, socklen_t sockAddrLen, ResultCode* resultCode) override;
+            bool WaitForData(AZStd::chrono::microseconds timeOut) override;
+            void StopWaitForData() override;
+
+        private:
+            AZ::u64 RoundUpAndDivide(AZ::u64 Value, AZ::u64 RoundTo) const
+            {
+                return ((Value + RoundTo - 1) / RoundTo);
+            }
+            AZ::u64 RoundUp(AZ::u64 Value, AZ::u64 RoundTo) const
+            {
+                // rounds value up to multiple of RoundTo
+                // Example: RoundTo: 4
+                // Value:  0 1 2 3 4 5 6 7 8
+                // Result: 0 4 4 4 4 8 8 8 8
+                return RoundUpAndDivide(Value, RoundTo) * RoundTo;
+            }
+            char *AllocRIOBuffer(AZ::u64 bufferSize, AZ::u64 numBuffers, AZ::u64* amountAllocated=nullptr);
+            bool FreeRIOBuffer(char *buffer);
+
+        protected:
+            //static const GUID               k_functionTableId = WSAID_MULTIPLE_RIO;
+            bool                            m_workersQuit = false;
+            AZStd::thread                   m_workerSendThread;
+            AZStd::mutex                    m_WorkerSendMutex;
+            AZStd::condition_variable       m_triggerWorkerSend;
+            AZStd::atomic<int>              m_workerBufferCount;
+            RIO_EXTENSION_FUNCTION_TABLE    m_RIO_FN_TABLE;
+            RIO_RQ                          m_requestQueue = RIO_INVALID_RQ;
+            RIO_CQ                          m_RIORecvQueue = RIO_INVALID_CQ;
+            AZStd::mutex                    m_RIOSendQueueMutex;
+            RIO_CQ                          m_RIOSendQueue = RIO_INVALID_CQ;
+            int                             m_RIONextSendBuffer = 0;
+            int                             m_workerNextSendBuffer = 0;
+            int                             m_RIONextRecvBuffer = 0;
+            int                             m_RIOSendBufferCount = 64;
+            int                             m_RIORecvBufferCount = 2048;
+            int                             m_RIOSendBuffersInUse = 0;
+            int                             m_RIORecvBuffersInUse = 0;
+            bool                            m_isInitialized = false;
+            AZ::u64                         m_pageSize = 0;
+            AZStd::vector<RIO_BUF>          m_RIORecvBuffer;
+            AZStd::vector<RIO_BUF>          m_RIORecvAddressBuffer;
+            AZStd::vector<RIO_BUF>          m_RIOSendBuffer;
+            AZStd::vector<RIO_BUF>          m_RIOSendAddressBuffer;
+            static const int                m_RIOBufferSize = 1536;
+            char*                           m_rawRecvBuffer = nullptr;
+            char*                           m_rawRecvAddressBuffer = nullptr;
+            char*                           m_rawSendBuffer = nullptr;
+            char*                           m_rawSendAddressBuffer = nullptr;
+            enum
+            {
+                ReceiveEvent = 0,
+                SendEvent,
+                WakeupOnSend,
+                NumberOfEvents
+            };
+            WSAEVENT                        m_events[NumberOfEvents];            //send and recv events
+        };
+#endif
+        SocketType      m_socket;
+        unsigned short  m_port;
+        bool            m_isStoppedWaitForData;     ///< True if last WaitForData was interrupted otherwise false.
+        bool            m_isFullPackets;            ///< True if we use max packet size vs internet safe packet size (64KB vs 1500 usually)
+        bool            m_isCrossPlatform;          ///< True if we support cross platform communication. Then we make sure we use common features.
+        bool            m_isIpv6;                   ///< True if we use version 6 of the internet protocol, otherwise false.
+        bool            m_isDatagram;               ///< True if the socket was created with SOCK_DGRAM
+        AZStd::unique_ptr<PlatformSocketDriver> m_platformDriver;      ///< Platform specific implementation of socket calls
+        bool            m_isHighPerformance;        ///< True if using platform-specific high-performance implementation
+    };
+
+    namespace SocketOperations
+    {
+        AZ::u32 HostToNetLong(AZ::u32 hstLong);
+        AZ::u32 NetToHostLong(AZ::u32 netLong);
+        AZ::u16 HostToNetShort(AZ::u16 hstShort);
+        AZ::u16 NetToHostShort(AZ::u16 netShort);
+
+        SocketDriverCommon::SocketType CreateSocket(bool isDatagram, Driver::BSDSocketFamilyType familyType);
+
+        enum class SocketOption : AZ::s32
+        {
+            NonBlockingIO,
+            ReuseAddress,
+            KeepAlive,
+            Broadcast,
+            SendBuffer,
+            ReceiveBuffer
+        };
+        Driver::ResultCode SetSocketOptionValue(SocketDriverCommon::SocketType sock, SocketOption option, const char* optval, AZ::s32 optlen);
+        Driver::ResultCode SetSocketOptionBoolean(SocketDriverCommon::SocketType sock, SocketOption option, bool enable);
+        Driver::ResultCode EnableTCPNoDelay(SocketDriverCommon::SocketType sock, bool enable);
+        Driver::ResultCode SetSocketBlockingMode(SocketDriverCommon::SocketType sock, bool blocking);
+        Driver::ResultCode SetSocketLingerTime(SocketDriverCommon::SocketType sock, bool bDoLinger, AZ::u16 timeout);
+
+        enum class ConnectionResult : AZ::s32
+        {
+            Okay,
+            AlreadyConnecting,
+            Refused,
+            InProgress,
+            ConnectFailed,
+            NetworkUnreachable,
+            TimedOut,
+            SocketConnected
+        };
+        Driver::ResultCode Connect(SocketDriverCommon::SocketType sock, const sockaddr* sockAddr, size_t sockAddrSize, ConnectionResult& outConnectionResult);
+        Driver::ResultCode Connect(SocketDriverCommon::SocketType sock, const SocketDriverAddress& addr, ConnectionResult& outConnectionResult);
+        Driver::ResultCode Listen(SocketDriverCommon::SocketType sock, AZ::s32 backlog);
+        Driver::ResultCode Accept(SocketDriverCommon::SocketType sock, sockaddr* outAddr, socklen_t& outAddrSize, SocketDriverCommon::SocketType& outSocket);
+        Driver::ResultCode CloseSocket(SocketDriverCommon::SocketType sock);
+        Driver::ResultCode Send(SocketDriverCommon::SocketType sock, const char* buf, AZ::u32 bufLen, AZ::u32& bytesSent);
+        Driver::ResultCode Receive(SocketDriverCommon::SocketType sock, char* buf, AZ::u32& inOutlen);
+        Driver::ResultCode Bind(SocketDriverCommon::SocketType sock, const sockaddr* sockAddr, size_t sockAddrSize);
+        bool IsWritable(SocketDriverCommon::SocketType sock, AZStd::chrono::microseconds timeOut);
+        bool IsReceivePending(SocketDriverCommon::SocketType sock, AZStd::chrono::microseconds timeOut);
+    }
+
+    class SocketDriver
+        : public SocketDriverCommon
+    {
+        friend class SocketDriverAddress;
+    public:
+        GM_CLASS_ALLOCATOR(SocketDriver);
+
+        SocketDriver(bool isFullPackets, bool isCrossPlatform, bool isHighPerformance = false)
+            : SocketDriverCommon(isFullPackets, isCrossPlatform, isHighPerformance) {}
+        virtual ~SocketDriver() {}
+
+        /**
+        * Creates internal driver address to be used for send/receive calls.
+        * \note if the ip and the port are the same, the same pointer will be returned. You can use the returned pointer
+        * to compare for unique addresses.
+        * \note Driver address allocates internal resources, use it only when you intend to communicate. Otherwise operate with
+        * the string address.
+        */
+        virtual AZStd::intrusive_ptr<DriverAddress> CreateDriverAddress(const string& address);
+        virtual AZStd::intrusive_ptr<DriverAddress> CreateDriverAddress(const sockaddr* addr);
+        /// Called only from the DriverAddress when the use count becomes 0
+        virtual void    DestroyDriverAddress(DriverAddress* address);
+
+        typedef AZStd::unordered_set<SocketDriverAddress, SocketDriverAddress::Hasher> AddressSetType;
+        AddressSetType  m_addressMap;
+    };
+
+    namespace Utils
+    {
+        ///< Retrieves ip address corresponding to a host name. Blocks thread until dns resolving is happened.
+        bool GetIpByHostName(int familyType, const char* hostName, string& ip);
+    }
+
+    /**
+    * Utility class to help retrieve socket address information
+    */
+    class SocketAddressInfo
+    {
+    public:
+        SocketAddressInfo();
+        ~SocketAddressInfo();
+
+        void Reset();
+
+        enum class AdditionalOptionFlags
+        {
+            None = 0x00,        // Nothing to specify
+            Passive = 0x01,     // For wild card IP address
+            NumericHost = 0x02, // then 'address' must be a numerical network address
+        };
+
+        /**
+        * Resolves the an address for either the local host machine (when address is nullptr) or a remote address where address points to a valid string
+        * \param address when nullptr it we will assume "any address".
+        * \param port When left 0, we use implicit bind (assigned by the system); in native Endian
+        * \param familyType family type, for the BSD socket it can be BSD_AF_INET or BSD_AF_INET6
+        * \param isDatagram When True then the address hint with be SOCK_DGRAM otherwise SOCK_STREAM
+        * \param flags combined AI_* flags to use as a hints
+        */
+        bool Resolve(const char* address, AZ::u16 port, Driver::BSDSocketFamilyType familyType, bool isDatagram, AdditionalOptionFlags flags);
+
+        /**
+          * If Resolve() is True, then this returns the address information requested to resolve
+          */
+        const addrinfo* GetAddressInfo() const { return m_addrInfo; }
+
+        /**
+          * If Resolve() is True and valid socket, return the assigned port after a succesful bind() call
+          */
+        AZ::u16 RetrieveSystemAssignedPort(SocketDriverCommon::SocketType socket) const;
+
+    private:
+        addrinfo* m_addrInfo;
+    };
+}
+
+

Code/Framework/GridMate/GridMate/Carrier/StreamSecureSocketDriver.cpp

@@ -0,0 +1,767 @@
+/*
+* All or portions of this file Copyright (c) Amazon.com, Inc. or its affiliates or
+* its licensors.
+*
+* For complete copyright and license terms please see the LICENSE at the root of this
+* distribution (the "License"). All use of this software is governed by the License,
+* or, if provided, by the license below or the license accompanying this file. Do not
+* remove or modify any license notices. This file is distributed on an "AS IS" BASIS,
+* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+*
+*/
+
+#include <GridMate/Carrier/StreamSecureSocketDriver.h>
+
+#if AZ_TRAIT_GRIDMATE_ENABLE_OPENSSL
+
+#include <AzCore/Math/MathUtils.h>
+
+#include <openssl/ssl.h>
+#include <openssl/err.h>
+#include <openssl/bio.h>
+#include <openssl/pem.h>
+#include <openssl/x509.h>
+#include <openssl/rand.h>
+#include <openssl/hmac.h>
+
+//////////////////////////////////////////////////////////////////////////
+// StreamSecureSocketDriver
+//////////////////////////////////////////////////////////////////////////
+namespace GridMate
+{
+    X509* CreateCertificateFromEncodedPEM(const char* encodedPem);
+    EVP_PKEY* CreatePrivateKeyFromEncodedPEM(const char* encodedPem);
+    void CreateCertificateChainFromEncodedPEM(const char* encodedPem, AZStd::vector<X509*>& certificateChain);
+
+    static AZStd::atomic_int s_initializeOpenSSLCount;
+
+    bool InitializeOpenSSL()
+    {
+        if (s_initializeOpenSSLCount.fetch_add(1) == 0)
+        {
+            SSL_library_init();
+            SSL_load_error_strings();
+            ERR_load_BIO_strings();
+            OpenSSL_add_all_algorithms();
+        }
+        return true;
+    }
+
+    bool ShutdownOpenSSL()
+    {
+        if (s_initializeOpenSSLCount.fetch_sub(1) == 1)
+        {
+            ERR_remove_state(0);
+            ERR_free_strings();
+            EVP_cleanup();
+            sk_SSL_COMP_free(SSL_COMP_get_compression_methods());
+            CRYPTO_cleanup_all_ex_data();
+        }
+        return true;
+    }
+}
+
+//////////////////////////////////////////////////////////////////////////
+// SecureContextHandle
+//////////////////////////////////////////////////////////////////////////
+namespace GridMate
+{
+    struct SecureContextHandleImpl
+        : public StreamSecureSocketDriver::SecureContextHandle
+    {
+        GM_CLASS_ALLOCATOR(SecureContextHandleImpl);
+
+        SecureContextHandleImpl()
+            : m_ctx(nullptr)
+            , m_privateKey(nullptr)
+            , m_certificate(nullptr)
+        {
+        }
+
+        ~SecureContextHandleImpl() override
+        {
+            Teardown();
+        }
+
+        Driver::ResultCode Prepare(StreamSecureSocketDriver::StreamSecureSocketDriverDesc& desc)
+        {
+            if (!InitializeOpenSSL())
+            {
+                return Driver::EC_SECURE_CREATE;
+            }
+            m_ctx = SSL_CTX_new(TLSv1_2_method());
+            if (m_ctx == nullptr)
+            {
+                return Driver::EC_SECURE_CREATE;
+            }
+
+            // Only support a single cipher suite in OpenSSL that supports:
+            //
+            //  ECDHE       Master key exchange using ephemeral elliptic curve diffie-hellman.
+            //  RSA         Authentication (public and private key) used to sign ECDHE parameters and can be checked against a CA.
+            //  AES256      AES cipher for symmetric key encryption using a 256-bit key.
+            //  GCM         Mode of operation for symmetric key encryption.
+            //  SHA384      SHA-2 hashing algorithm.
+            if (SSL_CTX_set_cipher_list(m_ctx, "ECDHE-RSA-AES256-GCM-SHA384") != 1)
+            {
+                return Driver::EC_SECURE_CREATE;
+            }
+
+            // Automatically generate parameters for elliptic-curve diffie-hellman (i.e. curve type and coefficients).
+            SSL_CTX_set_ecdh_auto(m_ctx, 1);
+
+            if (desc.m_privateKeyPEM || desc.m_certificatePEM)
+            {
+                if (desc.m_certificatePEM)
+                {
+                    m_certificate = CreateCertificateFromEncodedPEM(desc.m_certificatePEM);
+                    if (m_certificate == nullptr || SSL_CTX_use_certificate(m_ctx, m_certificate) != 1)
+                    {
+                        return Driver::EC_SECURE_CERT;
+                    }
+                }
+                else
+                {
+                    AZ_TracePrintf("GridMateSecure", "If a private key is provided, so must a corresponding certificate.\n");
+                    return Driver::EC_SECURE_CONFIG;
+                }
+
+                if (desc.m_privateKeyPEM)
+                {
+                    m_privateKey = CreatePrivateKeyFromEncodedPEM(desc.m_privateKeyPEM);
+                    if (m_privateKey == nullptr || SSL_CTX_use_PrivateKey(m_ctx, m_privateKey) != 1)
+                    {
+                        return Driver::EC_SECURE_PKEY;
+                    }
+                }
+                else
+                {
+                    AZ_TracePrintf("GridMateSecure", "If a certificate is provided, so must a corresponding private key.\n");
+                    return Driver::EC_SECURE_PKEY;
+                }
+            }
+            
+            // Determine if both client and server must be authenticated or only the server.
+            // The default behavior only authenticates the server, and not the client.
+            int verificationMode = SSL_VERIFY_PEER;
+            if (desc.m_authenticateClient)
+            {
+                verificationMode = SSL_VERIFY_FAIL_IF_NO_PEER_CERT;
+            }
+
+            if (desc.m_certificateAuthorityPEM)
+            {
+                // SSL context should already have empty cert storage
+                X509_STORE* caLocalStore = SSL_CTX_get_cert_store(m_ctx);
+                if (caLocalStore == nullptr)
+                {
+                    return Driver::EC_SECURE_CA_CERT;
+                }
+
+                AZStd::vector<X509*> certificateChain;
+                CreateCertificateChainFromEncodedPEM(desc.m_certificateAuthorityPEM, certificateChain);
+                if (certificateChain.size() == 0)
+                {
+                    return Driver::EC_SECURE_CA_CERT;
+                }
+
+                for (auto certificate : certificateChain)
+                {
+                    X509_STORE_add_cert(caLocalStore, certificate);
+                }
+                SSL_CTX_set_verify(m_ctx, verificationMode, nullptr);
+            }
+            else
+            {
+                auto fnVerifyCertificate = [](int ok, X509_STORE_CTX* ctx) -> int
+                {
+                    // Called when a certificate has been received and needs to be verified (e.g.
+                    // verify that it has been signed by the appropriate CA, has the correct
+                    // hostname, etc).
+                    (void)ok;
+                    (void)ctx;
+                    return 1;
+                };
+                SSL_CTX_set_verify(m_ctx, verificationMode, fnVerifyCertificate);
+            }
+
+            return Driver::EC_OK;
+        }
+
+        bool Teardown()
+        {
+            if (m_certificate)
+            {
+                X509_free(m_certificate);
+                m_certificate = nullptr;
+            }
+
+            if (m_privateKey)
+            {
+                EVP_PKEY_free(m_privateKey);
+                m_privateKey = nullptr;
+            }
+
+            if (m_ctx)
+            {
+                // Calls to SSL_CTX_free() also free any attached X509_STORE objects.
+                SSL_CTX_free(m_ctx);
+                m_ctx = nullptr;
+            }
+
+            return ShutdownOpenSSL();
+        }
+
+        SSL_CTX* m_ctx; // main SSL context 
+        EVP_PKEY* m_privateKey;
+        X509* m_certificate;
+    };
+}
+
+//////////////////////////////////////////////////////////////////////////
+// StreamSecureSocketDriver
+//////////////////////////////////////////////////////////////////////////
+namespace GridMate
+{
+    StreamSecureSocketDriver::StreamSecureSocketDriver(AZ::u32 maxConnections, AZ::u32 maxPacketSize, AZ::u32 inboundBufferSize, AZ::u32 outboundBufferSize)
+        : StreamSocketDriver(maxConnections, maxPacketSize, inboundBufferSize, outboundBufferSize)
+    {
+    }
+
+    StreamSecureSocketDriver::~StreamSecureSocketDriver()
+    {
+        m_handle.release();
+        m_connectionFactory = [](AZ::u32 inboundBufferSize, AZ::u32 outputBufferSize)
+        {
+            (void)inboundBufferSize;
+            (void)outputBufferSize;
+            AZ_TracePrintf("GridMateSecure", "Tried to create a new connection during shutdown.\n");
+            return (SecureConnection*)nullptr;
+        };
+    }
+
+    Driver::ResultCode StreamSecureSocketDriver::InitializeSecurity(AZ::s32 familyType, const char* address, AZ::u32 port, AZ::u32 receiveBufferSize, AZ::u32 sendBufferSize, StreamSecureSocketDriverDesc& desc)
+    {
+        Driver::ResultCode code = Initialize(familyType, address, port, false, receiveBufferSize, sendBufferSize);
+        if (code != EC_OK)
+        {
+            return code;
+        }
+        SecureContextHandleImpl* pHandle = aznew SecureContextHandleImpl();
+        code = pHandle->Prepare(desc);
+        if (code != Driver::EC_OK)
+        {
+            delete pHandle;
+            return code;
+        }
+        m_handle.reset(pHandle);
+        m_connectionFactory = [this](AZ::u32 inboundBufferSize, AZ::u32 outputBufferSize)
+        {
+            if (m_handle)
+            {
+                return aznew SecureConnection(inboundBufferSize, outputBufferSize, *m_handle.get());
+            }
+            return (SecureConnection*)nullptr;
+        };
+        return EC_OK;
+    }
+}
+
+//////////////////////////////////////////////////////////////////////////
+// SecureConnectionContext
+//////////////////////////////////////////////////////////////////////////
+namespace GridMate
+{
+    struct SecureConnectionContextImpl
+        : public StreamSecureSocketDriver::SecureConnectionContext
+    {
+        GM_CLASS_ALLOCATOR(SecureConnectionContextImpl);
+
+        explicit SecureConnectionContextImpl(SSL_CTX* sslContext, AZ::u32 scratchSize)
+            : m_ssl(nullptr)
+            , m_bioIn(nullptr)
+            , m_bioOut(nullptr)
+            , m_sslCtx(sslContext)
+            , m_scratch(nullptr)
+            , m_scratchSize(scratchSize)
+        {
+        }
+
+        ~SecureConnectionContextImpl() override
+        {
+            Teardown();
+        }
+
+        SecureConnectionContextImpl(const SecureConnectionContextImpl &) = delete;
+        SecureConnectionContextImpl& operator =(const SecureConnectionContextImpl&) = delete;
+
+        bool PrepareToAccept() override
+        {
+            if (Prepare())
+            {
+                SSL_set_accept_state(m_ssl);
+                return true;
+            }
+            return false;
+        }
+
+        bool PrepareToConnect() override
+        {
+            if (Prepare())
+            {
+                SSL_set_connect_state(m_ssl);
+                return true;
+            }
+            return false;
+        }
+
+        bool Prepare()
+        {
+            do
+            {
+                m_ssl = SSL_new(m_sslCtx);
+                if (m_ssl == nullptr)
+                {
+                    AZ_Warning("GridMate", m_ssl == nullptr, "SSL_new() failed!");
+                    return false;
+                }
+
+                m_bioIn = BIO_new(BIO_s_mem());
+                if (m_bioIn == nullptr)
+                {
+                    AZ_Warning("GridMate", m_bioIn == nullptr, "BIO_new() for m_bioIn failed.");
+                    break;
+                }
+
+                m_bioOut = BIO_new(BIO_s_mem());
+                if (m_bioOut == nullptr)
+                {
+                    AZ_Warning("GridMate", m_bioOut == nullptr, "BIO_new() for m_bioOut failed.");
+                    break;
+                }
+
+                m_scratch = static_cast<char*>(azmalloc(m_scratchSize));
+                if (m_scratch == nullptr)
+                {
+                    AZ_Warning("GridMate", m_scratch == nullptr, "Could not allocate scratch buffer.");
+                    break;
+                }
+            }
+            while (0);
+
+            // did everything successfully create and/or allocate?
+            if (m_ssl && m_bioIn && m_bioOut && m_scratch)
+            {
+                BIO_set_mem_eof_return(m_bioIn, -1);
+                BIO_set_mem_eof_return(m_bioOut, -1);
+                SSL_set_bio(m_ssl, m_bioIn, m_bioOut);
+                return true;
+            }
+
+            Teardown();
+            return false;
+        }
+
+        void Teardown()
+        {
+            if (m_scratch != nullptr)
+            {
+                azfree(m_scratch);
+                m_scratch = nullptr;
+            }
+
+            if (m_ssl != nullptr)
+            {
+                SSL_free(m_ssl);
+                m_ssl = nullptr;
+            }
+
+            if (m_bioIn != nullptr)
+            {
+                BIO_free(m_bioIn);
+                m_bioIn = nullptr;
+            }
+
+            if (m_bioOut != nullptr)
+            {
+                BIO_free(m_bioOut);
+                m_bioOut = nullptr;
+            }
+        }
+
+        SSL* m_ssl;                     // the SSL which represents a "connection"
+        BIO* m_bioIn;                   // we use memory read BIO
+        BIO* m_bioOut;                  // we use memory write BIO
+        SSL_CTX* m_sslCtx;              // the parent context this SSL instance belongs
+        char* m_scratch;                // a scratch buffer to temporarily read and write
+        const AZ::u32 m_scratchSize;    // the size of the scratch buffer
+    };
+}
+
+//////////////////////////////////////////////////////////////////////////
+// StreamSecureSocketDriver::SecureConnection
+//////////////////////////////////////////////////////////////////////////
+namespace GridMate
+{
+    SecureConnectionContextImpl* CastContext(AZStd::unique_ptr<StreamSecureSocketDriver::SecureConnectionContext>& ptr)
+    {
+        return static_cast<SecureConnectionContextImpl*>(ptr.get());
+    }
+
+    StreamSecureSocketDriver::SecureConnection::SecureConnection(AZ::u32 inboundBufferSize, AZ::u32 outputBufferSize, StreamSecureSocketDriver::SecureContextHandle& handle)
+        : Connection(inboundBufferSize, outputBufferSize)
+        , m_inboundRawBuffer(inboundBufferSize)
+    {
+        SecureContextHandleImpl* refHandle = static_cast<SecureContextHandleImpl*>(&handle);
+        m_context.reset(aznew SecureConnectionContextImpl(refHandle->m_ctx, AZ::GetMax(inboundBufferSize, outputBufferSize)));
+    }
+
+    StreamSecureSocketDriver::SecureConnection::~SecureConnection()
+    {
+        m_context.release();
+    }
+
+    bool StreamSecureSocketDriver::SecureConnection::SendPacket(const char* data, AZ::u32 dataSize)
+    {
+        const SecureConnectionContextImpl* ctx = CastContext(m_context);
+        const AZ::u32 kMaxPacketSendSize = (1 << (sizeof(AZ::u16) * 8)) - 1;
+        const AZ::s32 kPacketDelimeterSize = sizeof(decltype(Packet::m_size));
+
+        if (nullptr == ctx || nullptr == ctx->m_ssl)
+        {
+            return false;
+        }
+        else if (!IsValidPacketDataSize(dataSize, kMaxPacketSendSize))
+        {
+            return false;
+        }
+        else if ((kPacketDelimeterSize + dataSize) > ctx->m_scratchSize)
+        {
+            AZ_TracePrintf("GridMate", "Failed to memory write the packet");
+            return false;
+        }
+
+        AZ::u16 packetSize = static_cast<AZ::u16>(dataSize);
+        packetSize = SocketOperations::HostToNetShort(packetSize);
+
+        std::memcpy(ctx->m_scratch, &packetSize, kPacketDelimeterSize);
+        std::memcpy(ctx->m_scratch + kPacketDelimeterSize, data, dataSize);
+
+        AZ::s32 wrote = SSL_write(ctx->m_ssl, ctx->m_scratch, kPacketDelimeterSize + dataSize);
+        if (wrote > 0)
+        {
+            AZ_Warning("GridMate", wrote == static_cast<AZ::s32>(kPacketDelimeterSize + dataSize), "SSL_write only wrote %d", wrote);
+            return true;
+        }
+        else
+        {
+            // In this case a call to SSL_get_error with the return value of SSL_write() will yield SSL_ERROR_WANT_READ or SSL_ERROR_WANT_WRITE.
+            if (SSL_get_error(ctx->m_ssl, wrote) == SSL_ERROR_WANT_READ || SSL_get_error(ctx->m_ssl, wrote) == SSL_ERROR_WANT_WRITE)
+            {
+                AZ_TracePrintf("GridMate", "Writing was blocked by an internal SSL process.\n");
+                return true;
+            }
+        }
+        return false;
+    }
+
+    bool StreamSecureSocketDriver::SecureConnection::OnStateAccept(AZ::HSM& sm, const AZ::HSM::Event& e)
+    {
+        if (e.id == AZ::HSM::EnterEventId)
+        {
+            if (m_context->PrepareToAccept())
+            {
+                return true;
+            }
+            else
+            {
+                sm.Transition(static_cast<AZ::HSM::StateId>(ConnectionState::IN_ERROR));
+                return true;
+            }
+        }
+
+        if (e.id != ConnectionEvents::CE_UPDATE)
+        {
+            return StreamSocketDriver::Connection::OnStateAccept(sm, e);
+        }
+
+        if (!m_socketErrors.empty())
+        {
+            sm.Transition(static_cast<AZ::HSM::StateId>(ConnectionState::IN_ERROR));
+            return true;
+        }
+
+        return ProcessHandshake();
+    }
+
+    bool StreamSecureSocketDriver::SecureConnection::OnStateConnect(AZ::HSM& sm, const AZ::HSM::Event& e)
+    {
+        if (e.id == AZ::HSM::EnterEventId)
+        {
+            if (m_context->PrepareToConnect())
+            {
+                return true;
+            }
+            else
+            {
+                sm.Transition(static_cast<AZ::HSM::StateId>(ConnectionState::IN_ERROR));
+                return true;
+            }
+        }
+
+        if (e.id != ConnectionEvents::CE_UPDATE)
+        {
+            return StreamSocketDriver::Connection::OnStateConnect(sm, e);
+        }
+
+        if (!m_socketErrors.empty())
+        {
+            sm.Transition(static_cast<AZ::HSM::StateId>(ConnectionState::IN_ERROR));
+            return true;
+        }
+
+        return ProcessHandshake();
+    }
+
+    Driver::ErrorCodes StreamSecureSocketDriver::SecureConnection::ReadBytes(void* buf, AZ::u32& num, bool& resume)
+    {
+        const SecureConnectionContextImpl* ctx = CastContext(m_context);
+
+        resume = true;
+        const AZ::u32 bytesToRead = AZ::GetMin(num, ctx->m_scratchSize);
+        AZ::s32 ret = SSL_read(ctx->m_ssl, buf, bytesToRead);
+        switch (SSL_get_error(ctx->m_ssl, ret))
+        {
+            case SSL_ERROR_NONE:
+            {
+                num = ret;
+                return Driver::EC_OK;
+            }
+            case SSL_ERROR_ZERO_RETURN:
+            {
+                // end of data
+                num = 0;
+                resume = false;
+                return Driver::EC_OK;
+            }
+            case SSL_ERROR_WANT_READ:
+            {
+                num = 0;
+                return Driver::EC_OK;
+            }
+            case SSL_ERROR_WANT_WRITE:
+            {
+                num = 0;
+                return Driver::EC_OK;
+            }
+            default:
+            {
+                num = 0;
+                resume = false;
+                break;
+            }
+        }
+        return Driver::EC_RECEIVE;
+    }
+
+    bool StreamSecureSocketDriver::SecureConnection::WriteIntoRingBufferSafe(RingBuffer& inboundBuffer)
+    {
+        AZ::u32 inOutBytesSize = 0;
+        char* bytesBuffer = inboundBuffer.ReserveForWrite(inOutBytesSize);
+        if (bytesBuffer == nullptr || inOutBytesSize == 0)
+        {
+            AZ_TracePrintf("GridMate", "Connection read buffer is full for %s\n", m_remoteAddress->ToString().c_str());
+            return false;
+        }
+
+        bool resume = true;
+        if (ReadBytes(bytesBuffer, inOutBytesSize, resume) != Driver::EC_OK)
+        {
+            StoreLastSocketError();
+            m_sm.Transition(static_cast<AZ::HSM::StateId>(ConnectionState::IN_ERROR));
+            return true;
+        }
+
+        // should continue?
+        if (!resume)
+        {
+            m_sm.Transition(static_cast<AZ::HSM::StateId>(ConnectionState::DISCONNECTED));
+            return true;
+        }
+
+        // nothing read in the buffer, thus nothing to store in inboundBuffer
+        if (inOutBytesSize == 0)
+        {
+            return false;
+        }
+
+        bool bWroteToMarker = false;
+        inboundBuffer.CommitAsWrote(inOutBytesSize, bWroteToMarker);
+        if (bWroteToMarker && inOutBytesSize > 0)
+        {
+            // attempt to fill out the other side of the ring buffer
+            inOutBytesSize = 0;
+            bytesBuffer = inboundBuffer.ReserveForWrite(inOutBytesSize);
+            if (bytesBuffer == nullptr || inOutBytesSize == 0)
+            {
+                AZ_TracePrintf("GridMate", "Connection read buffer is full for %s\n", m_remoteAddress->ToString().c_str());
+                return false;
+            }
+            if (ReadBytes(bytesBuffer, inOutBytesSize, resume) != Driver::EC_OK)
+            {
+                StoreLastSocketError();
+                m_sm.Transition(static_cast<AZ::HSM::StateId>(ConnectionState::IN_ERROR));
+                return true;
+            }
+            if (!resume)
+            {
+                m_sm.Transition(static_cast<AZ::HSM::StateId>(ConnectionState::DISCONNECTED));
+                return true;
+            }
+            inboundBuffer.CommitAsWrote(inOutBytesSize, bWroteToMarker);
+        }
+        return false;
+    }
+
+    bool StreamSecureSocketDriver::SecureConnection::OnStateEstablished(AZ::HSM& sm, const AZ::HSM::Event& e)
+    {
+        if (e.id != ConnectionEvents::CE_UPDATE)
+        {
+            return StreamSocketDriver::Connection::OnStateEstablished(sm, e);
+        }
+
+        if (!m_socketErrors.empty())
+        {
+            sm.Transition(static_cast<AZ::HSM::StateId>(ConnectionState::IN_ERROR));
+            return true;
+        }
+
+        bool didSwitch = ProcessNetwork();
+        if (didSwitch)
+        {
+            return didSwitch;
+        }
+
+        const SecureConnectionContextImpl* ctx = CastContext(m_context);
+        do
+        {
+            // anything to read from SSL connection (aka inbound traffic)?
+            if (WriteIntoRingBufferSafe(m_inboundBuffer))
+            {
+                return true;
+            }
+        } 
+        while (SSL_pending(ctx->m_ssl));
+
+        // no state change, yet
+        return false;
+    }
+
+    bool StreamSecureSocketDriver::SecureConnection::ProcessNetwork()
+    {
+        const SecureConnectionContextImpl* ctx = CastContext(m_context);
+
+        // read from the socket first
+        bool switchState = false;
+        ProcessInbound(switchState, m_inboundRawBuffer);
+        if (switchState)
+        {
+            return true;
+        }
+
+        // network traffic to process?
+        if (m_inboundRawBuffer.GetSpaceToRead() > 0)
+        {
+            AZ::u32 bytesToRead = AZ::GetMin(ctx->m_scratchSize, m_inboundRawBuffer.GetSpaceToRead());
+            if (m_inboundRawBuffer.Fetch(ctx->m_scratch, bytesToRead))
+            {
+                AZ::s32 wrote = BIO_write(ctx->m_bioIn, ctx->m_scratch, bytesToRead);
+                if (wrote <= 0)
+                {
+                    if (!BIO_should_retry(ctx->m_bioIn))
+                    {
+                        StoreLastSocketError();
+                        return false;
+                    }
+                }
+            }
+        }
+
+        // anything written into the SSL to be sent out?
+        size_t pending = BIO_ctrl_pending(ctx->m_bioOut);
+        while (pending > 0 && m_outboundBuffer.GetSpaceToWrite() >= static_cast<AZ::u32>(pending))
+        {
+            AZ::s32 read = BIO_read(ctx->m_bioOut, ctx->m_scratch, ctx->m_scratchSize);
+            if (read > 0)
+            {
+                m_outboundBuffer.Store(ctx->m_scratch, static_cast<AZ::u32>(read));
+            }
+            else if (read == 0)
+            {
+                // closed
+                m_sm.Transition(static_cast<AZ::HSM::StateId>(ConnectionState::DISCONNECTED));
+                return true;
+            }
+            else
+            {
+                if (BIO_should_retry(ctx->m_bioOut))
+                {
+                    // try again latter
+                    break;
+                }
+                StoreLastSocketError();
+                return false;
+            }
+            pending = BIO_ctrl_pending(ctx->m_bioOut);
+        }
+
+        // process any more out going network traffic
+        switchState = false;
+        ProcessOutbound(switchState, m_outboundBuffer);
+        if (switchState)
+        {
+            return true;
+        }
+
+        // no state machine change
+        return false;
+    }
+
+    bool StreamSecureSocketDriver::SecureConnection::ProcessHandshake()
+    {
+        if (ProcessNetwork())
+        {
+            return true;
+        }
+
+        const SecureConnectionContextImpl* ctx = CastContext(m_context);
+
+        // is the handshake done?
+        if (SSL_is_init_finished(ctx->m_ssl) == 1)
+        {
+            m_sm.Transition(static_cast<AZ::HSM::StateId>(ConnectionState::ESTABLISHED));
+            return true;
+        }
+        else
+        {
+            // update the SSL internals
+            AZ::s32 hsRet = SSL_do_handshake(ctx->m_ssl);
+            if (hsRet < 0)
+            {
+                AZ::s32 exRet = SSL_get_error(ctx->m_ssl, hsRet);
+                if (exRet != SSL_ERROR_WANT_READ && exRet != SSL_ERROR_WANT_WRITE)
+                {
+                    // The TLS/SSL handshake was not successful because a fatal error occurred either at the protocol level or a connection failure occurred.
+                    StoreLastSocketError();
+                    m_sm.Transition(static_cast<AZ::HSM::StateId>(ConnectionState::IN_ERROR));
+                    return true;
+                }
+            }
+        }
+
+        // no state change, yet
+        return false;
+    }
+}
+
+#endif // AZ_TRAIT_GRIDMATE_ENABLE_OPENSSL

Code/Framework/GridMate/GridMate/Carrier/StreamSecureSocketDriver.h

@@ -0,0 +1,101 @@
+/*
+* All or portions of this file Copyright (c) Amazon.com, Inc. or its affiliates or
+* its licensors.
+*
+* For complete copyright and license terms please see the LICENSE at the root of this
+* distribution (the "License"). All use of this software is governed by the License,
+* or, if provided, by the license below or the license accompanying this file. Do not
+* remove or modify any license notices. This file is distributed on an "AS IS" BASIS,
+* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+*
+*/
+#pragma once
+
+#include <AzCore/PlatformDef.h>
+#include <GridMate_Traits_Platform.h>
+
+#if AZ_TRAIT_GRIDMATE_ENABLE_OPENSSL
+
+#include <GridMate/Carrier/StreamSocketDriver.h>
+#include <AzCore/std/smart_ptr/unique_ptr.h>
+#include <AzCore/std/functional.h>
+
+namespace GridMate
+{
+    /**
+     * Handles TLS for TCP stream socket transportation
+     */
+    class StreamSecureSocketDriver
+        : public StreamSocketDriver
+    {
+    public:
+        struct StreamSecureSocketDriverDesc
+        {
+            StreamSecureSocketDriverDesc()
+                : m_privateKeyPEM(nullptr)
+                , m_certificatePEM(nullptr)
+                , m_certificateAuthorityPEM(nullptr)
+                , m_authenticateClient(false)
+            {
+            };
+
+            const char* m_privateKeyPEM;               // A base-64 encoded PEM format private key.
+            const char* m_certificatePEM;              // A base-64 encoded PEM format certificate.
+            const char* m_certificateAuthorityPEM;     // A base-64 encoded PEM format CA root certificate.
+            bool        m_authenticateClient;          // Ensure that a client must be authenticated (the server is always authenticated). Only required to be set on the server!
+        };
+
+        GM_CLASS_ALLOCATOR(StreamSecureSocketDriver);
+
+        StreamSecureSocketDriver(AZ::u32 maxConnections = 32, AZ::u32 maxPacketSize = 1024 * 64, AZ::u32 inboundBufferSize = 1024 * 64, AZ::u32 outboundBufferSize = 1024 * 64);
+        ~StreamSecureSocketDriver() override;
+
+        // Security operation(s)
+        ResultCode InitializeSecurity(AZ::s32 familyType, const char* address, AZ::u32 port, AZ::u32 receiveBufferSize, AZ::u32 sendBufferSize, StreamSecureSocketDriverDesc& desc);
+
+    public:
+        struct SecureContextHandle
+        {
+            virtual ~SecureContextHandle() {}
+        };
+        struct SecureConnectionContext
+        {
+            virtual ~SecureConnectionContext() {}
+            virtual bool PrepareToAccept() = 0;
+            virtual bool PrepareToConnect() = 0;
+        };
+
+    protected:
+        class SecureConnection
+            : public Connection
+        {
+        public:
+            GM_CLASS_ALLOCATOR(SecureConnection);
+
+            SecureConnection(AZ::u32 inboundBufferSize, AZ::u32 outputBufferSize, SecureContextHandle& handle);
+            ~SecureConnection() override;
+
+            bool SendPacket(const char* data, AZ::u32 dataSize) override;
+
+        protected:
+            bool OnStateAccept(AZ::HSM& sm, const AZ::HSM::Event& e) override;
+            bool OnStateConnect(AZ::HSM& sm, const AZ::HSM::Event& e) override;
+            bool OnStateEstablished(AZ::HSM& sm, const AZ::HSM::Event& e) override;
+
+            bool ProcessNetwork();
+            bool ProcessHandshake();
+
+        private:
+            Driver::ErrorCodes ReadBytes(void* buf, AZ::u32& num, bool& resume);
+            bool WriteIntoRingBufferSafe(RingBuffer& inboundBuffer);
+
+            AZStd::unique_ptr<SecureConnectionContext> m_context;
+            RingBuffer m_inboundRawBuffer;
+        };
+        
+        // for secure connections
+        AZStd::unique_ptr<SecureContextHandle> m_handle;
+    };
+}
+
+#endif // AZ_TRAIT_GRIDMATE_ENABLE_OPENSSL

Code/Framework/GridMate/GridMate/Carrier/StreamSocketDriver.h

@@ -0,0 +1,283 @@
+/*
+* All or portions of this file Copyright (c) Amazon.com, Inc. or its affiliates or
+* its licensors.
+*
+* For complete copyright and license terms please see the LICENSE at the root of this
+* distribution (the "License"). All use of this software is governed by the License,
+* or, if provided, by the license below or the license accompanying this file. Do not
+* remove or modify any license notices. This file is distributed on an "AS IS" BASIS,
+* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+*
+*/
+#pragma once
+#ifndef GM_STREAM_SOCKET_DRIVER_H
+#define GM_STREAM_SOCKET_DRIVER_H
+
+#include <GridMate/Carrier/Driver.h>
+#include <GridMate/Carrier/SocketDriver.h>
+#include <GridMate/Serialize/Buffer.h>
+#include <GridMate/EBus.h>
+
+#include <AzCore/State/HSM.h>
+#include <AzCore/std/containers/unordered_map.h>
+#include <AzCore/std/smart_ptr/unique_ptr.h>
+#include <AzCore/std/functional.h>
+
+namespace GridMate
+{
+    class StreamSocketDriverEventsInterface
+        : public GridMateEBusTraits
+    {
+    public:
+        virtual void OnConnectionEstablished(const SocketDriverAddress& address) = 0;
+        virtual void OnConnectionDisconnected(const SocketDriverAddress& address) = 0;
+    };
+    typedef AZ::EBus<StreamSocketDriverEventsInterface> StreamSocketDriverEventsBus;
+
+    /**
+     * Handles TCP socket streaming protocol
+     */
+    class StreamSocketDriver
+        : public SocketDriver
+    {
+    public:
+        GM_CLASS_ALLOCATOR(StreamSocketDriver);
+
+        using SocketDriverAddressPtr = AZStd::intrusive_ptr<SocketDriverAddress>;
+
+        StreamSocketDriver(AZ::u32 maxConnections = 32, AZ::u32 maxPacketSize = 1024 * 64, AZ::u32 inboundBufferSize = 1024 * 64, AZ::u32 outboundBufferSize = 1024 * 64);
+        virtual ~StreamSocketDriver();
+
+        // Driver
+        void Update() override;
+
+        // SocketDriver
+        AZ::u32 GetMaxNumConnections() const override;
+        AZ::u32 GetMaxSendSize() const override;
+        AZ::u32 GetPacketOverheadSize() const override;
+        ResultCode Initialize(AZ::s32 familyType = BSD_AF_INET, const char* address = nullptr, AZ::u32 port = 0, bool isBroadcast = false, AZ::u32 receiveBufferSize = 0, AZ::u32 sendBufferSize = 0) override;
+        AZ::u32 GetPort() const override;
+        ResultCode Send(const AZStd::intrusive_ptr<DriverAddress>& to, const char* data, AZ::u32 dataSize) override;
+        AZ::u32 Receive(char* data, AZ::u32 maxDataSize, AZStd::intrusive_ptr<DriverAddress>& from, ResultCode* resultCode = nullptr) override;
+
+        // Stream operations
+        ResultCode ConnectTo(const SocketDriverAddressPtr& addr);
+        ResultCode DisconnectFrom(const SocketDriverAddressPtr& addr);
+        ResultCode StartListen(AZ::s32 backlog);
+        ResultCode StopListen();
+        AZ::u32 GetNumberOfConnections() const;
+        bool IsConnectedTo(const SocketDriverAddressPtr& to) const;
+        bool IsListening() const { return m_isListening; }
+
+    protected:
+        ResultCode SetSocketOptions(bool isBroadcast, AZ::u32 receiveBufferSize, AZ::u32 sendBufferSize) override;
+        void CloseSocket();
+        ResultCode PrepareSocket(AZ::u16 desiredPort, SocketAddressInfo& socketAddressInfo, SocketType& socket);
+
+        // state machine
+        enum class ConnectionState
+        {
+            TOP,
+            ACCEPT,          // accepts incoming connections
+            CONNECTING,      // attempts to start connection
+            CONNECT,         // polls an established connection
+            ESTABLISHED,     // stream connection has been made
+            DISCONNECTED,    // normal disconnect
+            IN_ERROR,        // error
+            MAX
+        };
+
+        struct Packet
+        {
+            Packet()
+                : m_size(0)
+                , m_data(nullptr)
+            {
+            }
+            AZ::u16 m_size;
+            char* m_data;
+        };
+
+        class RingBuffer
+        {
+        public:
+            RingBuffer(AZ::u32 capacity);
+            ~RingBuffer();
+            RingBuffer(const RingBuffer& other) = delete;
+            RingBuffer& operator=(const RingBuffer& other) = delete;
+
+            AZ_FORCE_INLINE AZ::u32 GetCapacity() const { return m_capacity; }
+            AZ_FORCE_INLINE AZ::u32 GetSpaceToWrite() const { return m_capacity - m_bytesStored; }
+            AZ_FORCE_INLINE AZ::u32 GetSpaceToRead() const { return m_bytesStored; }
+
+            char* ReserveForWrite(AZ::u32& inOutSize);
+            bool CommitAsWrote(AZ::u32 dataSize, bool& bWroteToMarker);
+            void CommitAsRead(AZ::u32 size);
+
+            bool Store(const char* data, AZ::u32 dataSize);
+
+            template <typename T>
+            bool Store(const T data);
+
+            bool Peek(char* data, AZ::u32 dataSize);
+
+            template <typename T>
+            bool Peek(T* data);
+
+            bool Fetch(char* data, AZ::u32 dataSize);
+
+            template <typename T>
+            bool Fetch(T* data);
+
+            void Release();
+
+        protected:
+            AZ_FORCE_INLINE void Prepare()
+            {
+                if (!m_data)
+                {
+                    m_data = new char[m_capacity];
+                }
+            }
+
+            AZ_FORCE_INLINE void InternalWrite(const char* data, AZ::u32 dataSize)
+            {
+                memcpy(m_data + m_indexEnd, data, dataSize);
+                m_indexEnd += dataSize;
+                if (m_indexEnd == m_capacity)
+                {
+                    m_indexEnd = 0;
+                }
+            }
+
+            AZ_FORCE_INLINE void InternalRead(char* data, AZ::u32 dataSize)
+            {
+                memcpy(data, m_data + m_indexStart, dataSize);
+                m_indexStart += dataSize;
+                if (m_indexStart == m_capacity)
+                {
+                    m_indexStart = 0;
+                }
+            }
+
+            AZ_FORCE_INLINE AZ::u32 GetSpaceUntilMarker() const
+            {
+                if (m_data == nullptr)
+                {
+                    return 0;
+                }
+                AZ_Assert(m_bytesStored <= m_capacity, "m_bytesUsed exceeds m_capacity");
+                if (m_bytesStored == m_capacity)
+                {
+                    return 0;
+                }
+                if (m_indexEnd >= m_indexStart)
+                {
+                    return m_capacity - m_indexEnd;
+                }
+                return m_indexStart - m_indexEnd;
+            }
+
+        private:
+            char* m_data;
+            AZ::u32 m_capacity;
+            AZ::u32 m_bytesStored;
+            AZ::u32 m_indexStart;
+            AZ::u32 m_indexEnd;
+        };
+
+        class Connection
+        {
+        public:
+            GM_CLASS_ALLOCATOR(Connection);
+            
+            Connection(AZ::u32 inboundBufferSize, AZ::u32 outputBufferSize);
+            virtual ~Connection();
+
+            virtual bool Initialize(ConnectionState startState, SocketType s, SocketDriverAddressPtr remoteAddress);
+            virtual void Shutdown();
+            virtual void Update();
+            virtual void Close();
+            ConnectionState GetConnectionState() const;
+
+            virtual bool SendPacket(const char* data, AZ::u32 dataSize);
+            virtual bool GetPacket(Packet& packet, char* data, AZ::u32 maxDataSize);
+
+        protected:
+            virtual bool OnStateTop(AZ::HSM& sm, const AZ::HSM::Event& e);
+            virtual bool OnStateAccept(AZ::HSM& sm, const AZ::HSM::Event& e);
+            virtual bool OnStateConnecting(AZ::HSM& sm, const AZ::HSM::Event& e);
+            virtual bool OnStateConnect(AZ::HSM& sm, const AZ::HSM::Event& e);
+            virtual bool OnStateEstablished(AZ::HSM& sm, const AZ::HSM::Event& e);
+            virtual bool OnStateDisconnected(AZ::HSM& sm, const AZ::HSM::Event& e);
+            virtual bool OnStateError(AZ::HSM& sm, const AZ::HSM::Event& e);
+
+            void StoreLastSocketError();
+
+            void ProcessInbound(bool& switchState, RingBuffer& inboundBuffer);
+            void ProcessOutbound(bool& switchState, RingBuffer& outboundBuffer);
+
+            AZ_FORCE_INLINE bool IsValidPacketDataSize(AZ::u32 dataSize, const AZ::u32 maxDataSize) const
+            {
+                if (dataSize == 0)
+                {
+                    // is an empty buffer?
+                    AZ_Assert(false, "inOutDataSize should be a non-zero value");
+                    return false;
+                }
+                else if ((0x80000000 & dataSize) != 0)
+                {
+                    // is negative?
+                    AZ_Assert(false, "dataSize should be a positive value");
+                    return false;
+                }
+                else if (dataSize > maxDataSize)
+                {
+                    // is negative?
+                    AZ_Assert(false, "dataSize can not exceed the max send byte size of %d", maxDataSize);
+                    return false;
+                }
+
+                return true;
+            }
+
+            enum ConnectionEvents
+            {
+                CE_UPDATE = 1,
+                CE_CLOSE  = 2,
+            };
+
+            SocketDriverAddressPtr m_remoteAddress;
+            bool m_initialized;
+            SocketType m_socket;
+            AZStd::vector<AZ::s64> m_socketErrors;
+            AZ::HSM m_sm;
+            RingBuffer m_inboundBuffer;
+            RingBuffer m_outboundBuffer;
+        };
+
+        struct SocketPtrHasher
+        {
+            AZStd::size_t operator()(const SocketDriverAddressPtr& v) const;
+        };
+
+        using ConnectionMap = AZStd::unordered_map<SocketDriverAddressPtr, Connection*, SocketPtrHasher>;
+        using ConnectionFactory = AZStd::function<Connection*(AZ::u32 inboundBufferSize, AZ::u32 outputBufferSize)>;
+
+        AZ::u32 m_maxConnections;                   // max connections for the driver
+        AZ::u32 m_incomingBufferSize;               // the size of the inbound ring buffer per connection
+        AZ::u32 m_outgoingBufferSize;               // the size of the outbound ring buffer per connection
+        AZ::u32 m_maxPacketSize;                    // the max packet size expected to be sent through the driver
+        AZ::u32 m_maxSendSize;                      // used to set up the TCP socket option SO_SNDBUF
+        AZ::u32 m_maxReceiveSize;                   // used to set up the TCP socket option SO_RCVBUF
+        bool m_isListening;                         // listening for new connections
+        AZStd::string m_boundAddress;               // the bound address name for the family
+        BSDSocketFamilyType m_boundSocketFamily;    // socket family type to make sockets with: either IPv4 or IPv6
+
+        // connection storage for both accepted sockets and direct connect sockets
+        ConnectionFactory m_connectionFactory;
+        ConnectionMap m_connections;
+    };
+}
+
+#endif // GM_STREAM_SOCKET_DRIVER_H

Code/Framework/GridMate/GridMate/Carrier/TrafficControl.h

@@ -0,0 +1,216 @@
+/*
+* All or portions of this file Copyright (c) Amazon.com, Inc. or its affiliates or
+* its licensors.
+*
+* For complete copyright and license terms please see the LICENSE at the root of this
+* distribution (the "License"). All use of this software is governed by the License,
+* or, if provided, by the license below or the license accompanying this file. Do not
+* remove or modify any license notices. This file is distributed on an "AS IS" BASIS,
+* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+*
+*/
+#ifndef GM_TRAFFIC_CONTROL_H
+#define GM_TRAFFIC_CONTROL_H
+
+#include <GridMate/Types.h>
+#include <GridMate/String/string.h>
+
+#include <AzCore/std/smart_ptr/intrusive_ptr.h>
+
+namespace GridMate
+{
+    class DriverAddress;
+
+    // u16 sequence counters (max SequenceNumberHalfSpan-1 packets in flight)
+    typedef AZ::u16 SequenceNumber;
+    static const SequenceNumber SequenceNumberMax = 0xffff;
+
+    // u32 sequence counters
+    //    typedef unsigned int SequenceNumber;
+    //    static const SequenceNumber SequenceNumberMax = 0xffffffff;
+
+    static const SequenceNumber SequenceNumberHalfSpan = SequenceNumberMax / 2;
+
+    /**
+     * Traffic control interface implements the traffic flow to all connections.
+     * It should handle issues like congestion, etc.
+     * \note All the code is executed in a thread context! Any interaction with
+     * the outside code should be made thread safe.
+     */
+    class TrafficControl
+    {
+    public:
+        struct DataGramControlData
+        {
+            SequenceNumber          m_sequenceNumber;
+            TimeStamp               m_time;
+            unsigned short          m_size;             ///< Datagram size in bytes.
+            unsigned short          m_effectiveSize;    ///< Datagram effective byte size (no headers just user data)
+        };
+        struct TrafficControlConnection
+        {
+            TrafficControlConnection()
+                : m_trafficData(NULL)
+            {}
+            void*   m_trafficData;      ///< Specialized traffic control implementations can store user data here.
+        };
+        /// Carrier thread connection identifier.
+        typedef TrafficControlConnection* TrafficControlConnectionId;
+
+        virtual ~TrafficControl()  {}
+
+        /// Called when Carrier has established a new connection.
+        virtual void OnConnect(TrafficControlConnectionId id, const AZStd::intrusive_ptr<DriverAddress>& address) = 0;
+        /// Called when Carrier has lost a connection.
+        virtual void OnDisconnect(TrafficControlConnectionId id) = 0;
+        /// Called when Carrier completed successful handshake. Usually NAT punch happens during the handshake, which can result is high packet loss.
+        virtual void OnHandshakeComplete(TrafficControlConnectionId id) = 0;
+        /// Called when Carrier has send a package.
+        virtual void OnSend(TrafficControlConnectionId id, DataGramControlData& info) = 0;
+        /// Called when Carrier has send an ACK/NACK data with the packet.
+        virtual void OnSendAck(TrafficControlConnectionId id) = 0;
+        /// Called when Carrier has resend a package.
+        virtual void OnReSend(TrafficControlConnectionId id, DataGramControlData& info, unsigned int resendDataSize) = 0;
+        /// Called when Carrier confirmed a package delivery.
+        virtual void OnAck(TrafficControlConnectionId id, DataGramControlData& info, bool& windowChanged) = 0;
+        /// Called when we receive a NAck for a package delivery.
+        virtual void OnNAck(TrafficControlConnectionId id, DataGramControlData& info) = 0;
+        /// Called when Carrier receives a package.
+        virtual void OnReceived(TrafficControlConnectionId id, DataGramControlData& info) = 0;
+        /// Return true if we can send a package. Otherwise false.
+        virtual bool IsSend(TrafficControlConnectionId id) = 0;
+        /// Return true if you should send ACK/NACK data at this time.
+        virtual bool IsSendAck(TrafficControlConnectionId id) = 0;
+        /// Return number of bytes we are allowed to send at the moment. The size can/will vary over time.
+        virtual unsigned int GetAvailableWindowSize(TrafficControlConnectionId id) const = 0;
+        /**
+        * Called for every package waiting for Ack. If this function returns true the packet will be considered lost.
+        * You should resend it and call OnReSend function ASAP.
+        */
+        virtual bool IsResend(TrafficControlConnectionId id, const DataGramControlData& info, unsigned int resendDataSize) = 0;
+
+        /**
+        * Returns the timestamp for retransmission
+        */
+        virtual TimeStamp GetResendTime(TrafficControlConnectionId id, const DataGramControlData& info) = 0;
+
+        /// Verify traffic conditions and disconnect if needed. This usually happens when we have bad conditions. Too much latency or high packet loss.
+        virtual bool IsDisconnect(TrafficControlConnectionId id, float conditionThreshold) = 0;
+
+        /// Verify we are able to receive data from a given address
+        virtual bool IsCanReceiveData(TrafficControlConnectionId id) const = 0;
+
+        /**
+         * Returns true if you need to send a ACK only (empty datagram) due to time and/or number of received datagrams.
+         * If you already have data to send ACK will be included in the datagram anyway. This function should be checked only
+         * if you have no data to send.
+         */
+        virtual bool IsSendACKOnly(TrafficControlConnectionId id) const = 0;
+
+        /// Update/Tick returns true if we have updated the statistics (which we can read by \ref QueryStatistics)
+        virtual bool Update() = 0;
+
+        /**
+         *
+         */
+        struct Statistics
+        {
+            unsigned int    m_dataSend;         ///< Data send in bytes.
+            unsigned int    m_dataReceived;     ///< Data received in bytes.
+
+            unsigned int    m_dataResend;           ///< Data resend in bytes.
+            unsigned int    m_dataAcked;            ///< Data received in bytes.
+
+            unsigned int    m_packetSend;           ///< Number of packets/datagrams send.
+            unsigned int    m_packetReceived;       ///< Number of packets/datagrams received.
+
+            unsigned int    m_packetLost;           ///< Number of packets/datagrams lost.
+            unsigned int    m_packetAcked;      ///< Number of packets/datagrams acked/confirmed received.
+
+            float           m_rtt;              ///< Round trip time in milliseconds.
+            float           m_packetLoss;           ///< Packet loss percentage (smooth average) [0.0,1.0]
+
+            float           m_connectionFactor; ///< [0.0,1.0] 0 is good connection when 1.0 is reached a bad connection will be reported (unless disconnect detection is off)
+
+            //float         m_flow;             ///< A value reporting the state of the flow control. 1.0 if unrestricted flow (full capacity), 0.0 is max restricted (send at minimal rate - otherwise disconnect will occur).
+        };
+
+        /**
+         * Stores connection statistics, it's ok to pass NULL for any of the statistics
+         * \param id Connection ID
+         * \param lastSecond last second statistics for all data
+         * \param lifetime lifetime statistics for all data
+         * \param effectiveLastSecond last second statistics for effective data (actual data - carrier overhead excluded)
+         * \param effectiveLifetime lifetime statistics for effective data (actual data - carrier overhead excluded)
+         */
+        virtual void        QueryStatistics(TrafficControlConnectionId id, Statistics* lastSecond = nullptr, Statistics* lifetime = nullptr, Statistics* effectiveLastSecond = nullptr, Statistics* effectiveLifetime = nullptr) const = 0;
+
+        struct CongestionState
+        {
+            unsigned int    m_dataInTransfer;   ///< Data in progess (out of the toSend queue)
+            unsigned int    m_congestionWindow; ///< If the traffic controller uses congestionWindow it's size will be set >0
+        };
+
+        /**
+         * Stores current congestion state into the provided block.
+         */
+        virtual void        QueryCongestionState(TrafficControlConnectionId id, CongestionState* congestionState) const = 0;
+    };
+
+
+    //////////////////////////////////////////////////////////////////////////
+    // Utility functions
+    inline bool SequenceNumberIsSequential(SequenceNumber a, SequenceNumber b)
+    {
+        return ((b > a) && (b - a <= SequenceNumberHalfSpan)) || ((b < a) && (a - b > SequenceNumberHalfSpan));
+    }
+    inline SequenceNumber SequenceNumberSequentialDistance(SequenceNumber a, SequenceNumber b)
+    {
+        SequenceNumber dist;
+        if (b > a)
+        {
+            dist = b - a;
+            if (dist <= SequenceNumberHalfSpan)
+            {
+                return dist;
+            }
+        }
+        else if (b < a)
+        {
+            dist = (SequenceNumberMax - a) + b + 1;
+            if (dist <= SequenceNumberHalfSpan)
+            {
+                return dist;
+            }
+        }
+        return 0; // invalid distance if a != b, otherwise correct
+    }
+
+    // Checks if a > b
+    inline bool SequenceNumberGreaterThan(SequenceNumber a, SequenceNumber b)
+    {
+        SequenceNumber off = b - a;
+        return (b != a) && (off > SequenceNumberHalfSpan);
+    }
+
+    // Check if a >= b
+    inline bool SequenceNumberGreaterEqualThan(SequenceNumber a, SequenceNumber b)
+    {
+        if (a == b)
+        {
+            return true;
+        }
+        return SequenceNumberGreaterThan(a, b);
+    }
+
+    // Checks if a < b
+    inline bool SequenceNumberLessThan(SequenceNumber a, SequenceNumber b)
+    {
+        SequenceNumber off = b - a;
+        return (b != a) && (off < SequenceNumberHalfSpan);
+    }
+    //////////////////////////////////////////////////////////////////////////
+}
+
+#endif // GM_TRAFFIC_CONTROL_H
+

Code/Framework/GridMate/GridMate/Carrier/Utils.h

@@ -0,0 +1,28 @@
+/*
+* All or portions of this file Copyright (c) Amazon.com, Inc. or its affiliates or
+* its licensors.
+*
+* For complete copyright and license terms please see the LICENSE at the root of this
+* distribution (the "License"). All use of this software is governed by the License,
+* or, if provided, by the license below or the license accompanying this file. Do not
+* remove or modify any license notices. This file is distributed on an "AS IS" BASIS,
+* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+*
+*/
+#ifndef GM_CARRIER_UTILS_H
+#define GM_CARRIER_UTILS_H
+
+#include <GridMate/String/string.h>
+
+namespace GridMate
+{
+    namespace Utils
+    {
+        ///< Returns the machines address(ip) in a string. familyType is platform dependent.
+        string GetMachineAddress(int familyType = 0);
+        ///< Returns a broadcast address based on a family type. On ipv6 we emulate broadbast using the multicast on the all nodes address (FF02::1). familyType is platform dependent.
+        const char* GetBroadcastAddress(int familyType = 0);
+    }
+}
+
+#endif // GM_DEFAULT_TRAFFIC_CONTROL_H

Code/Framework/GridMate/GridMate/Containers/list.h

@@ -0,0 +1,24 @@
+/*
+* All or portions of this file Copyright (c) Amazon.com, Inc. or its affiliates or
+* its licensors.
+*
+* For complete copyright and license terms please see the LICENSE at the root of this
+* distribution (the "License"). All use of this software is governed by the License,
+* or, if provided, by the license below or the license accompanying this file. Do not
+* remove or modify any license notices. This file is distributed on an "AS IS" BASIS,
+* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+*
+*/
+#ifndef GM_CONTAINERS_LIST_H
+#define GM_CONTAINERS_LIST_H
+
+#include <GridMate/Memory.h>
+#include <AzCore/std/containers/list.h>
+
+namespace GridMate
+{
+    template<class T, class Allocator = SysContAlloc>
+    using list = AZStd::list<T, Allocator>;
+}
+
+#endif // GM_CONTAINERS_LIST_H

Code/Framework/GridMate/GridMate/Containers/queue.h

@@ -0,0 +1,24 @@
+/*
+* All or portions of this file Copyright (c) Amazon.com, Inc. or its affiliates or
+* its licensors.
+*
+* For complete copyright and license terms please see the LICENSE at the root of this
+* distribution (the "License"). All use of this software is governed by the License,
+* or, if provided, by the license below or the license accompanying this file. Do not
+* remove or modify any license notices. This file is distributed on an "AS IS" BASIS,
+* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+*
+*/
+#ifndef GM_CONTAINERS_QUEUE_H
+#define GM_CONTAINERS_QUEUE_H
+
+#include <GridMate/Memory.h>
+#include <AzCore/std/containers/queue.h>
+
+namespace GridMate
+{
+    template<class T, class Container = AZStd::deque<T, SysContAlloc> >
+    using queue = AZStd::queue<T, Container>;
+}
+
+#endif // GM_CONTAINERS_QUEUE_H

Code/Framework/GridMate/GridMate/Containers/set.h

@@ -0,0 +1,27 @@
+/*
+* All or portions of this file Copyright (c) Amazon.com, Inc. or its affiliates or
+* its licensors.
+*
+* For complete copyright and license terms please see the LICENSE at the root of this
+* distribution (the "License"). All use of this software is governed by the License,
+* or, if provided, by the license below or the license accompanying this file. Do not
+* remove or modify any license notices. This file is distributed on an "AS IS" BASIS,
+* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+*
+*/
+#ifndef GM_CONTAINERS_SET_H
+#define GM_CONTAINERS_SET_H
+
+#include <GridMate/Memory.h>
+#include <AzCore/std/containers/set.h>
+
+namespace GridMate
+{
+    template<class Key, class Compare = AZStd::less<Key>, class Allocator = SysContAlloc>
+    using set = AZStd::set<Key, Compare, Allocator>;
+
+    template<class Key, class Compare = AZStd::less<Key>, class Allocator = SysContAlloc>
+    using multiset = AZStd::multiset<Key, Compare, Allocator>;
+}
+
+#endif // GM_CONTAINERS_SET_H

Code/Framework/GridMate/GridMate/Containers/slist.h

@@ -0,0 +1,24 @@
+/*
+* All or portions of this file Copyright (c) Amazon.com, Inc. or its affiliates or
+* its licensors.
+*
+* For complete copyright and license terms please see the LICENSE at the root of this
+* distribution (the "License"). All use of this software is governed by the License,
+* or, if provided, by the license below or the license accompanying this file. Do not
+* remove or modify any license notices. This file is distributed on an "AS IS" BASIS,
+* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+*
+*/
+#ifndef GM_CONTAINERS_SLIST_H
+#define GM_CONTAINERS_SLIST_H
+
+#include <GridMate/Memory.h>
+#include <AzCore/std/containers/forward_list.h>
+
+namespace GridMate
+{
+    template<class T, class Allocator = SysContAlloc>
+    using forward_list = AZStd::forward_list<T, Allocator>;
+}
+
+#endif // GM_CONTAINERS_SLIST_H

Code/Framework/GridMate/GridMate/Containers/unordered_map.h

@@ -0,0 +1,26 @@
+/*
+* All or portions of this file Copyright (c) Amazon.com, Inc. or its affiliates or
+* its licensors.
+*
+* For complete copyright and license terms please see the LICENSE at the root of this
+* distribution (the "License"). All use of this software is governed by the License,
+* or, if provided, by the license below or the license accompanying this file. Do not
+* remove or modify any license notices. This file is distributed on an "AS IS" BASIS,
+* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+*
+*/
+#ifndef GM_CONTAINERS_UNORDERED_MAP_H
+#define GM_CONTAINERS_UNORDERED_MAP_H
+
+#include <GridMate/Memory.h>
+#include <AzCore/std/containers/unordered_map.h>
+
+namespace GridMate
+{
+    template<class Key, class MappedType, class Hasher = AZStd::hash<Key>, class EqualKey = AZStd::equal_to<Key>, class Allocator = SysContAlloc>
+    using unordered_map = AZStd::unordered_map<Key, MappedType, Hasher, EqualKey, Allocator>;
+
+    template<class Key, class MappedType, class Hasher = AZStd::hash<Key>, class EqualKey = AZStd::equal_to<Key>, class Allocator = SysContAlloc>
+    using unordered_multimap = AZStd::unordered_multimap<Key, MappedType, Hasher, EqualKey, Allocator>;
+}
+#endif // GM_CONTAINERS_UNORDERED_MAP_H

Code/Framework/GridMate/GridMate/Containers/unordered_set.h

@@ -0,0 +1,27 @@
+/*
+* All or portions of this file Copyright (c) Amazon.com, Inc. or its affiliates or
+* its licensors.
+*
+* For complete copyright and license terms please see the LICENSE at the root of this
+* distribution (the "License"). All use of this software is governed by the License,
+* or, if provided, by the license below or the license accompanying this file. Do not
+* remove or modify any license notices. This file is distributed on an "AS IS" BASIS,
+* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+*
+*/
+#ifndef GM_CONTAINERS_UNORDERED_SET_H
+#define GM_CONTAINERS_UNORDERED_SET_H
+
+#include <GridMate/Memory.h>
+#include <AzCore/std/containers/unordered_set.h>
+
+namespace GridMate
+{
+    template<class Key, class Hasher = AZStd::hash<Key>, class EqualKey = AZStd::equal_to<Key>, class Allocator = SysContAlloc>
+    using unordered_set = AZStd::unordered_set<Key, Hasher, EqualKey, Allocator>;
+
+    template<class Key, class Hasher = AZStd::hash<Key>, class EqualKey = AZStd::equal_to<Key>, class Allocator = SysContAlloc>
+    using unordered_multiset = AZStd::unordered_multiset<Key, Hasher, EqualKey, Allocator>;
+}
+
+#endif // GM_CONTAINERS_UNORDERED_SET_H

Code/Framework/GridMate/GridMate/Containers/vector.h

@@ -0,0 +1,23 @@
+/*
+* All or portions of this file Copyright (c) Amazon.com, Inc. or its affiliates or
+* its licensors.
+*
+* For complete copyright and license terms please see the LICENSE at the root of this
+* distribution (the "License"). All use of this software is governed by the License,
+* or, if provided, by the license below or the license accompanying this file. Do not
+* remove or modify any license notices. This file is distributed on an "AS IS" BASIS,
+* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+*
+*/
+#ifndef GM_CONTAINERS_VECTOR_H
+#define GM_CONTAINERS_VECTOR_H
+
+#include <GridMate/Memory.h>
+#include <AzCore/std/containers/vector.h>
+
+namespace GridMate
+{
+    template<class T, class Allocator = SysContAlloc>
+    using vector = AZStd::vector<T, Allocator>;
+}
+#endif // GM_CONTAINERS_VECTOR_H

Code/Framework/GridMate/GridMate/Docs.h

@@ -0,0 +1,35 @@
+/*
+* All or portions of this file Copyright (c) Amazon.com, Inc. or its affiliates or
+* its licensors.
+*
+* For complete copyright and license terms please see the LICENSE at the root of this
+* distribution (the "License"). All use of this software is governed by the License,
+* or, if provided, by the license below or the license accompanying this file. Do not
+* remove or modify any license notices. This file is distributed on an "AS IS" BASIS,
+* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+*
+*/
+/**
+ * \mainpage
+ * Welcome to the GridMate network library.
+ *
+ * Check the latest \ref ReleaseNotes "release notes" for this version of GridMate.
+ *
+ * You can start learning by looking at the \ref Library "Library overview".
+ *
+ * Or if you can't wait, jump to the \ref GMExamples "code examples" to see how GridMate is used.
+ */
+
+/**
+ * \page Library Library Overview
+ *
+ * \subpage Fundamentals "Fundamental Concepts"
+ *
+ * \ref GMExamples "Code examples"
+ *
+ */
+
+/**
+ * \namespace GridMate
+ * \brief The main namespace for the GridMate library.
+ */

Code/Framework/GridMate/GridMate/Drillers/CarrierDriller.cpp

@@ -0,0 +1,228 @@
+/*
+* All or portions of this file Copyright (c) Amazon.com, Inc. or its affiliates or
+* its licensors.
+*
+* For complete copyright and license terms please see the LICENSE at the root of this
+* distribution (the "License"). All use of this software is governed by the License,
+* or, if provided, by the license below or the license accompanying this file. Do not
+* remove or modify any license notices. This file is distributed on an "AS IS" BASIS,
+* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+*
+*/
+
+#include <GridMate/Drillers/CarrierDriller.h>
+#include <AzCore/Math/Crc.h>
+
+using namespace AZ::Debug;
+
+namespace GridMate
+{
+    namespace Debug
+    {
+        //=========================================================================
+        // CarrierDriller
+        // [4/14/2011]
+        //=========================================================================
+        CarrierDriller::CarrierDriller()
+        {
+            m_drillerTag = AZ_CRC("CarrierDriller", 0x72a37d06);
+        }
+
+        //=========================================================================
+        // Start
+        // [4/14/2011]
+        //=========================================================================
+        void CarrierDriller::Start(const Param* params, int numParams)
+        {
+            (void)params;
+            (void)numParams;
+            CarrierDrillerBus::Handler::BusConnect();
+
+            /* get carriers and output all the data
+            m_output->BeginTag(m_drillerTag);
+                m_output->Write(AZ_CRC("CarrierId"),carrier);
+                m_output->BeginTag(AZ_CRC("StartDrill"));
+                for(unsigned int iConn = 0; iConn < carrier->GetNumConnections(); ++iConn )
+                {
+                    ConnectionID connId = carrier->GetConnectionId(iConn);
+                    m_output->BeginTag(AZ_CRC("Connection"));
+                        m_output->Write(AZ_CRC("Id"),connId);
+                        m_output->Write(AZ_CRC("Address"),carrier->ConnectionToAddress(connId));
+                        m_output->Write(AZ_CRC("State"),static_cast<int>(carrier->GetConnectionState(connId)));
+                    m_output->EndTag(AZ_CRC("Connection"));
+                }
+                m_output->EndTag(AZ_CRC("StartDrill"));
+            m_output->EndTag(m_drillerTag);*/
+        }
+
+        //=========================================================================
+        // Stop
+        // [4/14/2011]
+        //=========================================================================
+        void CarrierDriller::Stop()
+        {
+            CarrierDrillerBus::Handler::BusDisconnect();
+        }
+
+        //=========================================================================
+        // OnUpdateStatistics
+        // [4/14/2011]
+        //=========================================================================
+        void CarrierDriller::OnUpdateStatistics(const GridMate::string& address, const TrafficControl::Statistics& lastSecond, const TrafficControl::Statistics& lifeTime, const TrafficControl::Statistics& effectiveLastSecond, const TrafficControl::Statistics& effectiveLifeTime)
+        {
+            m_output->BeginTag(m_drillerTag);
+            m_output->BeginTag(AZ_CRC("Statistics", 0xe2d38b22));
+            m_output->Write(AZ_CRC("Address", 0x0d4e6f81), address);
+            m_output->BeginTag(AZ_CRC("LastSecond", 0x5e6ccbee));
+            m_output->Write(AZ_CRC("DataSend", 0xae94c282), lastSecond.m_dataSend);
+            m_output->Write(AZ_CRC("DataReceived", 0xd92f8e4b), lastSecond.m_dataReceived);
+            m_output->Write(AZ_CRC("DataResend", 0xe44a3086), lastSecond.m_dataResend);
+            m_output->Write(AZ_CRC("DataAcked", 0xbb5e5496), lastSecond.m_dataAcked);
+            m_output->Write(AZ_CRC("PacketSend", 0x5b52fa79), lastSecond.m_packetSend);
+            m_output->Write(AZ_CRC("PacketReceived", 0xf247dd9e), lastSecond.m_packetReceived);
+            m_output->Write(AZ_CRC("PacketLost", 0xbc64441e), lastSecond.m_packetLost);
+            m_output->Write(AZ_CRC("PacketAcked", 0x91c4b93a), lastSecond.m_packetAcked);
+            m_output->Write(AZ_CRC("PacketLoss", 0x2200d1bd), lastSecond.m_packetLoss);
+            m_output->Write(AZ_CRC("rtt", 0xb40f6cfb), lastSecond.m_rtt);
+            //m_output->Write(AZ_CRC("flow"),lastSecond.m_flow);
+            m_output->EndTag(AZ_CRC("LastSecond", 0x5e6ccbee));
+            m_output->BeginTag(AZ_CRC("LifeTime", 0x3de73088));
+            m_output->Write(AZ_CRC("DataSend", 0xae94c282), lifeTime.m_dataSend);
+            m_output->Write(AZ_CRC("DataReceived", 0xd92f8e4b), lifeTime.m_dataReceived);
+            m_output->Write(AZ_CRC("DataResend", 0xe44a3086), lifeTime.m_dataResend);
+            m_output->Write(AZ_CRC("DataAcked", 0xbb5e5496), lifeTime.m_dataAcked);
+            m_output->Write(AZ_CRC("PacketSend", 0x5b52fa79), lifeTime.m_packetSend);
+            m_output->Write(AZ_CRC("PacketReceived", 0xf247dd9e), lifeTime.m_packetReceived);
+            m_output->Write(AZ_CRC("PacketLost", 0xbc64441e), lifeTime.m_packetLost);
+            m_output->Write(AZ_CRC("PacketAcked", 0x91c4b93a), lifeTime.m_packetAcked);
+            m_output->Write(AZ_CRC("PacketLoss", 0x2200d1bd), lifeTime.m_packetLoss);
+            m_output->Write(AZ_CRC("rtt", 0xb40f6cfb), lifeTime.m_rtt);
+            //m_output->Write(AZ_CRC("flow"),lifeTime.m_flow);
+            m_output->EndTag(AZ_CRC("LifeTime", 0x3de73088));
+            m_output->BeginTag(AZ_CRC("EffectiveLastSecond", 0x8f84642f));
+            m_output->Write(AZ_CRC("DataSend", 0xae94c282), effectiveLastSecond.m_dataSend);
+            m_output->Write(AZ_CRC("DataReceived", 0xd92f8e4b), effectiveLastSecond.m_dataReceived);
+            m_output->Write(AZ_CRC("DataResend", 0xe44a3086), effectiveLastSecond.m_dataResend);
+            m_output->Write(AZ_CRC("DataAcked", 0xbb5e5496), effectiveLastSecond.m_dataAcked);
+            m_output->Write(AZ_CRC("PacketSend", 0x5b52fa79), effectiveLastSecond.m_packetSend);
+            m_output->Write(AZ_CRC("PacketReceived", 0xf247dd9e), effectiveLastSecond.m_packetReceived);
+            m_output->Write(AZ_CRC("PacketLost", 0xbc64441e), effectiveLastSecond.m_packetLost);
+            m_output->Write(AZ_CRC("PacketAcked", 0x91c4b93a), effectiveLastSecond.m_packetAcked);
+            m_output->Write(AZ_CRC("PacketLoss", 0x2200d1bd), effectiveLastSecond.m_packetLoss);
+            m_output->Write(AZ_CRC("rtt", 0xb40f6cfb), effectiveLastSecond.m_rtt);
+            //m_output->Write(AZ_CRC("flow"),effectiveLastSecond.m_flow);
+            m_output->EndTag(AZ_CRC("EffectiveLastSecond", 0x8f84642f));
+            m_output->BeginTag(AZ_CRC("EffectiveLifeTime", 0x4644a47a));
+            m_output->Write(AZ_CRC("DataSend", 0xae94c282), effectiveLifeTime.m_dataSend);
+            m_output->Write(AZ_CRC("DataReceived", 0xd92f8e4b), effectiveLifeTime.m_dataReceived);
+            m_output->Write(AZ_CRC("DataResend", 0xe44a3086), effectiveLifeTime.m_dataResend);
+            m_output->Write(AZ_CRC("DataAcked", 0xbb5e5496), effectiveLifeTime.m_dataAcked);
+            m_output->Write(AZ_CRC("PacketSend", 0x5b52fa79), effectiveLifeTime.m_packetSend);
+            m_output->Write(AZ_CRC("PacketReceived", 0xf247dd9e), effectiveLifeTime.m_packetReceived);
+            m_output->Write(AZ_CRC("PacketLost", 0xbc64441e), effectiveLifeTime.m_packetLost);
+            m_output->Write(AZ_CRC("PacketAcked", 0x91c4b93a), effectiveLifeTime.m_packetAcked);
+            m_output->Write(AZ_CRC("PacketLoss", 0x2200d1bd), effectiveLifeTime.m_packetLoss);
+            m_output->Write(AZ_CRC("rtt", 0xb40f6cfb), effectiveLifeTime.m_rtt);
+            //m_output->Write(AZ_CRC("flow"),effectiveLifeTime.m_flow);
+            m_output->EndTag(AZ_CRC("EffectiveLifeTime", 0x4644a47a));
+            m_output->EndTag(AZ_CRC("Statistics", 0xe2d38b22));
+            m_output->EndTag(m_drillerTag);
+        }
+        //=========================================================================
+        // OnConnectionStateChanged
+        // [4/14/2011]
+        //=========================================================================
+        void CarrierDriller::OnConnectionStateChanged(Carrier* carrier, ConnectionID id, Carrier::ConnectionStates newState)
+        {
+            m_output->BeginTag(m_drillerTag);
+            m_output->Write(AZ_CRC("CarrierId", 0x93f4bfbe), carrier);
+            m_output->BeginTag(AZ_CRC("ConnectionState", 0x38a6a5da));
+            m_output->Write(AZ_CRC("Id", 0xbf396750), id);
+            m_output->Write(AZ_CRC("State", 0xa393d2fb), static_cast<int>(newState));
+            m_output->EndTag(AZ_CRC("ConnectionState", 0x38a6a5da));
+            m_output->EndTag(m_drillerTag);
+        }
+        //=========================================================================
+        // OnIncomingConnection
+        // [4/14/2011]
+        //=========================================================================
+        void CarrierDriller::OnIncomingConnection(Carrier* carrier, ConnectionID id)
+        {
+            m_output->BeginTag(m_drillerTag);
+            m_output->Write(AZ_CRC("CarrierId", 0x93f4bfbe), carrier);
+            m_output->BeginTag(AZ_CRC("IncomingConnection", 0x8c9d071a));
+            m_output->Write(AZ_CRC("Id", 0xbf396750), id);
+            m_output->Write(AZ_CRC("Address", 0x0d4e6f81), carrier->ConnectionToAddress(id));
+            m_output->EndTag(AZ_CRC("IncomingConnection", 0x8c9d071a));
+            m_output->EndTag(m_drillerTag);
+        }
+        //=========================================================================
+        // OnFailedToConnect
+        // [4/14/2011]
+        //=========================================================================
+        void CarrierDriller::OnFailedToConnect(Carrier* carrier, ConnectionID id, CarrierDisconnectReason reason)
+        {
+            m_output->BeginTag(m_drillerTag);
+            m_output->Write(AZ_CRC("CarrierId", 0x93f4bfbe), carrier);
+            m_output->BeginTag(AZ_CRC("FailedToConnect", 0xb6539549));
+            m_output->Write(AZ_CRC("Id", 0xbf396750), id);
+            m_output->Write(AZ_CRC("Reason", 0x3bb8880c), ReasonToString(reason));
+            m_output->EndTag(AZ_CRC("FailedToConnect", 0xb6539549));
+            m_output->EndTag(m_drillerTag);
+        }
+        //=========================================================================
+        // OnConnectionEstablished
+        // [4/14/2011]
+        //=========================================================================
+        void CarrierDriller::OnConnectionEstablished(Carrier* carrier, ConnectionID id)
+        {
+            m_output->BeginTag(m_drillerTag);
+            m_output->Write(AZ_CRC("CarrierId", 0x93f4bfbe), carrier);
+            m_output->BeginTag(AZ_CRC("ConnectionEstablished", 0xcde31aa7));
+            m_output->Write(AZ_CRC("Id", 0xbf396750), id);
+            m_output->EndTag(AZ_CRC("ConnectionEstablished", 0xcde31aa7));
+            m_output->EndTag(m_drillerTag);
+        }
+        //=========================================================================
+        // OnDisconnect
+        // [4/14/2011]
+        //=========================================================================
+        void CarrierDriller::OnDisconnect(Carrier* carrier, ConnectionID id, CarrierDisconnectReason reason)
+        {
+            m_output->BeginTag(m_drillerTag);
+            m_output->Write(AZ_CRC("CarrierId", 0x93f4bfbe), carrier);
+            m_output->BeginTag(AZ_CRC("Disconnect", 0x003a4b91));
+            m_output->Write(AZ_CRC("Id", 0xbf396750), id);
+            m_output->Write(AZ_CRC("Reason", 0x3bb8880c), ReasonToString(reason));
+            m_output->EndTag(AZ_CRC("Disconnect", 0x003a4b91));
+            m_output->EndTag(m_drillerTag);
+        }
+        //=========================================================================
+        // OnDriverError
+        // [12/14/2016]
+        //=========================================================================
+        void CarrierDriller::OnDriverError(Carrier* carrier, ConnectionID id, const DriverError& error)
+        {
+            m_output->BeginTag(m_drillerTag);
+            m_output->Write(AZ_CRC("CarrierId", 0x93f4bfbe), carrier);
+            m_output->BeginTag(AZ_CRC("DriverError", 0xe7522aff));
+            m_output->Write(AZ_CRC("Id", 0xbf396750), id);
+            m_output->Write(AZ_CRC("ErrorCode", 0x499e660e), static_cast<int>(error.m_errorCode));
+            m_output->EndTag(AZ_CRC("DriverError", 0xe7522aff));
+            m_output->EndTag(m_drillerTag);
+        }
+        //=========================================================================
+        // OnSecurityError
+        //=========================================================================
+        void CarrierDriller::OnSecurityError(Carrier* carrier, ConnectionID id, const SecurityError& error)
+        {
+            m_output->BeginTag(m_drillerTag);
+            m_output->Write(AZ_CRC("CarrierId", 0x93f4bfbe), carrier);
+            m_output->BeginTag(AZ_CRC("SecurityError", 0xdfe940ab));
+            m_output->Write(AZ_CRC("Id", 0xbf396750), id);
+            m_output->Write(AZ_CRC("ErrorCode", 0x499e660e), static_cast<int>(error.m_errorCode));
+            m_output->EndTag(AZ_CRC("SecurityError", 0xdfe940ab));
+            m_output->EndTag(m_drillerTag);
+        }
+    } // namespace Debug
+} // namespace GridMate

Code/Framework/GridMate/GridMate/Drillers/CarrierDriller.h

@@ -0,0 +1,66 @@
+/*
+* All or portions of this file Copyright (c) Amazon.com, Inc. or its affiliates or
+* its licensors.
+*
+* For complete copyright and license terms please see the LICENSE at the root of this
+* distribution (the "License"). All use of this software is governed by the License,
+* or, if provided, by the license below or the license accompanying this file. Do not
+* remove or modify any license notices. This file is distributed on an "AS IS" BASIS,
+* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+*
+*/
+#ifndef GM_CARRIER_DRILLER_H
+#define GM_CARRIER_DRILLER_H
+
+#include <GridMate/Types.h>
+#include <GridMate/Carrier/Carrier.h>
+#include <AzCore/Driller/Driller.h>
+
+namespace GridMate
+{
+    namespace Debug
+    {
+        /**
+         * Carrier driller
+         * \note Be careful which buses you attach. The drillers work in Multi threaded environment and expect that
+         * a driller mutex (DrillerManager::DrillerManager) will be automatically locked on every write.
+         * Otherwise in output stream corruption will happen (even is the stream is thread safe).
+         */
+        class CarrierDriller
+            : public AZ::Debug::Driller
+            , public CarrierDrillerBus::Handler
+        {
+            int m_drillerTag;
+        public:
+            AZ_CLASS_ALLOCATOR(CarrierDriller, AZ::OSAllocator, 0);
+            CarrierDriller();
+
+            //////////////////////////////////////////////////////////////////////////
+            // Driller
+            const char*  GroupName() const override { return "GridMate"; }
+            const char*  GetName() const override { return "CarrierDriller"; }
+            const char*  GetDescription() const override { return "Drills Carrier/transport layer,traffic control, driver,etc."; }
+            void         Start(const Param* params = nullptr, int numParams = 0) override;
+            void         Stop() override;
+            //////////////////////////////////////////////////////////////////////////
+
+            //////////////////////////////////////////////////////////////////////////
+            // Carrier Driller Bus
+            void OnUpdateStatistics(const GridMate::string& address, const TrafficControl::Statistics& lastSecond, const TrafficControl::Statistics& lifeTime, const TrafficControl::Statistics& effectiveLastSecond, const TrafficControl::Statistics& effectiveLifeTime) override;
+            void OnConnectionStateChanged(Carrier* carrier, ConnectionID id, Carrier::ConnectionStates newState) override;
+            //////////////////////////////////////////////////////////////////////////
+
+            //////////////////////////////////////////////////////////////////////////
+            // Carrier Event Bus
+            void OnIncomingConnection(Carrier* carrier, ConnectionID id) override;
+            void OnFailedToConnect(Carrier* carrier, ConnectionID id, CarrierDisconnectReason reason) override;
+            void OnConnectionEstablished(Carrier* carrier, ConnectionID id) override;
+            void OnDisconnect(Carrier* carrier, ConnectionID id, CarrierDisconnectReason reason) override;
+            void OnDriverError(Carrier* carrier, ConnectionID id, const DriverError& error) override;
+            void OnSecurityError(Carrier* carrier, ConnectionID id, const SecurityError& error) override;
+            //////////////////////////////////////////////////////////////////////////
+        };
+    }
+}
+
+#endif // GM_CARRIER_DRILLER_H

Code/Framework/GridMate/GridMate/Drillers/ReplicaDriller.cpp

@@ -0,0 +1,145 @@
+/*
+* All or portions of this file Copyright (c) Amazon.com, Inc. or its affiliates or
+* its licensors.
+*
+* For complete copyright and license terms please see the LICENSE at the root of this
+* distribution (the "License"). All use of this software is governed by the License,
+* or, if provided, by the license below or the license accompanying this file. Do not
+* remove or modify any license notices. This file is distributed on an "AS IS" BASIS,
+* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+*
+*/
+#include <GridMate/Drillers/ReplicaDriller.h>
+#include <GridMate/Replica/ReplicaChunk.h>
+#include <GridMate/Replica/ReplicaChunkDescriptor.h>
+#include <GridMate/Replica/RemoteProcedureCall.h>
+#include <GridMate/String/string.h>
+
+using namespace AZ::Debug;
+
+namespace GridMate
+{
+    namespace Debug
+    {
+        const AZ::Crc32 ReplicaDriller::Tags::REPLICA_DRILLER = AZ_CRC("ReplicaDriller", 0xd832f49a);
+
+        // Event Types
+        const AZ::Crc32 ReplicaDriller::Tags::CHUNK_SEND_DATASET = AZ_CRC("ChunkSendDataSet", 0x085ea99b);
+        const AZ::Crc32 ReplicaDriller::Tags::CHUNK_RECEIVE_DATASET = AZ_CRC("ChunkReceiveDataSet", 0x8d4536db);
+        const AZ::Crc32 ReplicaDriller::Tags::CHUNK_SEND_RPC = AZ_CRC("ChunkSendRPC", 0x7c40afe0);
+        const AZ::Crc32 ReplicaDriller::Tags::CHUNK_RECEIVE_RPC = AZ_CRC("ChunkReceiveRPC", 0xb49b302d);
+
+        // Data Fields
+        const AZ::Crc32 ReplicaDriller::Tags::REPLICA_NAME = AZ_CRC("ReplicaName", 0xc69b68ee);
+        const AZ::Crc32 ReplicaDriller::Tags::REPLICA_ID = AZ_CRC("ReplicaID", 0x394dd741);
+        const AZ::Crc32 ReplicaDriller::Tags::CHUNK_TYPE = AZ_CRC("TypeName", 0x115f811d);
+        const AZ::Crc32 ReplicaDriller::Tags::CHUNK_INDEX = AZ_CRC("ChunkIndex", 0x25ba3370);
+        const AZ::Crc32 ReplicaDriller::Tags::DATA_SET_NAME = AZ_CRC("DataSetName", 0xf22dbaae);
+        const AZ::Crc32 ReplicaDriller::Tags::DATA_SET_INDEX = AZ_CRC("DataSetIndex", 0x58d2421f);
+        const AZ::Crc32 ReplicaDriller::Tags::RPC_NAME = AZ_CRC("RPCName", 0x4c4cbf3a);
+        const AZ::Crc32 ReplicaDriller::Tags::RPC_INDEX = AZ_CRC("RPCIndex", 0xaf0e7447);
+        const AZ::Crc32 ReplicaDriller::Tags::SIZE = AZ_CRC("Size", 0xf7c0246a);
+        const AZ::Crc32 ReplicaDriller::Tags::TIME_PROCESSED_MILLISEC = AZ_CRC("Time", 0x6f949845);
+
+        ReplicaDriller::ReplicaDriller()
+        {
+        }
+
+        void ReplicaDriller::Start(const Param* params, int numParams)
+        {
+            (void)params;
+            (void)numParams;
+            ReplicaDrillerBus::Handler::BusConnect();
+        }
+
+        void ReplicaDriller::Stop()
+        {
+            ReplicaDrillerBus::Handler::BusDisconnect();
+        }
+
+        void ReplicaDriller::OnSendDataSet(ReplicaChunkBase* chunk, AZ::u32 chunkIndex, DataSetBase* dataSet, PeerId from, PeerId to, const void* data, size_t len)
+        {
+            (void)from;
+            (void)to;
+            (void)data;
+
+            const char* dataSetName = chunk->GetDescriptor()->GetDataSetName(chunk, dataSet);
+            size_t dataSetIndex = chunk->GetDescriptor()->GetDataSetIndex(chunk, dataSet);
+
+            m_output->BeginTag(Tags::REPLICA_DRILLER);
+            m_output->BeginTag(Tags::CHUNK_SEND_DATASET);
+            OutputBaseReplicaChunkTags(chunk, chunkIndex, len);
+            m_output->Write(Tags::DATA_SET_NAME, dataSetName);
+            m_output->Write(Tags::DATA_SET_INDEX, dataSetIndex);
+            m_output->EndTag(Tags::CHUNK_SEND_DATASET);
+            m_output->EndTag(Tags::REPLICA_DRILLER);
+        }
+
+        void ReplicaDriller::OnReceiveDataSet(ReplicaChunkBase* chunk, AZ::u32 chunkIndex, DataSetBase* dataSet, PeerId from, PeerId to, const void* data, size_t len)
+        {
+            (void)from;
+            (void)to;
+            (void)data;
+
+            const char* dataSetName = chunk->GetDescriptor()->GetDataSetName(chunk, dataSet);
+            size_t dataSetIndex = chunk->GetDescriptor()->GetDataSetIndex(chunk, dataSet);
+
+            m_output->BeginTag(Tags::REPLICA_DRILLER);
+            m_output->BeginTag(Tags::CHUNK_RECEIVE_DATASET);
+            OutputBaseReplicaChunkTags(chunk, chunkIndex, len);
+            m_output->Write(Tags::DATA_SET_NAME, dataSetName);
+            m_output->Write(Tags::DATA_SET_INDEX, dataSetIndex);
+            m_output->EndTag(Tags::CHUNK_RECEIVE_DATASET);
+            m_output->EndTag(Tags::REPLICA_DRILLER);
+        }
+
+        void ReplicaDriller::OnSendRpc(ReplicaChunkBase* chunk, AZ::u32 chunkIndex, Internal::RpcRequest* rpc, PeerId from, PeerId to, const void* data, size_t len)
+        {
+            (void)from;
+            (void)to;
+            (void)data;
+
+            const char* rpcName = chunk->GetDescriptor()->GetRpcName(chunk, rpc->m_rpc);
+            size_t rpcIndex = chunk->GetDescriptor()->GetRpcIndex(chunk, rpc->m_rpc);
+
+
+            m_output->BeginTag(Tags::REPLICA_DRILLER);
+            m_output->BeginTag(Tags::CHUNK_SEND_RPC);
+            OutputBaseReplicaChunkTags(chunk, chunkIndex, len);
+            m_output->Write(Tags::RPC_NAME, rpcName);
+            m_output->Write(Tags::RPC_INDEX, rpcIndex);
+            m_output->EndTag(Tags::CHUNK_SEND_RPC);
+            m_output->EndTag(Tags::REPLICA_DRILLER);
+        }
+        void ReplicaDriller::OnReceiveRpc(ReplicaChunkBase* chunk, AZ::u32 chunkIndex, Internal::RpcRequest* rpc, PeerId from, PeerId to, const void* data, size_t len)
+        {
+            (void)from;
+            (void)to;
+            (void)data;
+
+            const char* rpcName = chunk->GetDescriptor()->GetRpcName(chunk, rpc->m_rpc);
+            size_t rpcIndex = chunk->GetDescriptor()->GetRpcIndex(chunk, rpc->m_rpc);
+
+            m_output->BeginTag(Tags::REPLICA_DRILLER);
+            m_output->BeginTag(Tags::CHUNK_RECEIVE_RPC);
+            OutputBaseReplicaChunkTags(chunk, chunkIndex, len);
+            m_output->Write(Tags::RPC_NAME, rpcName);
+            m_output->Write(Tags::RPC_INDEX, rpcIndex);
+            m_output->EndTag(Tags::CHUNK_RECEIVE_RPC);
+            m_output->EndTag(Tags::REPLICA_DRILLER);
+        }
+
+        void ReplicaDriller::OutputBaseReplicaChunkTags(ReplicaChunkBase* chunk, AZ::u32 chunkIndex, size_t len)
+        {
+            const char* chunkTypeName = chunk->GetDescriptor()->GetChunkName();            
+            const char* replicaName = chunk->GetReplica()->GetDebugName();            
+
+            m_output->Write(Tags::REPLICA_NAME, replicaName);
+            m_output->Write(Tags::REPLICA_ID, chunk->GetReplicaId());
+            m_output->Write(Tags::CHUNK_TYPE, chunkTypeName);
+            m_output->Write(Tags::CHUNK_INDEX, chunkIndex);
+            m_output->Write(Tags::SIZE, len);
+            m_output->Write(Tags::TIME_PROCESSED_MILLISEC, AZStd::chrono::milliseconds(AZStd::chrono::system_clock::now().time_since_epoch()).count());
+        }
+    }
+}

Code/Framework/GridMate/GridMate/Drillers/ReplicaDriller.h

@@ -0,0 +1,81 @@
+/*
+* All or portions of this file Copyright (c) Amazon.com, Inc. or its affiliates or
+* its licensors.
+*
+* For complete copyright and license terms please see the LICENSE at the root of this
+* distribution (the "License"). All use of this software is governed by the License,
+* or, if provided, by the license below or the license accompanying this file. Do not
+* remove or modify any license notices. This file is distributed on an "AS IS" BASIS,
+* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+*
+*/
+#ifndef GM_REPLICA_DRILLER_H
+#define GM_REPLICA_DRILLER_H
+
+#include <GridMate/Types.h>
+#include <AzCore/Driller/Driller.h>
+#include <GridMate/Replica/Replica.h>
+#include <GridMate/Replica/ReplicaDrillerEvents.h>
+
+namespace GridMate
+{
+    namespace Debug
+    {
+        class ReplicaDriller
+            : public AZ::Debug::Driller
+            , public GridMate::Debug::ReplicaDrillerBus::Handler
+        {
+        public:
+            struct Tags
+            {
+                // Driller
+                static const AZ::Crc32 REPLICA_DRILLER;
+
+                // Event Types
+                static const AZ::Crc32 CHUNK_SEND_DATASET;
+                static const AZ::Crc32 CHUNK_RECEIVE_DATASET;
+                static const AZ::Crc32 CHUNK_SEND_RPC;
+                static const AZ::Crc32 CHUNK_RECEIVE_RPC;
+
+                // Data Fields
+                static const AZ::Crc32 REPLICA_NAME;
+                static const AZ::Crc32 REPLICA_ID;
+                static const AZ::Crc32 CHUNK_TYPE;
+                static const AZ::Crc32 CHUNK_INDEX;
+                static const AZ::Crc32 DATA_SET_NAME;
+                static const AZ::Crc32 DATA_SET_INDEX;
+                static const AZ::Crc32 RPC_NAME;
+                static const AZ::Crc32 RPC_INDEX;
+                static const AZ::Crc32 SIZE;
+                static const AZ::Crc32 TIME_PROCESSED_MILLISEC;
+            };
+
+            AZ_CLASS_ALLOCATOR(ReplicaDriller, AZ::OSAllocator, 0);
+            ReplicaDriller();
+
+            //////////////////////////////////////////////////////////////////////////
+            // Driller
+            const char*  GroupName() const override      { return "GridMate"; }
+            const char*  GetName() const override        { return "ReplicaDriller"; }
+            const char*  GetDescription() const override { return "Drills replicas."; }
+            void         Start(const Param* params = NULL, int numParams = 0) override;
+            void         Stop() override;
+            //////////////////////////////////////////////////////////////////////////
+
+            //////////////////////////////////////////////////////////////////////////
+            // ReplicaDrillerEvents
+            void OnSendDataSet(ReplicaChunkBase* chunk, AZ::u32 chunkIndex, DataSetBase* dataSet, PeerId from, PeerId to, const void* data, size_t len) override;
+            void OnReceiveDataSet(ReplicaChunkBase* chunk, AZ::u32 chunkIndex, DataSetBase* dataSet, PeerId from, PeerId to, const void* data, size_t len) override;
+
+            void OnSendRpc(ReplicaChunkBase* chunk, AZ::u32 chunkIndex, Internal::RpcRequest* rpc, PeerId from, PeerId to, const void* data, size_t len) override;
+            void OnReceiveRpc(ReplicaChunkBase* chunk, AZ::u32 chunkIndex, Internal::RpcRequest* rpc, PeerId from, PeerId to, const void* data, size_t len) override;
+            //////////////////////////////////////////////////////////////////////////
+
+        private:
+
+            void OutputBaseReplicaChunkTags(ReplicaChunkBase* chunk, AZ::u32 chunkIndex, size_t len);
+        };
+    }
+}
+
+#endif

Code/Framework/GridMate/GridMate/Drillers/SessionDriller.cpp

@@ -0,0 +1,279 @@
+/*
+* All or portions of this file Copyright (c) Amazon.com, Inc. or its affiliates or
+* its licensors.
+*
+* For complete copyright and license terms please see the LICENSE at the root of this
+* distribution (the "License"). All use of this software is governed by the License,
+* or, if provided, by the license below or the license accompanying this file. Do not
+* remove or modify any license notices. This file is distributed on an "AS IS" BASIS,
+* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+*
+*/
+
+#include <GridMate/Drillers/SessionDriller.h>
+
+using namespace AZ::Debug;
+
+namespace GridMate
+{
+    namespace Debug
+    {
+        //=========================================================================
+        // SessionDriller
+        // [4/14/2011]
+        //=========================================================================
+        SessionDriller::SessionDriller()
+        {
+            m_drillerTag = AZ_CRC("SessionDriller", 0x30b916a9);
+        }
+
+        //=========================================================================
+        // Start
+        // [4/14/2011]
+        //=========================================================================
+        void SessionDriller::Start(const Param* params, int numParams)
+        {
+            (void)params;
+            (void)numParams;
+            // Collect current session information ?
+            SessionDrillerBus::Handler::BusConnect();
+
+            //
+            //m_output->BeginTag(m_drillerTag);
+            //m_output->BeginTag(AZ_CRC("StartDrill"));
+            //if(sessionMgr->m_activeSession)
+            //{
+            //  GridSession* gs = sessionMgr->m_activeSession;
+            //  // store the current session state
+            //  m_output->BeginTag(AZ_CRC("Session"));
+            //  m_output->Write(AZ_CRC("SessionId"),gs->GetId());
+            //  m_output->Write(AZ_CRC("Carrier"),gs->GetCarrier());
+            //  m_output->Write(AZ_CRC("ReplicaMgr"),gs->GetReplicaMgr());
+            //  m_output->Write(AZ_CRC("Topology"),(char)gs->GetTopology());
+            //  m_output->Write(AZ_CRC("Time"),gs->GetTime());
+            //  m_output->Write(AZ_CRC("State"),(char)gs->m_sm.GetCurrentState());
+            //  m_output->Write(AZ_CRC("IsHost"),gs->IsHost());
+            //  // There are endless params add as needed
+
+            //  for(unsigned int i = 0; i < gs->GetNumberOfMembers(); ++i )
+            //  {
+            //      GridMember* gm = gs->GetMember(i);
+            //      m_output->BeginTag(AZ_CRC("Member"));
+            //      m_output->Write(AZ_CRC("Id"),gm->GetId().ToString());
+            //      m_output->Write(AZ_CRC("Name"),gm->GetName());
+            //      m_output->Write(AZ_CRC("ConnectionId"),gm->GetConnectionId());
+            //      m_output->Write(AZ_CRC("NAT"),(char)gm->GetNatType());
+            //      m_output->Write(AZ_CRC("CommFilter"),gm->GetCommFilter());
+            //      m_output->Write(AZ_CRC("IsHost"),gm->IsHost());
+            //      m_output->Write(AZ_CRC("IsLocal"),gm->IsLocal());
+            //      m_output->Write(AZ_CRC("IsInvited"),gm->IsInvited());
+            //      m_output->EndTag(AZ_CRC("Member"));
+            //  }
+
+            //  m_output->EndTag(AZ_CRC("Session"));
+            //}
+            //else if(sessionMgr->m_activeSearch)
+            //{
+            //  GridSearch* gs = sessionMgr->m_activeSearch;
+            //  m_output->BeginTag(AZ_CRC("GridSearch"));
+            //  m_output->Write(AZ_CRC("SearchId"),gs);
+            //  m_output->Write(AZ_CRC("IsDone"),gs->IsDone());
+            //  m_output->Write(AZ_CRC("NumResults"),gs->GetNumResults());
+            //  // add platform specific drill or just generic reporting
+            //  m_output->EndTag(AZ_CRC("GridSearch"));
+            //}
+            //m_output->EndTag(AZ_CRC("StartDrill"));
+            //m_output->EndTag(m_drillerTag);
+        }
+
+        //=========================================================================
+        // Stop
+        // [4/14/2011]
+        //=========================================================================
+        void SessionDriller::Stop()
+        {
+            SessionDrillerBus::Handler::BusDisconnect();
+        }
+
+        //=========================================================================
+        // OnSessionServiceReady
+        // [4/15/2011]
+        //=========================================================================
+        void
+        SessionDriller::OnSessionServiceReady()
+        {
+            //  m_output->BeginTag(m_drillerTag);
+            //  m_output->EndTag(m_drillerTag);
+        }
+
+        //=========================================================================
+        // OnGridSearchComplete
+        // [4/15/2011]
+        //=========================================================================
+        void
+        SessionDriller::OnGridSearchComplete(GridSearch* gridSearch)
+        {
+            m_output->BeginTag(m_drillerTag);
+            m_output->BeginTag(AZ_CRC("GridSearchComplete", 0x974b5717));
+            m_output->Write(AZ_CRC("SearchId", 0x4f7ef2d2), gridSearch);
+            m_output->Write(AZ_CRC("NumResults", 0xdfb1542f), gridSearch->GetNumResults());
+            // add platform specific drill or just generic reporting
+            m_output->EndTag(AZ_CRC("GridSearchComplete", 0x974b5717));
+            m_output->EndTag(m_drillerTag);
+        }
+
+        //=========================================================================
+        // OnMemberJoined
+        // [4/15/2011]
+        //=========================================================================
+        void
+        SessionDriller::OnMemberJoined(GridSession* session, GridMember* member)
+        {
+            m_output->BeginTag(m_drillerTag);
+            m_output->BeginTag(AZ_CRC("MemberJoined", 0xbde4706c));
+            m_output->Write(AZ_CRC("SessionId", 0xacd49154), session->GetId());
+            m_output->Write(AZ_CRC("Id", 0xbf396750), member->GetId().ToString());
+            m_output->Write(AZ_CRC("Name", 0x5e237e06), member->GetName());
+            m_output->Write(AZ_CRC("ConnectionId", 0x4592a200), member->GetConnectionId());
+            m_output->Write(AZ_CRC("NAT", 0x9686d0fb), (char)member->GetNatType());
+            //m_output->Write(AZ_CRC("MuteList"),member->GetMuteList());
+            m_output->Write(AZ_CRC("IsHost", 0xce28a9cf), member->IsHost());
+            m_output->Write(AZ_CRC("IsLocal", 0x4300d6d2), member->IsLocal());
+            m_output->Write(AZ_CRC("IsInvited", 0x29d785f7), member->IsInvited());
+            m_output->EndTag(AZ_CRC("MemberJoined", 0xbde4706c));
+            m_output->EndTag(m_drillerTag);
+        }
+
+        //=========================================================================
+        // OnMemberLeaving
+        // [4/15/2011]
+        //=========================================================================
+        void
+        SessionDriller::OnMemberLeaving(GridSession* session, GridMember* member)
+        {
+            m_output->BeginTag(m_drillerTag);
+            m_output->BeginTag(AZ_CRC("MemberLeaving", 0xd10ee176));
+            m_output->Write(AZ_CRC("SessionId", 0xacd49154), session->GetId());
+            m_output->Write(AZ_CRC("Id", 0xbf396750), member->GetId().ToString());
+            m_output->EndTag(AZ_CRC("MemberLeaving", 0xd10ee176));
+            m_output->EndTag(m_drillerTag);
+        }
+
+        //=========================================================================
+        // OnMemberKicked
+        // [4/15/2011]
+        //=========================================================================
+        void
+        SessionDriller::OnMemberKicked(GridSession* session, GridMember* member)
+        {
+            m_output->BeginTag(m_drillerTag);
+            m_output->BeginTag(AZ_CRC("MemberKicked", 0x908e74e6));
+            m_output->Write(AZ_CRC("SessionId", 0xacd49154), session->GetId());
+            m_output->Write(AZ_CRC("Id", 0xbf396750), member->GetId().ToString());
+            m_output->EndTag(AZ_CRC("MemberKicked", 0x908e74e6));
+            m_output->EndTag(m_drillerTag);
+        }
+
+        //=========================================================================
+        // OnSessionCreated
+        // [4/15/2011]
+        //=========================================================================
+        void
+        SessionDriller::OnSessionCreated(GridSession* session)
+        {
+            m_output->BeginTag(m_drillerTag);
+            m_output->BeginTag(AZ_CRC("SessionCreated", 0x24655a62));
+            m_output->Write(AZ_CRC("SessionId", 0xacd49154), session->GetId());
+            m_output->Write(AZ_CRC("Carrier", 0x4739f11c), session->GetCarrier());
+            m_output->Write(AZ_CRC("ReplicaMgr", 0x41cf3853), session->GetReplicaMgr());
+            m_output->Write(AZ_CRC("Topology", 0x1198610c), (char)session->GetTopology());
+            m_output->Write(AZ_CRC("Time", 0x6f949845), session->GetTime());
+            m_output->Write(AZ_CRC("State", 0xa393d2fb), (char)session->m_sm.GetCurrentState());
+            m_output->Write(AZ_CRC("IsHost", 0xce28a9cf), session->IsHost());
+            m_output->EndTag(AZ_CRC("SessionCreated", 0x24655a62));
+            m_output->EndTag(m_drillerTag);
+        }
+        //=========================================================================
+        // OnSessionJoined
+        // [4/15/2011]
+        //=========================================================================
+        void
+        SessionDriller::OnSessionJoined(GridSession* session)
+        {
+            m_output->BeginTag(m_drillerTag);
+            m_output->BeginTag(AZ_CRC("SessionJoined", 0x04b85d49));
+            m_output->Write(AZ_CRC("SessionId", 0xacd49154), session->GetId());
+            m_output->Write(AZ_CRC("Carrier", 0x4739f11c), session->GetCarrier());
+            m_output->Write(AZ_CRC("ReplicaMgr", 0x41cf3853), session->GetReplicaMgr());
+            m_output->Write(AZ_CRC("Topology", 0x1198610c), (char)session->GetTopology());
+            m_output->Write(AZ_CRC("Time", 0x6f949845), session->GetTime());
+            m_output->Write(AZ_CRC("State", 0xa393d2fb), (char)session->m_sm.GetCurrentState());
+            m_output->Write(AZ_CRC("IsHost", 0xce28a9cf), session->IsHost());
+            m_output->EndTag(AZ_CRC("SessionJoined", 0x04b85d49));
+            m_output->EndTag(m_drillerTag);
+        }
+        //=========================================================================
+        // OnSessionDelete
+        // [4/15/2011]
+        //=========================================================================
+        void
+        SessionDriller::OnSessionDelete(GridSession* session)
+        {
+            m_output->BeginTag(m_drillerTag);
+            m_output->Write(AZ_CRC("SessionDelete", 0x6b5728cd), session->GetId());
+            m_output->EndTag(m_drillerTag);
+        }
+        //=========================================================================
+        // OnSessionError
+        // [4/15/2011]
+        //=========================================================================
+        void
+        SessionDriller::OnSessionError(GridSession* session, const string& errorMsg)
+        {
+            m_output->BeginTag(m_drillerTag);
+            m_output->BeginTag(AZ_CRC("SessionError", 0xc689cc40));
+            m_output->Write(AZ_CRC("SessionId", 0xacd49154), session ? session->GetId() : "NoId");
+            m_output->Write(AZ_CRC("Error", 0x5dddbc71), errorMsg);
+            m_output->EndTag(AZ_CRC("SessionError", 0xc689cc40));
+            m_output->EndTag(m_drillerTag);
+        }
+        //=========================================================================
+        // OnSessionStart
+        // [4/15/2011]
+        //=========================================================================
+        void
+        SessionDriller::OnSessionStart(GridSession* session)
+        {
+            m_output->BeginTag(m_drillerTag);
+            m_output->Write(AZ_CRC("SessionStart", 0x042d25be), session->GetId());
+            m_output->EndTag(m_drillerTag);
+        }
+        //=========================================================================
+        // OnSessionEnd
+        // [4/15/2011]
+        //=========================================================================
+        void
+        SessionDriller::OnSessionEnd(GridSession* session)
+        {
+            m_output->BeginTag(m_drillerTag);
+            m_output->Write(AZ_CRC("SessionEnd", 0x07821a5e), session->GetId());
+            m_output->EndTag(m_drillerTag);
+        }
+        //=========================================================================
+        // OnWriteStatistics
+        // [6/8/2011]
+        //=========================================================================
+        void
+        SessionDriller::OnWriteStatistics(GridSession* session, GridMember* member, StatisticsData& data)
+        {
+            m_output->BeginTag(m_drillerTag);
+            m_output->BeginTag(AZ_CRC("WriteStatistics", 0xcf7f12aa));
+            m_output->Write(AZ_CRC("SessionId", 0xacd49154), session->GetId());
+            m_output->Write(AZ_CRC("Id", 0xbf396750), member->GetId().ToString());
+            // data...
+            (void)data;
+            m_output->EndTag(AZ_CRC("WriteStatistics", 0xcf7f12aa));
+            m_output->EndTag(m_drillerTag);
+        }
+    } // namespace Debug
+} // namespace GridMate

Code/Framework/GridMate/GridMate/Drillers/SessionDriller.h

@@ -0,0 +1,81 @@
+/*
+* All or portions of this file Copyright (c) Amazon.com, Inc. or its affiliates or
+* its licensors.
+*
+* For complete copyright and license terms please see the LICENSE at the root of this
+* distribution (the "License"). All use of this software is governed by the License,
+* or, if provided, by the license below or the license accompanying this file. Do not
+* remove or modify any license notices. This file is distributed on an "AS IS" BASIS,
+* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+*
+*/
+#ifndef GM_SESSION_DRILLER_H
+#define GM_SESSION_DRILLER_H
+
+#include <GridMate/Types.h>
+#include <GridMate/Session/Session.h>
+#include <AzCore/Driller/Driller.h>
+
+namespace GridMate
+{
+    namespace Debug
+    {
+        /**
+         * Session Driller
+         *  \note Be careful which buses you attach. The drillers work in Multi threaded environment and expect that
+         * a driller mutex (DrillerManager::DrillerManager) will be automatically locked on every write.
+         * Otherwise in output stream corruption will happen (even is the stream is thread safe).
+         */
+        class SessionDriller
+            : public AZ::Debug::Driller
+            , public SessionDrillerBus::Handler
+        {
+            int m_drillerTag;
+        public:
+            AZ_CLASS_ALLOCATOR(SessionDriller, AZ::OSAllocator, 0);
+            SessionDriller();
+
+            //////////////////////////////////////////////////////////////////////////
+            // Driller
+            virtual const char*  GroupName() const          { return "GridMate"; }
+            virtual const char*  GetName() const            { return "SessionDriller"; }
+            virtual const char*  GetDescription() const     { return "Drills GridSession, Search, etc."; }
+            virtual void         Start(const Param* params = NULL, int numParams = 0);
+            virtual void         Stop();
+            //////////////////////////////////////////////////////////////////////////
+
+            //////////////////////////////////////////////////////////////////////////
+            // Session Event Bus
+            /// Callback that is called when the Session service is ready to process sessions.
+            virtual void OnSessionServiceReady();
+            //virtual OnCommucationChanged() = 0  Callback that notifies the title when a member's communication settings change.
+            /// Callback that notifies the title when a game search query have completed.
+            virtual void OnGridSearchComplete(GridSearch* gridSearch);
+            /// Callback that notifies the title when a new member joins the game session.
+            virtual void OnMemberJoined(GridSession* session, GridMember* member);
+            /// Callback that notifies the title that a member is leaving the game session. member pointer is NOT valid after the callback returns.
+            virtual void OnMemberLeaving(GridSession* session, GridMember* member);
+            // \todo a better way will be (after we solve migration) is to supply a reason to OnMemberLeaving... like the member was kicked.
+            // this will require that we actually remove the replica at the same moment.
+            /// Callback that host decided to kick a member. You will receive a OnMemberLeaving when the actual member leaves the session.
+            virtual void OnMemberKicked(GridSession* session, GridMember* member);
+            /// After this callback it is safe to access session features. If host session is fully operational if client wait for OnSessionJoined.
+            virtual void OnSessionCreated(GridSession* session);
+            /// Called on client machines to indicate that we join successfully.
+            virtual void OnSessionJoined(GridSession* session);
+            /// Callback that notifies the title when a session will be left. session pointer is NOT valid after the callback returns.
+            virtual void OnSessionDelete(GridSession* session);
+            /// Called when a session error occurs.
+            virtual void OnSessionError(GridSession* session, const string& errorMsg);
+            /// Called when the actual game(match) starts
+            virtual void OnSessionStart(GridSession* session);
+            /// Called when the actual game(match) ends
+            virtual void OnSessionEnd(GridSession* session);
+            /// Called when we have our last chance to write statistics data for member in the session.
+            virtual void OnWriteStatistics(GridSession* session, GridMember* member, StatisticsData& data);
+            //////////////////////////////////////////////////////////////////////////
+        };
+    }
+}
+
+#endif // GM_SESSION_DRILLER_H

Code/Framework/GridMate/GridMate/EBus.h

@@ -0,0 +1,32 @@
+/*
+* All or portions of this file Copyright (c) Amazon.com, Inc. or its affiliates or
+* its licensors.
+*
+* For complete copyright and license terms please see the LICENSE at the root of this
+* distribution (the "License"). All use of this software is governed by the License,
+* or, if provided, by the license below or the license accompanying this file. Do not
+* remove or modify any license notices. This file is distributed on an "AS IS" BASIS,
+* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+*
+*/
+#ifndef GM_EBUS_H
+#define GM_EBUS_H
+
+#include <GridMate/Memory.h>
+#include <AzCore/EBus/EBus.h>
+#include <AzCore/std/parallel/mutex.h>
+
+namespace GridMate
+{
+    class IGridMate;
+
+    struct GridMateEBusTraits
+        : public AZ::EBusTraits
+    {
+        static const AZ::EBusAddressPolicy AddressPolicy = AZ::EBusAddressPolicy::ById; ///< Allow multiple instances of gridmate in an unordered_map.
+        typedef AZStd::recursive_mutex MutexType;       ///< We do allow running multiple instances gridmate on different threads.
+        typedef IGridMate* BusIdType;       ///< Use the GridMate instance as an ID
+    };
+}
+
+#endif // GM_EBUS_H

Code/Framework/GridMate/GridMate/GridMate.cpp

@@ -0,0 +1,223 @@
+/*
+* All or portions of this file Copyright (c) Amazon.com, Inc. or its affiliates or
+* its licensors.
+*
+* For complete copyright and license terms please see the LICENSE at the root of this
+* distribution (the "License"). All use of this software is governed by the License,
+* or, if provided, by the license below or the license accompanying this file. Do not
+* remove or modify any license notices. This file is distributed on an "AS IS" BASIS,
+* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+*
+*/
+
+#include <AzCore/Memory/AllocationRecords.h>
+#include <AzCore/std/hash.h>
+
+#include <GridMate/GridMate.h>
+#include <GridMate/GridMateService.h>
+#include <GridMate/GridMateEventsBus.h>
+#include <GridMate/Version.h>
+
+namespace GridMate
+{
+    class GridMateImpl
+        : public IGridMate
+    {
+    private:
+        struct ServiceInfo
+        {
+            GridMateService* m_service;
+            GridMateServiceId m_serviceId;
+            bool m_isOwnService;
+        };
+
+        typedef AZStd::vector<ServiceInfo, GridMateStdAlloc> ServiceTable;
+
+    public:
+        AZ_CLASS_ALLOCATOR(GridMateImpl, GridMateAllocator, 0);
+
+        GridMateImpl(const GridMateDesc& desc);
+        virtual ~GridMateImpl();
+
+        void Update() override;
+
+        EndianType GetDefaultEndianType() const override { return m_endianType; }
+
+        void RegisterService(GridMateServiceId id, GridMateService* service, bool delegateOwnership = false) override;
+        void UnregisterService(GridMateServiceId id) override;
+        bool HasService(GridMateServiceId id) override;
+        GridMateService* GetServiceById(GridMateServiceId id) override;
+
+        EndianType m_endianType;
+
+        ServiceTable m_services;
+
+        struct StaticInfo
+        {
+            StaticInfo()
+                : m_numGridMates(0)
+                , m_gridMateAllocatorRefCount(0)
+            { }
+
+            int m_numGridMates;
+            int m_gridMateAllocatorRefCount;
+        };
+        static StaticInfo   s_info;
+    };
+}
+
+GridMate::GridMateImpl::StaticInfo GridMate::GridMateImpl::s_info;
+
+using namespace GridMate;
+
+//=========================================================================
+// GridMateCreate
+//=========================================================================
+IGridMate* GridMate::GridMateCreate(const GridMateDesc& desc)
+{
+    // Memory
+    if (AZ::AllocatorInstance<GridMateAllocator>::IsReady())
+    {
+        AZ_TracePrintf("GridMate", "GridMate Allocator has already started! Ignoring current allocator descriptor!\n");
+        if (GridMateImpl::s_info.m_numGridMates == 0) // add ref count if we did not start it at all
+        {
+            GridMateImpl::s_info.m_gridMateAllocatorRefCount = 1;
+        }
+    }
+    else
+    {
+        AZ::AllocatorInstance<GridMateAllocator>::Create(desc.m_allocatorDesc);
+    }
+
+    GridMateImpl::s_info.m_numGridMates++;
+    GridMateImpl::s_info.m_gridMateAllocatorRefCount++;
+
+    GridMateImpl* impl = aznew GridMateImpl(desc);
+    EBUS_EVENT_ID(impl, GridMateEventsBus, OnGridMateInitialized, impl);
+    return impl;
+}
+
+//=========================================================================
+// GridMateCreate
+//=========================================================================
+void GridMate::GridMateDestroy(IGridMate* gridMate)
+{
+    AZ_Assert(gridMate != nullptr, "Invalid GridMate interface pointer!");
+    EBUS_EVENT_ID(gridMate, GridMateEventsBus, OnGridMateShutdown, gridMate);
+
+    delete gridMate;
+    GridMateImpl::s_info.m_numGridMates--;
+    GridMateImpl::s_info.m_gridMateAllocatorRefCount--;
+
+    if (GridMateImpl::s_info.m_gridMateAllocatorRefCount == 0)
+    {
+        AZ::AllocatorInstance<GridMateAllocator>::Destroy();
+    }
+
+    if (GridMateImpl::s_info.m_numGridMates == 0)
+    {
+        GridMateImpl::s_info.m_gridMateAllocatorRefCount = 0;
+    }
+}
+
+//=========================================================================
+// GridMateImpl
+//=========================================================================
+GridMateImpl::GridMateImpl(const GridMateDesc& desc)
+{
+    m_endianType = desc.m_endianType;
+}
+
+//=========================================================================
+// ~GridMateImpl
+//=========================================================================
+GridMateImpl::~GridMateImpl()
+{
+    while (!m_services.empty())
+    {
+        ServiceInfo registeredService = m_services.back();
+        m_services.pop_back();
+        registeredService.m_service->OnServiceUnregistered(this);
+        if (registeredService.m_isOwnService)
+        {
+            delete registeredService.m_service;
+        }
+    }
+}
+
+void GridMateImpl::RegisterService(GridMateServiceId id, GridMateService* service, bool delegateOwnership)
+{
+    AZ_Assert(service, "Invalid service");
+
+    GridMateService* duplicate = GetServiceById(id);
+    AZ_Assert(!duplicate, "Trying to register the same GridMate service id twice.");
+    if (!duplicate)
+    {
+        service->OnServiceRegistered(this);
+        m_services.push_back();
+        ServiceInfo& serviceInfo = m_services.back();        
+        serviceInfo.m_isOwnService = delegateOwnership;
+        serviceInfo.m_service = service;
+        serviceInfo.m_serviceId = id;
+        EBUS_EVENT_ID(this, GridMateEventsBus, OnGridMateServiceAdded, this, service);
+    }
+    else
+    {        
+        if (delegateOwnership)
+        {
+            delete service;
+        }
+    }
+}
+
+void GridMateImpl::UnregisterService(GridMateServiceId id)
+{
+    for (auto iter = m_services.begin(); iter != m_services.end(); ++iter)
+    {
+        if (iter->m_serviceId == id)
+        {
+            ServiceInfo serviceInfo = *iter;
+            m_services.erase(iter);
+            serviceInfo.m_service->OnServiceUnregistered(this);
+            if (serviceInfo.m_isOwnService)
+            {
+                delete serviceInfo.m_service;
+            }
+            return;
+        }
+    }
+    AZ_Error("GridMate", false, "Trying to stop an unregistered session service.");
+}
+
+bool GridMateImpl::HasService(GridMateServiceId id)
+{
+    GridMateService* service = GetServiceById(id);
+    return service != nullptr;
+}
+
+GridMateService* GridMateImpl::GetServiceById(GridMateServiceId id)
+{
+    for (auto iter = m_services.begin(); iter != m_services.end(); ++iter)
+    {
+        if (id == iter->m_serviceId)
+        {
+            return iter->m_service;
+        }
+    }
+    return nullptr;
+}
+
+//=========================================================================
+// Update
+//=========================================================================
+void
+GridMateImpl::Update()
+{
+    for (auto serviceIter = m_services.begin(); serviceIter != m_services.end(); ++serviceIter)
+    {
+        serviceIter->m_service->OnGridMateUpdate(this);
+    }
+
+    EBUS_EVENT_ID(this, GridMateEventsBus, OnGridMateUpdate, this);
+
+}

Code/Framework/GridMate/GridMate/GridMate.h

@@ -0,0 +1,111 @@
+/*
+* All or portions of this file Copyright (c) Amazon.com, Inc. or its affiliates or
+* its licensors.
+*
+* For complete copyright and license terms please see the LICENSE at the root of this
+* distribution (the "License"). All use of this software is governed by the License,
+* or, if provided, by the license below or the license accompanying this file. Do not
+* remove or modify any license notices. This file is distributed on an "AS IS" BASIS,
+* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+*
+*/
+#ifndef GRID_MATE_H
+#define GRID_MATE_H 1
+
+#include <GridMate/Carrier/Carrier.h>
+#include <GridMate/Types.h>
+
+/// \file gridmate.h
+
+namespace GridMate
+{
+    /**
+    * GridMate creation descriptor.
+    */
+    struct GridMateDesc
+    {
+        GridMateDesc()
+            : m_allocatorDesc()
+            , m_endianType(EndianType::BigEndian) { }
+
+        /**
+        * GridMate default allocator. It will be used for all basic services and online module.
+        */
+        GridMateAllocator::Descriptor m_allocatorDesc;
+
+        /**
+        * Endianness serialized to the network.
+        */
+        EndianType m_endianType;
+    };
+
+    class GridMateService;
+    class SessionService;
+    struct SessionServiceDesc;
+    struct SessionParams;
+    struct JoinParams;
+    struct SearchParams;
+    struct SearchInfo;
+    struct InviteInfo;
+    struct SessionIdInfo;
+    class GridSession;
+    class GridSearch;
+
+    /**
+    * GridMate interface.
+    */
+    class IGridMate
+    {
+    public:
+        virtual ~IGridMate() { }
+        virtual void Update() = 0;
+
+        virtual EndianType GetDefaultEndianType() const = 0;
+
+        // Binds service to GridMate instance, GridMate owns session pointer after that and responsible for releasing it,
+        // if delegateOwnership flag is set -> GridMate will take ownership over service instance and will be soleily responsible for it's deletion
+        virtual void RegisterService(GridMateServiceId id, GridMateService* service, bool delegateOwnership = false) = 0;
+
+        // Unbinds service from GridMate instance, service should not be used after this is called
+        virtual void UnregisterService(GridMateServiceId id) = 0;
+
+        // Returns true if a service with the specified service id is currently registered with this GridMate
+        virtual bool HasService(GridMateServiceId id) = 0;
+
+        // Returns the service registered with the specified service id, or nullptr if not found.
+        virtual GridMateService* GetServiceById(GridMateServiceId id) = 0;
+    };
+
+
+
+    /**
+    * Helper function to start service of given type and register it with GridMate.
+    * Newly created service instance will be owned by GridMate.
+    */
+    template<class ServiceType, class ... Args>
+    ServiceType* StartGridMateService(IGridMate* gridMate, Args&& ... args)
+    {
+        ServiceType* service = aznew ServiceType(AZStd::forward<Args>(args) ...);
+        gridMate->RegisterService(ServiceType::GetGridMateServiceId(),service, true);
+        return service;
+    }
+
+    template<class ServiceType>
+    void StopGridMateService(IGridMate* gridMate)
+    {
+        gridMate->UnregisterService(ServiceType::GetGridMateServiceId());
+    }
+
+    template<class ServiceType>
+    bool HasGridMateService(IGridMate* gridMate)
+    {
+        return gridMate->HasService(ServiceType::GetGridMateServiceId());
+    }
+
+    /// Create GridMate interface object. You are allowed to have only one active at a time. \todo use shared_ptr or intrusive_ptr
+    IGridMate* GridMateCreate(const GridMateDesc& desc);
+    /// Destroys and frees all GridMate resources.
+    void GridMateDestroy(IGridMate* gridMate);
+}
+
+#endif // GRID_MATE_H

Code/Framework/GridMate/GridMate/GridMateEventsBus.h

@@ -0,0 +1,62 @@
+/*
+* All or portions of this file Copyright (c) Amazon.com, Inc. or its affiliates or
+* its licensors.
+*
+* For complete copyright and license terms please see the LICENSE at the root of this
+* distribution (the "License"). All use of this software is governed by the License,
+* or, if provided, by the license below or the license accompanying this file. Do not
+* remove or modify any license notices. This file is distributed on an "AS IS" BASIS,
+* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+*
+*/
+#ifndef GRIDMATEEVENTSBUS_H
+#define GRIDMATEEVENTSBUS_H
+
+#include <GridMate/Memory.h>
+#include <AzCore/EBus/EBus.h>
+
+namespace GridMate
+{
+    class IGridMate;
+    class GridMateService;
+
+    /*
+    * GridMate callbacks
+    * These callbacks are thrown on main GridMate thread(thread where GridMate's tick is pumped on)
+    */
+    class GridMateEvents
+        : public AZ::EBusTraits
+    {
+    public:
+        // Called after gridmate is initialized
+        virtual void OnGridMateInitialized(IGridMate* gridMate) { (void)gridMate; }
+
+        // Called on GridMate tick
+        virtual void OnGridMateUpdate(IGridMate* gridMate) { (void)gridMate; }
+
+        // Called when gridmate is shutting down, GridMate reference is still valid inside this call, but should not be used afterwards
+        virtual void OnGridMateShutdown(IGridMate* gridMate) { (void)gridMate; }
+
+        // Called when new service is added to GridMate
+        virtual void OnGridMateServiceAdded(IGridMate* gridMate, GridMateService* service) { (void)gridMate; (void)service; }
+
+        // Called when service is about to be deleted, service cannot be used afterwards.
+        virtual void OnGridMateServiceDelete(IGridMate* gridMate, GridMateService* service) { (void)gridMate; (void)service; }
+
+        // EBus Traits
+        AZ_CLASS_ALLOCATOR(GridMateEvents, GridMateAllocator, 0);
+
+        static const AZ::EBusAddressPolicy AddressPolicy = AZ::EBusAddressPolicy::ById;
+        static const AZ::EBusHandlerPolicy HandlerPolicy = AZ::EBusHandlerPolicy::MultipleAndOrdered; ///< Events are ordered, each handler may set its priority
+        typedef IGridMate* BusIdType; ///< Use the GridMate instance as an ID
+        bool Compare(const GridMateEvents* another) const { return GetPriority() > another->GetPriority(); }
+
+    protected:
+        static const unsigned int k_defaultPriority = 100; ///< priority that will be used for callback ordering (the smaller - the earler handler will be in events queue), default is 100
+        virtual unsigned int GetPriority() const { return k_defaultPriority; }
+    };
+
+    typedef AZ::EBus<GridMateEvents> GridMateEventsBus;
+}
+
+#endif

Code/Framework/GridMate/GridMate/GridMateService.h

@@ -0,0 +1,43 @@
+/*
+* All or portions of this file Copyright (c) Amazon.com, Inc. or its affiliates or
+* its licensors.
+*
+* For complete copyright and license terms please see the LICENSE at the root of this
+* distribution (the "License"). All use of this software is governed by the License,
+* or, if provided, by the license below or the license accompanying this file. Do not
+* remove or modify any license notices. This file is distributed on an "AS IS" BASIS,
+* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+*
+*/
+#ifndef GRIDMATESERVICE_H
+#define GRIDMATESERVICE_H
+
+#include <GridMate/Types.h>
+
+#define GRIDMATE_SERVICE_ID(GridMateService) static GridMate::GridMateServiceId GetGridMateServiceId()  { return AZ::Crc32(#GridMateService); }
+
+namespace GridMate
+{
+    class IGridMate;
+
+    /*
+    * Generic GridMate service interface
+    * All services should implement this interface
+    */
+    class GridMateService
+    {
+    public:
+        virtual ~GridMateService() {}
+
+        // Called when service is bound to GridMate instance
+        virtual void OnServiceRegistered(IGridMate* gridMate) = 0;
+
+        // Called when service is unregistered from given GridMate instance
+        virtual void OnServiceUnregistered(IGridMate* gridMate) = 0;
+
+        // Called on GridMate tick
+        virtual void OnGridMateUpdate(IGridMate* gridMate) { (void)gridMate; }
+    };
+}
+
+#endif

Code/Framework/GridMate/GridMate/MathUtils.h

@@ -0,0 +1,123 @@
+/*
+* All or portions of this file Copyright (c) Amazon.com, Inc. or its affiliates or
+* its licensors.
+*
+* For complete copyright and license terms please see the LICENSE at the root of this
+* distribution (the "License"). All use of this software is governed by the License,
+* or, if provided, by the license below or the license accompanying this file. Do not
+* remove or modify any license notices. This file is distributed on an "AS IS" BASIS,
+* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+*
+*/
+#ifndef GM_MATHUTILS_H
+#define GM_MATHUTILS_H
+
+#include <AzCore/base.h>
+
+namespace GridMate
+{
+    //-------------------------------------------------------------------------
+    // helper functions to encode float as int while preserving relative-order
+    // between +/- numbers and equality between +0.f and -0.f
+    // http://www.cygnus-software.com/papers/comparingfloats/Comparing%20floating%20point%20numbers.htm
+    //-------------------------------------------------------------------------
+    AZ_FORCE_INLINE int encode_float_as_int(float v)
+    {
+        union
+        {
+            int m_i;
+            float m_f;
+        };
+
+        m_f = v;
+        return m_i < 0 ? 0x80000000 - m_i : m_i;
+    }
+    //-------------------------------------------------------------------------
+    AZ_FORCE_INLINE float decode_float_as_int(int v)
+    {
+        union
+        {
+            int m_i;
+            float m_f;
+        };
+
+        m_i = v < 0 ? 0x80000000 - v : v;
+        return m_f;
+    }
+    //-------------------------------------------------------------------------
+    AZ_FORCE_INLINE AZ::u32 encode_int_as_uint(AZ::s32 val)
+    {
+        AZ::u64 val64 = (~static_cast<AZ::u32>(val)) + 0x80000000u; // packing signed int into unsigned: 0x00(127)..0x7F(0), 0x80(-1)..0xFE(-127)
+        return static_cast<AZ::u32>(val64);
+    }
+    //-------------------------------------------------------------------------
+    AZ_FORCE_INLINE AZ::s32 decode_int_as_uint(AZ::u32 val)
+    {
+        AZ::u64 val64 = val - 0x80000000u;
+        return static_cast<AZ::s32>(~val64);
+    }
+    //-------------------------------------------------------------------------
+    template<class T, unsigned int size>
+    class RollingSum
+    {
+    public:
+        AZ_FORCE_INLINE RollingSum()
+            : m_sum(T())
+            , m_pos(0)
+            , m_accumDt(0.f)
+            , m_accumValue(T())
+        {
+            for (T& val : m_history)
+            {
+                val = T();
+            }
+        }
+
+        AZ_FORCE_INLINE T GetSum() const
+        {
+            return m_sum;
+        }
+
+        AZ_FORCE_INLINE void Update(float dt, T value)
+        {
+            m_accumDt += dt;
+            m_accumValue += value;
+
+            const float thresholdDt = 1.f / size;
+            if (m_accumDt < thresholdDt)
+            {
+                return;
+            }
+
+            const float maxDt = size * thresholdDt; // clamping in case there was delay >1sec
+            m_accumValue = AZStd::GetMin(m_accumValue, m_accumValue * static_cast<T>(maxDt / m_accumDt));
+            m_accumDt = AZStd::GetMin(m_accumDt, maxDt);
+
+            unsigned int bytes = static_cast<unsigned int>(thresholdDt * m_accumValue / m_accumDt);
+            while (m_accumDt >= thresholdDt)
+            {
+                Add(bytes);
+                m_accumValue -= bytes;
+                m_accumDt -= thresholdDt;
+            }
+        }
+
+    private:
+        T m_sum;
+        T m_history[size];
+        size_t m_pos;
+
+        float m_accumDt;
+        T m_accumValue;
+
+        AZ_FORCE_INLINE void Add(T value)
+        {
+            m_sum -= m_history[m_pos];
+            m_history[m_pos] = value;
+            m_sum += value;
+            m_pos = (m_pos + 1) % size;
+        }
+    };
+}   // namespace GridMate
+
+#endif // GM_MATHUTILS_H

Code/Framework/GridMate/GridMate/Memory.h

@@ -0,0 +1,72 @@
+/*
+* All or portions of this file Copyright (c) Amazon.com, Inc. or its affiliates or
+* its licensors.
+*
+* For complete copyright and license terms please see the LICENSE at the root of this
+* distribution (the "License"). All use of this software is governed by the License,
+* or, if provided, by the license below or the license accompanying this file. Do not
+* remove or modify any license notices. This file is distributed on an "AS IS" BASIS,
+* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+*
+*/
+#ifndef GM_MEMORY_H
+#define GM_MEMORY_H
+
+#include <AzCore/base.h>
+#include <AzCore/Memory/OSAllocator.h>
+#include <AzCore/Memory/SystemAllocator.h>
+
+namespace GridMate
+{
+    /**
+    * GridMateAllocator is used by non-MP portions of GridMate
+    */
+    class GridMateAllocator
+        : public AZ::SimpleSchemaAllocator<AZ::ChildAllocatorSchema<AZ::SystemAllocator>>
+    {
+    public:
+        AZ_TYPE_INFO(GridMateAllocator, "{BB127E7A-E4EF-4480-8F17-0C10146D79E0}")
+
+        using Base = AZ::SimpleSchemaAllocator<AZ::ChildAllocatorSchema<AZ::SystemAllocator>>;
+        using Descriptor = Base::Descriptor;
+
+        GridMateAllocator()
+            : GridMateAllocator::Base("GridMate Allocator", "GridMate fundamental generic memory allocator")
+        {}
+    };
+
+    /**
+    * GridMateAllocatorMP is used by MP portions of GridMate
+    */
+    class GridMateAllocatorMP
+        : public AZ::SimpleSchemaAllocator<AZ::ChildAllocatorSchema<AZ::SystemAllocator>>
+    {
+        friend class AZ::AllocatorInstance<GridMateAllocatorMP>;
+    public:
+        AZ_TYPE_INFO(GridMateAllocatorMP, "{FABCBC6E-B3E5-4200-861E-A3EC22592678}")
+
+        using Base = AZ::SimpleSchemaAllocator<AZ::ChildAllocatorSchema<AZ::SystemAllocator>>;
+        using Descriptor = Base::Descriptor;
+
+        GridMateAllocatorMP()
+            : GridMateAllocatorMP::Base("GridMate Multiplayer Allocator", "GridMate Multiplayer data allocations (Session,Replica,Carrier)")
+        {}
+
+        // TODO: We have an aggressive memory policy in the Carrier. We have 2 ways to fix it.
+        // Either keep a cap and sacrifice performance or create a carrier->GarbageCollection and call it from here
+        //virtual void          GarbageCollect()                 { EBUS_EVENT(CarrierBus,GarbageCollect); m_allocator->GarbageCollect(); }
+    };
+
+    //! GridMate system container allocator.
+    typedef AZ::AZStdAlloc<GridMateAllocator> GridMateStdAlloc;
+
+    //! GridMate system container allocator.
+    typedef AZ::AZStdAlloc<GridMateAllocatorMP> SysContAlloc;
+}   // namespace GridMate
+
+
+#define GM_CLASS_ALLOCATOR(_type)       AZ_CLASS_ALLOCATOR(_type, GridMate::GridMateAllocatorMP, 0)
+#define GM_CLASS_ALLOCATOR_DECL         AZ_CLASS_ALLOCATOR_DECL
+#define GM_CLASS_ALLOCATOR_IMPL(_type)  AZ_CLASS_ALLOCATOR_IMPL(_type, GridMate::GridMateAllocatorMP, 0)
+
+#endif // GM_MEMORY_H

Code/Framework/GridMate/GridMate/Online/OnlineUtilityThread.h

@@ -0,0 +1,97 @@
+/*
+* All or portions of this file Copyright (c) Amazon.com, Inc. or its affiliates or
+* its licensors.
+*
+* For complete copyright and license terms please see the LICENSE at the root of this
+* distribution (the "License"). All use of this software is governed by the License,
+* or, if provided, by the license below or the license accompanying this file. Do not
+* remove or modify any license notices. This file is distributed on an "AS IS" BASIS,
+* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+*
+*/
+/**
+ * @file
+ * Provides EBus definitions for getting the utility thread tick
+ */
+#ifndef ONLINE_UTILITY_THREAD_H
+#define ONLINE_UTILITY_THREAD_H
+
+#include <GridMate/EBus.h>
+#include <AzCore/std/parallel/mutex.h>
+
+/**
+  * IMPORTANT NOTE TO SERVICES THAT USE THE UTILITY THREAD:
+  * The online service will start ticking the utility thread at construction
+  * time, but you have have to let it know when you need to be ticked. This
+  * is done two ways: first, send the NotifyOfNewWork event to the
+  * OnlineUtilityThreadCommandBus; second, return whether you still have
+  * work to do in OnlineUtilityThreadNotificationBus's event
+  * IsThereUtilityThreadWork. There, however, caveats the service
+  * must be aware of.
+  *  - For those that derive from OnlineUtilityNotificationBus::Handler,
+  *    do your BusConnect and BusDisConnect calls in your Init and Shutdown
+  *    calls, instead of at construction and destruction time. You shouldn't
+  *    be trying to use this utility thread outside of the time between these
+  *    calls to your service anyway, so this shouldn't cause any amount of
+  *    headache to conform to.
+  *  - When you call BusConnect and BusDiscconect, the online manager may
+  *    already be ticking that event - you may or may not receive your first
+  *    and/or last tick events the way you might expect, so be careful about
+  *    how you do you initialization and shutdown procedures.
+  *  - Your Init call should do as little work as possible. Set yourself up for
+  *    being ready to do actual initialization the first time you receive the
+  *    OnUtilityThreadTick event instead of doing it all in Init and blocking
+  *    the main thread.
+  *  - Your Shutdown call should abort any pending operations, including ones
+  *    it's already in the middle of.
+  *  - In your OnUtilityThreadTick event response, make sure you haven't already
+  *    been told to shut down. This is because the Shutdown call may have been
+  *    made soon after the OnUtilityThreadTick event was fired, and other
+  *    services took up a fair amount of time before the event got to you (with
+  *    the Shutdown call to your service being made between event-firing and
+  *    when the event reached you).
+  *  - Be VERY careful about Shutdown getting called before you finish
+  *    initializing in the utility thread (or even get a change to)! If you
+  *    use this utility thread, be sure to test whether you can shutdown
+  *    immediately after being initialized without breaking anything.
+  */
+namespace GridMate
+{
+    //-------------------------------------------------------------------------
+    // For ticking services that need a separate thread (outbound)
+    //  - BusConnect to OnlineUtilityThreadNotificationBus::Handler to receive OnUtilityThreadTick
+    //  - Return whether you have work left to do in IsThereWork
+    //-------------------------------------------------------------------------
+    class OnlineUtilityThreadNotifications
+        : public GridMateEBusTraits
+    {
+    public:
+        virtual ~OnlineUtilityThreadNotifications() {}
+
+        // Called on each iteration of the online manager's utility thread loop
+        virtual void OnUtilityThreadTick() = 0;
+
+        // Return whether there's work left to do here to keep the thread from doing busy waiting
+        virtual bool IsThereUtilityThreadWork() = 0;
+    };
+    typedef AZ::EBus<OnlineUtilityThreadNotifications> OnlineUtilityThreadNotificationBus;
+    //-------------------------------------------------------------------------
+
+    //-------------------------------------------------------------------------
+    // For services that need a separate thread (inbound)
+    //  - Fire the NotifyOfNewWork event to notify the thread that you have a new
+    //    request you'd like to take care of
+    //-------------------------------------------------------------------------
+    class OnlineUtilityThreadCommands
+        : public GridMateEBusTraits
+    {
+    public:
+        virtual ~OnlineUtilityThreadCommands() {}
+
+        virtual void NotifyOfNewWork() = 0;
+    };
+    typedef AZ::EBus<OnlineUtilityThreadCommands> OnlineUtilityThreadCommandBus;
+    //-------------------------------------------------------------------------
+} // namespace GridMate
+
+#endif // ONLINE_UTILITY_THREAD_H

Code/Framework/GridMate/GridMate/Online/UserServiceTypes.h

@@ -0,0 +1,119 @@
+/*
+* All or portions of this file Copyright (c) Amazon.com, Inc. or its affiliates or
+* its licensors.
+*
+* For complete copyright and license terms please see the LICENSE at the root of this
+* distribution (the "License"). All use of this software is governed by the License,
+* or, if provided, by the license below or the license accompanying this file. Do not
+* remove or modify any license notices. This file is distributed on an "AS IS" BASIS,
+* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+*
+*/
+#ifndef GM_USER_SERVICE_TYPES_H
+#define GM_USER_SERVICE_TYPES_H
+
+#include <GridMate/Types.h>
+#include <GridMate/String/string.h>
+
+namespace GridMate
+{
+    /**
+     * User signin state
+     */
+    enum OLSSigninState
+    {
+        OLS_SigninUnknown,
+        OLS_NotSignedIn,        // There is no user signed in
+        OLS_SignedInOffline,    // User signed in without online capabilities
+        OLS_SignedInOnline,     // User signed in with online capabilities
+        OLS_SigningOut,         // User is in the process of signing out
+    };
+
+    /**
+     * service network state
+     */
+    enum OLSOnlineState
+    {
+        OLS_OnlineUnknown,
+        OLS_NoNetwork,          // No NIC or network is unplugged
+        OLS_Offline,            // No online access
+        OLS_Online,             // Has online access
+    };
+
+    /**
+     * Supported privilege types
+     */
+    enum OLSUserPrivilege
+    {
+        OLS_UserPrivilegeMP,
+        OLS_UserPrivilegeRecordDVR,
+        OLS_UserPrivilegePurchaseContent,
+        OLS_UserPrivilegeVoiceChat,
+        OLS_UserPrivilegeLeaderboards
+    };
+
+    /**
+     * Base class for platform dependent player id.
+     */
+    struct PlayerId
+    {
+        PlayerId(ServiceType serviceType)
+            : m_serviceType(serviceType)  {}
+        virtual ~PlayerId()                                             {}
+
+        // Compare 2 PlayerId IDs
+        virtual bool Compare(const PlayerId& userId) const = 0;
+
+        // Returns a printable string representation of the id.
+        virtual gridmate_string ToString() const = 0;
+
+        ServiceType GetType() const { return m_serviceType; }
+
+    protected:
+        ServiceType m_serviceType;
+    };
+
+    /**
+     * Interface class for a local player/member.
+     */
+    class ILocalMember
+    {
+    public:
+        virtual ~ILocalMember() {}
+        // SignIn
+        virtual OLSSigninState  GetSigninState() const = 0;
+
+        virtual const PlayerId* GetPlayerId() const = 0;
+
+        // Pad number / info    ???
+        virtual unsigned int    GetControllerIndex() const = 0;
+        virtual const char*     GetName() const = 0;
+        virtual bool            IsGuest() const = 0;
+
+        // Friends List
+        virtual void            RefreshFriends() = 0;
+        virtual bool            IsFriendsListRefreshing() const = 0;
+        virtual unsigned int    GetFriendsCount() const = 0;
+        virtual const char*     GetFriendName(unsigned int idx) const = 0;
+        virtual const PlayerId* GetFriendPlayerId(unsigned int idx) const = 0;
+        virtual OLSSigninState  GetFriendSigninState(unsigned int idx) const = 0;
+        virtual bool            IsFriendPlayingTitle(unsigned int idx) const = 0;
+        virtual const char*     GetFriendPresenceDetails(unsigned int idx) const = 0;
+        virtual bool            IsFriendsWith(const PlayerId* playerId) const = 0;
+    };
+
+    /**
+    * Generic invite structure
+    * pPlatformSpecific contains the native structure used by each platform
+    */
+    struct  InviteInfo
+    {
+        InviteInfo()
+            : m_localMember(nullptr) {}
+
+        ILocalMember* m_localMember;
+    };
+}   // namespace GridMate
+
+#endif  // GM_USER_SERVICE_TYPES_H
+#pragma once

Code/Framework/GridMate/GridMate/Replica/BasicHostChunkDescriptor.h

@@ -0,0 +1,43 @@
+/*
+* All or portions of this file Copyright (c) Amazon.com, Inc. or its affiliates or
+* its licensors.
+*
+* For complete copyright and license terms please see the LICENSE at the root of this
+* distribution (the "License"). All use of this software is governed by the License,
+* or, if provided, by the license below or the license accompanying this file. Do not
+* remove or modify any license notices. This file is distributed on an "AS IS" BASIS,
+* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+*
+*/
+
+#pragma once
+
+#include <GridMate/Replica/ReplicaChunkDescriptor.h>
+#include <GridMate/Replica/ReplicaMgr.h>
+
+namespace GridMate
+{
+    /**
+    * BasicHostChunkDescriptor implements a helper descriptor
+    * that prevents chunk proxies from being created on the host.
+    * The idea is to prevent a malicious client from creating
+    * (and owning) chunk types that should always be authoritative
+    * on the host.
+    */
+    template<typename ReplicaChunkType>
+    class BasicHostChunkDescriptor
+        : public DefaultReplicaChunkDescriptor<ReplicaChunkType>
+    {
+    public:
+        ReplicaChunkBase* CreateFromStream(UnmarshalContext& mc) override
+        {
+            ReplicaChunkBase* replicaChunk = nullptr;
+            AZ_Assert(!mc.m_rm->IsSyncHost(), "Replicas of type %s can only be owned by the host!", DefaultReplicaChunkDescriptor<ReplicaChunkType>::GetChunkName());
+            if (!mc.m_rm->IsSyncHost())
+            {
+                replicaChunk = aznew ReplicaChunkType;
+            }
+            return replicaChunk;
+        }
+    };
+}

Code/Framework/GridMate/GridMate/Replica/DataSet.cpp

@@ -0,0 +1,57 @@
+/*
+* All or portions of this file Copyright (c) Amazon.com, Inc. or its affiliates or
+* its licensors.
+*
+* For complete copyright and license terms please see the LICENSE at the root of this
+* distribution (the "License"). All use of this software is governed by the License,
+* or, if provided, by the license below or the license accompanying this file. Do not
+* remove or modify any license notices. This file is distributed on an "AS IS" BASIS,
+* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+*
+*/
+
+#include <GridMate/Replica/Replica.h>
+#include <GridMate/Replica/ReplicaChunkDescriptor.h>
+#include <GridMate/Replica/ReplicaMgr.h>
+#include <GridMate/Replica/DataSet.h>
+
+namespace GridMate
+{
+    DataSetBase::DataSetBase(const char* debugName)
+        : m_maxIdleTicks(5.f)
+        , m_streamCache(EndianType::IgnoreEndian, 64)
+        , m_replicaChunk(nullptr)
+        , m_lastUpdateTime(0)
+        , m_isDefaultValue(true)
+        , m_revision(0)  //null stamp
+        , m_override(nullptr)
+    {
+        ReplicaChunkInitContext* initContext = ReplicaChunkDescriptorTable::Get().GetCurrentReplicaChunkInitContext();
+        AZ_Assert(initContext, "Replica's context was NOT pushed on the stack! Call Replica::Descriptor::Push() before construction!");
+        ReplicaChunkDescriptor* descriptor = initContext->m_descriptor;
+        AZ_Assert(descriptor, "Replica's descriptor was not stored in InitContext!");
+        descriptor->RegisterDataSet(debugName, this);
+    }
+
+    ReadBuffer DataSetBase::GetMarshalData() const
+    {
+        AZ_Assert(m_streamCache.Size() != 0, "The value was not written to the stream cache!");
+
+        return ReadBuffer(m_streamCache.GetEndianType(), m_streamCache.Get(), m_streamCache.GetExactSize());
+    }
+
+    void DataSetBase::SetDirty()
+    {
+        m_isDefaultValue = false;
+
+        if (m_replicaChunk)
+        {
+            m_replicaChunk->SignalDataSetChanged(*this);
+        }
+    }
+
+    bool DataSetBase::CanSet() const
+    {
+        return m_replicaChunk ? m_replicaChunk->IsMaster() : true;
+    }
+}

Code/Framework/GridMate/GridMate/Replica/DataSet.h

@@ -0,0 +1,557 @@
+/*
+* All or portions of this file Copyright (c) Amazon.com, Inc. or its affiliates or
+* its licensors.
+*
+* For complete copyright and license terms please see the LICENSE at the root of this
+* distribution (the "License"). All use of this software is governed by the License,
+* or, if provided, by the license below or the license accompanying this file. Do not
+* remove or modify any license notices. This file is distributed on an "AS IS" BASIS,
+* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+*
+*/
+#ifndef GM_DATASET_H
+#define GM_DATASET_H
+
+#include <AzCore/std/functional.h>
+#include <GridMate/Containers/vector.h>
+
+#include <AzCore/std/containers/unordered_map.h>
+#include <AzCore/std/smart_ptr/make_shared.h>
+#include <AzCore/std/smart_ptr/shared_ptr.h>
+#include <AzCore/std/smart_ptr/weak_ptr.h>
+
+#include <GridMate/Replica/Replica.h>
+#include <GridMate/Replica/ReplicaChunk.h>
+#include <GridMate/Replica/ReplicaCommon.h>
+#include <GridMate/Replica/Throttles.h>
+#include <GridMate/Replica/ReplicaTarget.h>
+
+#include <GridMate/Serialize/Buffer.h>
+#include <AzCore/std/containers/ring_buffer.h>
+
+
+namespace AzFramework
+{
+    class NetworkContext;
+}
+
+namespace GridMate
+{
+    /**
+     * \brief Default DataSet callbacks traits.
+     */
+    struct DataSetDefaultTraits
+    {
+        /**
+         * \brief Should a change in DataSet value invoke a callback on a master replica chunk? 
+         * 
+         * By default, DataSet::BindInterface<C, &C::Callback> only invokes on client/non-authoritative replica chunks.
+         * This switch enables the callback on server/authoritative replica chunks.
+         * Warning: this change should not be enabled on existing Lumberyard components as they were not written with this option in mind.
+         * 
+         * New user custom replica chunk will work just fine.
+         */
+        static const bool s_invokeAuthoritativeCallback = false;
+    };
+
+    /**
+     * \brief Turns on DataSet callbacks to be invoked on the master replica as well as client replicas.
+     */
+    struct DataSetInvokeEverywhereTraits : DataSetDefaultTraits
+    {
+        static const bool s_invokeAuthoritativeCallback = true;
+    };
+
+    class ReplicaChunkDescriptor;
+    class ReplicaChunkBase;
+    class ReplicaMarshalTaskBase;
+    class ReplicaPeer;
+    class ReplicaTarget;
+
+    /**
+    * DataSetBase
+    * Base type for all replica datasets
+    */
+    class DataSetBase
+    {
+        friend ReplicaChunkDescriptor;
+        friend ReplicaChunkBase;
+        friend AzFramework::NetworkContext;
+
+    public:
+        void SetMaxIdleTime(float dt) { m_maxIdleTicks = dt; }
+        float GetMaxIdleTime() const { return m_maxIdleTicks; }
+        bool CanSet() const;
+        bool IsDefaultValue() const { return m_isDefaultValue; }
+        void MarkAsDefaultValue() { m_isDefaultValue = true; }
+        void MarkAsNonDefaultValue() { m_isDefaultValue = false; }
+        /**
+        * Returns the last updated network time of the DataSet.
+        */
+        unsigned int GetLastUpdateTime() const { return m_lastUpdateTime; }
+
+        ReplicaChunkBase* GetReplicaChunkBase() const { return m_replicaChunk; }
+
+        AZ::u64 GetRevision() const { return m_revision; }
+
+        using DispatchCallback = AZStd::function<void(const TimeContext& tc)>;
+
+        /**
+         * \brief Delta compressed DataSets use an intermediary to catch dispatches of changed DataSets in their logic
+         * \param callback to a custom object when a DataSet changes
+         */
+        void SetDispatchOverride(DispatchCallback callback) { m_override = callback; }
+
+        /**
+         * \brief Delta compressed fields override a dispatch
+         * \return not-null if this DataSet is used for Delta Compression
+         */
+        const DispatchCallback& GetDispatchOverride() const { return m_override; }
+
+    protected:
+        explicit DataSetBase(const char* debugName);
+        virtual ~DataSetBase() = default;
+
+        virtual PrepareDataResult PrepareData(EndianType endianType, AZ::u32 marshalFlags) = 0;
+        virtual void Unmarshal(UnmarshalContext& mc) = 0;
+        virtual void ResetDirty() = 0;
+        virtual void SetDirty();
+        virtual void DispatchChangedEvent(const TimeContext& tc) { (void)tc; }
+        ReadBuffer GetMarshalData() const;
+
+        float               m_maxIdleTicks;             //Note: used only if ACK feedback disabled
+        WriteBufferDynamic  m_streamCache;
+        ReplicaChunkBase*   m_replicaChunk;             ///< raw pointer, assuming datasets do not exists without replica chunk
+        unsigned int        m_lastUpdateTime;
+
+        bool                m_isDefaultValue;
+        AZ::u64             m_revision;          ///< Latest revision number; 0 means unset
+
+        DispatchCallback m_override; // Used by delta compressed DataSets to combine dispatch callbacks
+    };
+
+    // This shim is here to temporarily allow support for Pointer type unmarshalling
+    // (current approach of detecting differences doesn't readily allow for this behavior)
+    // This is mainly to support ScriptProperties.
+    class MarshalerShim
+    {
+    public:
+        template<class MarshalerType, class DataType>
+        static bool Unmarshal(UnmarshalContext& mc, MarshalerType& marshaler, DataType& sourceValue, AZStd::false_type /*isPointerType*/)
+        {
+            DataType value;
+
+            // Expects a return of whether or not the value was actually read.
+            if (mc.m_iBuf->Read(value, marshaler))
+            {
+                if (!(value == sourceValue))
+                {
+                    sourceValue = value;
+                    return true;
+                }
+            }
+
+            return false;
+        }
+
+        template<class MarshalerType, class DataType>
+        static bool Unmarshal(UnmarshalContext& mc, MarshalerType& marshaler, DataType& sourceValue, AZStd::true_type /*isPointerType*/)
+        {
+            // Expects a return of whether or not the value was changed.
+            return marshaler.UnmarshalToPointer(sourceValue, (*mc.m_iBuf));
+        }
+    };
+
+    /**
+        Declares a networked DataSet of type DataType. Optionally pass in a marshaler that
+        can write the data to a stream. Otherwise the DataSet will expect to find a
+        ForwardMarshaler specialized on type DataType. Optionally pass in a throttler
+        that can decide when the data has changed enough to send to the downstream proxies.
+    **/
+    template<typename DataType, typename MarshalerType = Marshaler<DataType>, typename ThrottlerType = BasicThrottle<DataType>>
+    class DataSet
+        : public DataSetBase
+    {
+    public:
+        template<class C, void (C::* FuncPtr)(const DataType&, const TimeContext&), typename CallbackTraits = DataSetDefaultTraits>
+        class BindInterface;
+
+        template<class C, void (C::* FuncPtr)(const DataType&, const TimeContext&)>
+        class BindOverrideInterface;
+
+        struct StampedBuffer
+        {
+            AZ::u64                                 m_stamp;  ///< Counter stamp
+            AZStd::shared_ptr<WriteBufferDynamic>   m_buffer; ///< Marshalled value
+        };
+        //AZStd::ring_buffer<StampedBuffer>           m_values;       ///< History of values
+
+        /**
+            Constructs a DataSet.
+        **/
+        DataSet(const char* debugName, const DataType& value = DataType(), const MarshalerType& marshaler = MarshalerType(), const ThrottlerType& throttler = ThrottlerType())
+            : DataSetBase(debugName)
+            , m_value(value)
+            , m_throttler(throttler)
+            , m_marshaler(marshaler)
+            , m_idleTicks(-1.f)
+        {
+            m_throttler.UpdateBaseline(value);
+        }
+
+        /**
+            Modify the DataSet. Call this on the Master node to change the data,
+            which will be propagated to all proxies.
+        **/
+        void Set(const DataType& v)
+        {
+            if (CanSet())
+            {
+                m_value = v;
+                m_isDefaultValue = false;
+                SetDirty();
+            }
+        }
+
+        /**
+            Modify the DataSet. Call this on the Master node to change the data,
+            which will be propagated to all proxies.
+        **/
+        void Set(DataType&& v)
+        {
+            if (CanSet())
+            {
+                m_value = v;
+                m_isDefaultValue = false;
+                SetDirty();
+            }
+        }
+
+        /**
+            Modify the DataSet. Call this on the Master node to change the data,
+            which will be propagated to all proxies.
+        **/
+        template <class ... Args>
+        void SetEmplace(Args&& ... args)
+        {
+            if (CanSet())
+            {
+                m_value = DataType(AZStd::forward<Args>(args) ...);
+                m_isDefaultValue = false;
+                SetDirty();
+            }
+        }
+
+        /**
+            Modify the DataSet directly without copying it. Call this on the Master node,
+            passing in a function object that takes the value by reference, optionally
+            modifies the data, and returns true if the data was changed.
+        **/
+        template<typename FuncPtr>
+        bool Modify(FuncPtr func)
+        {
+            static_assert(AZStd::is_same<decltype(func(m_value)), bool>::value, "Function object must return dirty status");
+
+            bool dirty = false;
+            if (CanSet())
+            {
+                dirty = func(m_value);
+                if (dirty)
+                {
+                    m_isDefaultValue = false;
+                    SetDirty();
+                }
+            }
+            return dirty;
+        }
+
+        /**
+            Returns the current value of the DataSet.
+        **/
+        const DataType& Get() const { return m_value; }
+
+        /**
+            Returns the marshaler instance.
+        **/
+        MarshalerType& GetMarshaler() { return m_marshaler; }
+
+        /**
+            Returns the throttler instance.
+        **/
+        ThrottlerType& GetThrottler() { return m_throttler; }
+
+        //@{
+        /**
+            Returns equality for values of the same type
+        **/
+        bool operator==(const DataType& other) const { return m_value == other; }
+        bool operator==(const DataSet& other) const { return m_value == other.m_value; }
+
+        bool operator!=(const DataType& other) const { return !(m_value == other); }
+        bool operator!=(const DataSet& other) const { return !(m_value == other.m_value); }
+        //@}
+
+    protected:
+        DataSet(const DataSet& rhs) = delete;
+        DataSet& operator=(const DataSet&) = delete;
+
+        void SetDirty() override
+        {
+            if (!IsWithinToleranceThreshold())
+            {
+                DataSetBase::SetDirty();
+            }
+        }
+
+        PrepareDataResult PrepareData(EndianType endianType, AZ::u32 marshalFlags) override
+        {
+            PrepareDataResult pdr(false, false, false, false);
+
+            if (ReplicaTarget::IsAckEnabled())
+            {
+                if (!IsWithinToleranceThreshold())
+                {
+                    m_isDefaultValue = false;
+                    pdr.m_isDownstreamUnreliableDirty = true;
+
+                    m_throttler.UpdateBaseline(m_value);
+                    m_streamCache.Clear();
+                    m_streamCache.SetEndianType(endianType);
+                    m_streamCache.Write(m_value, m_marshaler);
+
+                    if (m_replicaChunk && m_replicaChunk->m_replica)    //If this data set is attached to a replica
+                    {
+                        auto revision = m_replicaChunk->m_replica->GetRevision() + 1;
+                        //m_values.push_back(StampedBuffer{ revision, AZStd::make_shared<WriteBufferDynamic>(m_streamCache) });
+                        AZ_Assert(m_replicaChunk->m_revision <= revision, "Replica Chunk out of sync with replica chnk %d replica+1 %d"
+                            , m_replicaChunk->m_revision, revision);
+                        m_replicaChunk->m_revision = m_revision = revision;
+                    }
+                }
+                else if ((marshalFlags & ReplicaMarshalFlags::ForceDirty)
+                    || (marshalFlags & ReplicaMarshalFlags::OmitUnmodified)
+                    || (m_isDefaultValue && m_streamCache.Size() == 0)
+                    )
+                {
+                    /*
+                     * If the dataset is not dirty but the current operation is forcing dirty then
+                     * we need to prepare the stream cache by writing the current value in,
+                     * otherwise, the marshalling logic will send nothing or an out of date value.
+                     *
+                     * This can occur with NewOwner command, for example.
+                     */
+
+                    m_streamCache.Clear();
+                    m_streamCache.SetEndianType(endianType);
+                    m_streamCache.Write(m_value, m_marshaler);
+                }
+            }
+            else
+            {
+                bool isDirty = false;
+                if (!IsWithinToleranceThreshold())
+                {
+                    m_isDefaultValue = false;
+
+                    isDirty = true;
+                    m_idleTicks = 0.f;
+                }
+                else if (m_idleTicks < m_maxIdleTicks)
+                {
+                    /*
+                    * This logic sends updates unreliable for some time and then sends reliable update at the end.
+                    * However, this is not necessary in the case of a value that is still a default one,
+                    * since the new proxy event (that occurs prior) is always sent reliably.
+                    */
+                    if (!m_isDefaultValue)
+                    {
+                        isDirty = true;
+                    }
+
+                    m_idleTicks += 1.f;
+                }
+
+                if (isDirty)
+                {
+                    m_throttler.UpdateBaseline(m_value);
+                    m_streamCache.Clear();
+                    m_streamCache.SetEndianType(endianType);
+                    m_streamCache.Write(m_value, m_marshaler);
+
+                    if (m_idleTicks >= m_maxIdleTicks)
+                    {
+                        pdr.m_isDownstreamReliableDirty = true;
+                    }
+                    else
+                    {
+                        pdr.m_isDownstreamUnreliableDirty = true;
+                    }
+                }
+                else if ((marshalFlags & ReplicaMarshalFlags::ForceDirty) || (marshalFlags & ReplicaMarshalFlags::OmitUnmodified))
+                {
+                    /*
+                    * If the dataset is not dirty but the current operation is forcing dirty then
+                    * we need to prepare the stream cache by writing the current value in,
+                    * otherwise, the marshalling logic will send nothing or an out of date value.
+                    *
+                    * This can occur with NewOwner command, for example.
+                    */
+
+                    m_streamCache.Clear();
+                    m_streamCache.SetEndianType(endianType);
+                    m_streamCache.Write(m_value, m_marshaler);
+                }
+            }
+
+            return pdr;
+        }
+
+        void Unmarshal(UnmarshalContext& mc) override
+        {
+            if (MarshalerShim::Unmarshal(mc, m_marshaler, m_value, typename AZStd::is_pointer<DataType>::type()))
+            {
+                m_lastUpdateTime = mc.m_timestamp;
+                m_replicaChunk->AddDataSetEvent(this);
+            }
+        }
+
+        void ResetDirty() override
+        {
+            m_idleTicks = m_maxIdleTicks;
+        }
+
+        bool IsWithinToleranceThreshold()
+        {
+            return m_throttler.WithinThreshold(m_value);
+        }
+
+        void DispatchChangedEvent(const TimeContext& tc) override
+        {
+            if (m_override)
+            {
+                /*
+                 * m_override is a AZStd::function<>, so its use can incur a cost.
+                 * However, its use is limited to Delta Compressed DataSets.
+                 * This @DispatchChangedEvent is only called on pure DataSet<>,
+                 * as opposed DataSet<>::BindInterface<> for regular DataSet usage.
+                 * And in the cases when m_override wasn't specified,
+                 * it's a simple "if (false)" pass-through.
+                 */
+                m_override(tc);
+            }
+        }
+
+        DataType m_value;
+        ThrottlerType m_throttler;
+        MarshalerType m_marshaler;
+        float m_idleTicks;
+    };
+    //-----------------------------------------------------------------------------
+
+    /**
+        Declares a DataSet with an event handler that is called when the DataSet is changed.
+        Use BindInterface<Class, FuncPtr> to dispatch to a method on the ReplicaChunk's
+        ReplicaChunkInterface event handler instance.
+    **/
+    template<typename DataType, typename MarshalerType, typename ThrottlerType>
+    template<class C, void (C::* FuncPtr)(const DataType&, const TimeContext&), typename CallbackTraits>
+    class DataSet<DataType, MarshalerType, ThrottlerType>::BindInterface
+        : public DataSet<DataType, MarshalerType, ThrottlerType>
+    {
+    public:
+        BindInterface(const char* debugName,
+            const DataType& value = DataType(),
+            const MarshalerType& marshaler = MarshalerType(),
+            const ThrottlerType& throttler = ThrottlerType())
+            : DataSet(debugName,
+                value,
+                marshaler,
+                throttler)
+        { }
+
+        void SetDirty() override
+        {
+            DataSet::SetDirty();
+
+            if (CallbackTraits::s_invokeAuthoritativeCallback)
+            {
+                DispatchChangedEvent({});
+            }
+        }
+
+    protected:
+        void DispatchChangedEvent(const TimeContext& tc) override
+        {
+            C* c = m_replicaChunk ? static_cast<C*>(m_replicaChunk->GetHandler()) : nullptr;
+            if (c)
+            {
+                TimeContext changeTime;
+                changeTime.m_realTime = m_lastUpdateTime;
+                changeTime.m_localTime = m_lastUpdateTime - (tc.m_realTime - tc.m_localTime);
+
+                (*c.*FuncPtr)(m_value, changeTime);
+            }
+        }
+    };
+
+    /**
+    * CtorContextBase
+    */
+    class CtorContextBase
+    {
+        friend class CtorDataSetBase;
+
+        static CtorContextBase* s_pCur;
+
+    protected:
+        //-----------------------------------------------------------------------------
+        class CtorDataSetBase
+        {
+        public:
+            CtorDataSetBase();
+            virtual ~CtorDataSetBase() { }
+
+            virtual void Marshal(WriteBuffer& wb) = 0;
+            virtual void Unmarshal(ReadBuffer& rb) = 0;
+        };
+        //-----------------------------------------------------------------------------
+        template<typename DataType, typename MarshalerType = Marshaler<DataType> >
+        class CtorDataSet
+            : public CtorDataSetBase
+        {
+            MarshalerType m_marshaler;
+            DataType m_value;
+
+        public:
+            CtorDataSet(const MarshalerType& marshaler = MarshalerType())
+                : m_marshaler(marshaler) { }
+
+            void Set(const DataType& val) { m_value = val; }
+            const DataType& Get() const { return m_value; }
+
+            virtual void Marshal(WriteBuffer& wb)
+            {
+                wb.Write(m_value, m_marshaler);
+            }
+
+            virtual void Unmarshal(ReadBuffer& rb)
+            {
+                rb.Read(m_value, m_marshaler);
+            }
+        };
+        //-----------------------------------------------------------------------------
+
+    private:
+        typedef vector<CtorDataSetBase*> MembersArrayType;
+        MembersArrayType m_members;
+
+    public:
+        CtorContextBase();
+
+        void Marshal(WriteBuffer& wb);
+        void Unmarshal(ReadBuffer& rb);
+    };
+    //-----------------------------------------------------------------------------
+} // namespace GridMate
+
+#endif // GM_DATASET_H

Code/Framework/GridMate/GridMate/Replica/DeltaCompressedDataSet.h

@@ -0,0 +1,261 @@
+/*
+* All or portions of this file Copyright (c) Amazon.com, Inc. or its affiliates or
+* its licensors.
+*
+* For complete copyright and license terms please see the LICENSE at the root of this
+* distribution (the "License"). All use of this software is governed by the License,
+* or, if provided, by the license below or the license accompanying this file. Do not
+* remove or modify any license notices. This file is distributed on an "AS IS" BASIS,
+* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+*
+*/
+#ifndef GM_DELTACOMPRESSED_DATASET_H
+#define GM_DELTACOMPRESSED_DATASET_H
+
+#pragma once
+
+#include <AzCore/Math/Vector3.h>
+#include <GridMate/Serialize/DataMarshal.h>
+#include <GridMate/Replica/DataSet.h>
+
+namespace GridMate
+{
+    namespace Helper
+    {
+        template<AZ::u32 DeltaRange>
+        AZ::u8 GetQuantized(float value)
+        {
+            /*
+             * Quantizing into a single byte, thus 255 values.
+             * [-DeltaRange           V    +DeltaRange]
+             * [0                     Q            255]
+             * Given V, solve for Q.
+             */
+            const float quantized = (value + DeltaRange) * 255.f / (2.f * DeltaRange);
+            const int clamped = AZ::GetClamp(static_cast<int>(quantized), 0, 255);
+            return static_cast<AZ::u8>(clamped);
+        }
+
+        template<AZ::u32 DeltaRange>
+        float GetUnquantized(AZ::u8 quantized)
+        {
+            /*
+             * Unquantizing from a single byte, out of 255 values.
+             * [0                     Q            255]
+             * [-DeltaRange           V    +DeltaRange]
+             * Given Q, solve for V.
+             */
+            return 2 * DeltaRange * quantized / 255.f - DeltaRange;
+        }
+
+        template<typename FieldType>
+        struct DeltaHelper;
+
+        /**
+         * \brief Works for integer and floating points numbers
+         */
+        template<typename FieldType>
+        struct DeltaHelper
+        {
+            static bool IsWithinDelta(const FieldType& base, const FieldType& another, AZ::u32 deltaRange)
+            {
+                return abs(base - another) < deltaRange;
+            }
+        };
+
+        /**
+         * \brief Specialization for AZ::Vector3
+         */
+        template<>
+        struct DeltaHelper<AZ::Vector3>
+        {
+            static bool IsWithinDelta(const AZ::Vector3& base, const AZ::Vector3& another, AZ::u32 deltaRange)
+            {
+                const AZ::Vector3 absDiff = (base - another).GetAbs();
+                return absDiff.GetX() < deltaRange && absDiff.GetY() < deltaRange && absDiff.GetZ() < deltaRange;
+            }
+        };
+    }
+
+    /**
+     * \brief Packing a value into a single byte within +/- @DeltaRange
+     */
+    template<AZ::u32 DeltaRange, typename FieldType>
+    class DeltaMarshaller;
+
+    // float specialization
+    template<AZ::u32 DeltaRange>
+    class DeltaMarshaller<DeltaRange, float>
+    {
+    public:
+        void Marshal(WriteBuffer& wb, const float &value)
+        {
+            wb.Write(Helper::GetQuantized<DeltaRange>(value));
+        }
+
+        void Unmarshal(float& value, ReadBuffer &rb)
+        {
+            AZ::u8 delta;
+            rb.Read(delta);
+            value = Helper::GetUnquantized<DeltaRange>(delta);
+        }
+    };
+
+    // AZ::Vector3 specialization
+    template<AZ::u32 DeltaRange>
+    class DeltaMarshaller<DeltaRange, AZ::Vector3>
+    {
+    public:
+        void Marshal(WriteBuffer& wb, const AZ::Vector3& value)
+        {
+            wb.Write(Helper::GetQuantized<DeltaRange>(value.GetX()));
+            wb.Write(Helper::GetQuantized<DeltaRange>(value.GetY()));
+            wb.Write(Helper::GetQuantized<DeltaRange>(value.GetZ()));
+        }
+
+        void Unmarshal(AZ::Vector3& value, ReadBuffer& rb)
+        {
+            AZ::u8 delta[3];
+            rb.Read(delta[0]);
+            rb.Read(delta[1]);
+            rb.Read(delta[2]);
+
+            value = AZ::Vector3(Helper::GetUnquantized<DeltaRange>(delta[0]), Helper::GetUnquantized<DeltaRange>(delta[1]), Helper::GetUnquantized<DeltaRange>(delta[2]));
+        }
+    };
+
+    /**
+     * \brief Delta compressed DataSet, stateless and cacheless. Stateless - because it does not keep per-player state of any kind.
+     * Cacheless - because it does not keep a history of its values.
+     * This approach requires only one extra copy of a field, because the field is split into two portions: absolute and relative portions.
+     * The value is always the sum of two portions. We leverage existing DataSets to omit sending the larger absolute value, thus achieving compression.
+     *
+     * \tparam FieldType
+     * \tparam DeltaRange
+     * \tparam MarshalerType
+     * \tparam DeltaMarshalerType
+     */
+    template<typename FieldType, AZ::u32 DeltaRange, typename MarshalerType = Marshaler<FieldType>, typename DeltaMarshalerType = DeltaMarshaller<DeltaRange, FieldType>>
+    class DeltaCompressedDataSet
+    {
+    public:
+        virtual ~DeltaCompressedDataSet() = default;
+
+        template<class C, void (C::* FuncPtr)(const FieldType&, const TimeContext&)>
+        class BindInterface;
+
+        /**
+            Constructs a DataSet.
+        **/
+        explicit DeltaCompressedDataSet(const char* debugName, const FieldType& value = FieldType())
+            : m_absolutePortion(debugName, value)
+            , m_relativePortion(debugName)
+        {
+            static_assert(DeltaRange > 0, "Delta range cannot be zero!");
+
+            // We need to intercept changes to our two DataSets, in order to calculate the combined value and report back to Replica Chunk on our time.
+            m_absolutePortion.SetDispatchOverride([this](const TimeContext& tc) {OnAbsolutePortionChanged(tc); });
+            m_relativePortion.SetDispatchOverride([this](const TimeContext& tc) {OnRelativePortionChanged(tc); });
+        }
+
+        /**
+            Modify the DataSet. Call this on the Master node to change the data,
+            which will be propagated to all proxies.
+        **/
+        void Set(const FieldType& v)
+        {
+            m_combinedValue = v;
+
+            if (Helper::DeltaHelper<FieldType>::IsWithinDelta(m_absolutePortion.Get(), v, DeltaRange))
+            {
+                // within bounds, so only the relative portion needs to be updated
+                m_relativePortion.Set(v - m_absolutePortion.Get());
+            }
+            else
+            {
+                // relative out of range, reset absolute
+                m_absolutePortion.Set(v);
+                m_relativePortion.Set(static_cast<FieldType>(0));
+            }
+        }
+
+        /**
+            Returns the current value of the DataSet.
+        **/
+        const FieldType& Get() const
+        {
+            return m_combinedValue;
+        }
+
+    protected:
+        virtual void OnAbsolutePortionChanged(const TimeContext& /*tc*/)
+        {
+            m_combinedValue = m_absolutePortion.Get() + m_relativePortion.Get();
+        }
+
+        virtual void OnRelativePortionChanged(const TimeContext& /*tc*/)
+        {
+            m_combinedValue = m_absolutePortion.Get() + m_relativePortion.Get();
+        }
+
+    private:
+        DataSet<FieldType, MarshalerType> m_absolutePortion;
+        DataSet<FieldType, DeltaMarshalerType> m_relativePortion;
+        FieldType m_combinedValue; // the latest value on either master or proxy
+    };
+
+    //-----------------------------------------------------------------------------
+
+    /**
+        Declares a DeltaCompressedDataSet with an event handler that is called when the value is changed.
+        Use BindInterface<Class, FuncPtr> to dispatch to a method on the ReplicaChunk's
+        ReplicaChunkInterface event handler instance.
+    **/
+    template<typename FieldType, AZ::u32 DeltaRange, typename MarshalerType, typename DeltaMarshalerType>
+    template<class C, void (C::* FuncPtr)(const FieldType&, const TimeContext&)>
+    class DeltaCompressedDataSet<FieldType, DeltaRange, MarshalerType, DeltaMarshalerType>::BindInterface
+        : public DeltaCompressedDataSet<FieldType, DeltaRange, MarshalerType, DeltaMarshalerType>
+    {
+    public:
+        explicit BindInterface(const char* debugName) : DeltaCompressedDataSet(debugName) { }
+
+    protected:
+        void OnAbsolutePortionChanged(const GridMate::TimeContext& tc) override
+        {
+            DeltaCompressedDataSet::OnAbsolutePortionChanged(tc);
+
+            m_lastUpdateTime = m_absolutePortion.GetLastUpdateTime();
+            if (m_relativePortion.GetLastUpdateTime() < m_lastUpdateTime)
+            {
+                // relative portion wasn't updated, so its callback won't be invoked this tick, therefore we need to dispatch change event now
+                DispatchChangedEvent(tc);
+            }
+        }
+
+        void OnRelativePortionChanged(const GridMate::TimeContext& tc) override
+        {
+            DeltaCompressedDataSet::OnRelativePortionChanged(tc);
+
+            m_lastUpdateTime = m_relativePortion.GetLastUpdateTime();
+            // Assuming that relative portion DataSet is dispatched after absolute portion by construction in DeltaCompressedDataSet
+            DispatchChangedEvent(tc);
+        }
+
+        void DispatchChangedEvent(const TimeContext& tc)
+        {
+            AZ_Assert(m_relativePortion.GetReplicaChunkBase(), "DataSets should be attached to replica chunks!");
+
+            if (C* c = static_cast<C*>(m_relativePortion.GetReplicaChunkBase()->GetHandler()))
+            {
+                const TimeContext changeTime{ m_lastUpdateTime, m_lastUpdateTime - (tc.m_realTime - tc.m_localTime) };
+                (*c.*FuncPtr)(Get(), changeTime);
+            }
+        }
+
+    private:
+        AZ::u32 m_lastUpdateTime = 0; // the latest update time among m_absolutePortion and m_relativePortion
+    };
+    //-----------------------------------------------------------------------------
+} // namespace GridMate
+
+#endif // GM_DELTACOMPRESSED_DATASET_H

Code/Framework/GridMate/GridMate/Replica/Interest/BitmaskInterestHandler.cpp

@@ -0,0 +1,425 @@
+/*
+* All or portions of this file Copyright (c) Amazon.com, Inc. or its affiliates or
+* its licensors.
+*
+* For complete copyright and license terms please see the LICENSE at the root of this
+* distribution (the "License"). All use of this software is governed by the License,
+* or, if provided, by the license below or the license accompanying this file. Do not
+* remove or modify any license notices. This file is distributed on an "AS IS" BASIS,
+* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+*
+*/
+
+#include <GridMate/Replica/Interest/BitmaskInterestHandler.h>
+
+#include <GridMate/Replica/Interpolators.h>
+#include <GridMate/Replica/Replica.h>
+#include <GridMate/Replica/ReplicaFunctions.h>
+#include <GridMate/Replica/ReplicaMgr.h>
+
+#include <GridMate/Replica/Interest/InterestManager.h>
+
+namespace GridMate
+{
+
+    void BitmaskInterestChunk::OnReplicaActivate(const ReplicaContext& rc)
+    {
+        m_interestHandler = static_cast<BitmaskInterestHandler*>(rc.m_rm->GetUserContext(AZ_CRC("BitmaskInterestHandler", 0x5bf5d75b)));
+        AZ_Warning("GridMate", m_interestHandler != nullptr, "No bitmask interest handler in the user context");
+        if (m_interestHandler)
+        {
+            m_interestHandler->OnNewRulesChunk(this, rc.m_peer);
+        }
+    }
+
+    void BitmaskInterestChunk::OnReplicaDeactivate(const ReplicaContext& rc)
+    {
+        if (m_interestHandler)
+        {
+            // even if rc.m_peer is null, we still need to call OnDeleteRulesChunk so that the interest handler can clear m_rulesReplica
+            m_interestHandler->OnDeleteRulesChunk(this, rc.m_peer);
+        }
+    }
+
+    bool BitmaskInterestChunk::AddRuleFn(RuleNetworkId netId, InterestBitmask bits, const RpcContext& ctx)
+    {
+        if (IsProxy())
+        {
+            auto rulePtr = m_interestHandler->CreateRule(ctx.m_sourcePeer);
+            rulePtr->Set(bits);
+            m_rules.insert(AZStd::make_pair(netId, rulePtr));
+        }
+
+        return true;
+    }
+
+    bool BitmaskInterestChunk::RemoveRuleFn(RuleNetworkId netId, const RpcContext&)
+    {
+        if (IsProxy())
+        {
+            m_rules.erase(netId);
+        }
+
+        return true;
+    }
+
+    bool BitmaskInterestChunk::UpdateRuleFn(RuleNetworkId netId, InterestBitmask bits, const RpcContext&)
+    {
+        if (IsProxy())
+        {
+            auto it = m_rules.find(netId);
+            if (it != m_rules.end())
+            {
+                it->second->Set(bits);
+            }
+        }
+
+        return true;
+    }
+
+    bool BitmaskInterestChunk::AddRuleForPeerFn(RuleNetworkId netId, PeerId peerId, InterestBitmask bitmask, const RpcContext&)
+    {
+        BitmaskInterestChunk::Ptr peerChunk = m_interestHandler->FindRulesChunkByPeerId(peerId);
+        if (peerChunk)
+        {
+            auto it = peerChunk->m_rules.find(netId);
+            if (it == peerChunk->m_rules.end())
+            {
+                auto rulePtr = m_interestHandler->CreateRule(peerId);
+                peerChunk->m_rules.insert(AZStd::make_pair(netId, rulePtr));
+                rulePtr->Set(bitmask);
+            }
+        }
+        return false;
+    }
+    ///////////////////////////////////////////////////////////////////////////
+
+
+    /*
+    * BitmaskInterest
+    */
+    BitmaskInterest::BitmaskInterest(BitmaskInterestHandler* handler)
+        : m_handler(handler)
+        , m_bits(0)
+    {
+        AZ_Assert(m_handler, "Invalid interest handler");
+    }
+    ///////////////////////////////////////////////////////////////////////////
+
+
+    /*
+    * BitmaskInterestRule
+    */
+    void BitmaskInterestRule::Set(InterestBitmask newBitmask)
+    {
+        m_bits = newBitmask;
+        m_handler->UpdateRule(this);
+    }
+
+    void BitmaskInterestRule::Destroy()
+    {
+        m_handler->DestroyRule(this);
+    }
+    ///////////////////////////////////////////////////////////////////////////
+
+
+    /*
+    * BitmaskInterestAttribute
+    */
+    void BitmaskInterestAttribute::Set(InterestBitmask newBitmask)
+    {
+        m_bits = newBitmask;
+        m_handler->UpdateAttribute(this);
+    }
+
+    void BitmaskInterestAttribute::Destroy()
+    {
+        m_handler->DestroyAttribute(this);
+    }
+    ///////////////////////////////////////////////////////////////////////////
+
+
+    /*
+    * BitmaskInterestHandler
+    */
+    BitmaskInterestHandler::BitmaskInterestHandler()
+        : m_im(nullptr)
+        , m_rm(nullptr)
+        , m_lastRuleNetId(0)
+        , m_rulesReplica(nullptr)
+    {
+    }
+
+    BitmaskInterestRule::Ptr BitmaskInterestHandler::CreateRule(PeerId peerId)
+    {
+        BitmaskInterestRule* rulePtr = aznew BitmaskInterestRule(this, peerId, GetNewRuleNetId());
+        m_rules.insert(rulePtr);
+
+        if (peerId == m_rm->GetLocalPeerId() && m_rulesReplica)
+        {
+            m_rulesReplica->AddRuleRpc(rulePtr->GetNetworkId(), rulePtr->Get());
+            m_localRules.insert(rulePtr);
+        }
+
+        return rulePtr;
+    }
+
+    void BitmaskInterestHandler::FreeRule(BitmaskInterestRule* rule)
+    {
+        //TODO: should be pool-allocated
+        m_rules.erase(rule);
+        delete rule;
+    }
+
+    void BitmaskInterestHandler::DestroyRule(BitmaskInterestRule* rule)
+    {
+        if (m_rm && rule->GetPeerId() == m_rm->GetLocalPeerId() && m_rulesReplica)
+        {
+            m_rulesReplica->RemoveRuleRpc(rule->GetNetworkId());
+        }
+
+        rule->m_bits = 0;
+        m_dirtyRules.insert(rule);
+        m_localRules.erase(rule);
+    }
+
+    void BitmaskInterestHandler::UpdateRule(BitmaskInterestRule* rule)
+    {
+        if (m_rm && m_rulesReplica && rule->GetPeerId() == m_rm->GetLocalPeerId())
+        {
+            m_rulesReplica->UpdateRuleRpc(rule->GetNetworkId(), rule->Get());
+        }
+
+        m_dirtyRules.insert(rule);
+    }
+
+    BitmaskInterestAttribute::Ptr BitmaskInterestHandler::CreateAttribute(ReplicaId replicaId)
+    {
+        auto ptr = aznew BitmaskInterestAttribute(this, replicaId);
+        m_attrs.insert(ptr);
+        return ptr;
+    }
+
+    void BitmaskInterestHandler::FreeAttribute(BitmaskInterestAttribute* attrib)
+    {
+        //TODO: should be pool-allocated
+        m_attrs.erase(attrib);
+        delete attrib;
+    }
+
+    void BitmaskInterestHandler::DestroyAttribute(BitmaskInterestAttribute* attrib)
+    {
+        attrib->m_bits = 0;
+        m_dirtyAttributes.insert(attrib);
+    }
+
+    void BitmaskInterestHandler::UpdateAttribute(BitmaskInterestAttribute* attrib)
+    {
+        m_dirtyAttributes.insert(attrib);
+    }
+
+    void BitmaskInterestHandler::OnNewRulesChunk(BitmaskInterestChunk::Ptr chunk, ReplicaPeer* peer)
+    {
+        if (chunk != m_rulesReplica) // non-local
+        {
+            m_peerChunks.insert(AZStd::make_pair(peer->GetId(), chunk));
+
+            for (auto& rule : m_localRules)
+            {
+                chunk->AddRuleForPeerRpc(rule->GetNetworkId(), rule->GetPeerId(), rule->Get());
+            }
+        }
+    }
+
+    void BitmaskInterestHandler::OnDeleteRulesChunk(BitmaskInterestChunk::Ptr chunk, ReplicaPeer* peer)
+    {
+        AZ_UNUSED(chunk);
+        m_rulesReplica = nullptr;
+
+        if (peer)
+        {
+            m_peerChunks.erase(peer->GetId());
+        }
+    }
+
+    RuleNetworkId BitmaskInterestHandler::GetNewRuleNetId()
+    {
+        ++m_lastRuleNetId;
+        if (m_rulesReplica)
+        {
+            return m_rulesReplica->GetReplicaId() | (static_cast<AZ::u64>(m_lastRuleNetId) << 32);
+        }
+
+        return (static_cast<AZ::u64>(m_lastRuleNetId) << 32);
+    }
+
+    BitmaskInterestChunk::Ptr BitmaskInterestHandler::FindRulesChunkByPeerId(PeerId peerId)
+    {
+        auto it = m_peerChunks.find(peerId);
+        if (it == m_peerChunks.end())
+        {
+            return nullptr;
+        }
+        else
+        {
+            return it->second;
+        }
+    }
+
+    const InterestMatchResult& BitmaskInterestHandler::GetLastResult()
+    {
+        return m_resultCache;
+    }
+
+    void BitmaskInterestHandler::Update()
+    {
+        m_resultCache.clear();
+
+        for (BitmaskInterestRule* rule : m_dirtyRules)
+        {
+            InterestBitmask j = 1;
+            for (size_t i = 0; i < k_numGroups; ++i, j <<= 1)
+            {
+                auto ruleIt = m_ruleGroups[i].find(rule);
+                bool isMatch = !!(rule->m_bits & j);
+                if (isMatch && ruleIt == m_ruleGroups[i].end())
+                {
+                    m_ruleGroups[i].insert(rule);
+
+                    // recalculate all the attributes in this bucket
+                    for (BitmaskInterestAttribute* attr : m_attrGroups[i])
+                    {
+                        m_dirtyAttributes.insert(attr);
+                    }
+                }
+                else if (!isMatch && ruleIt != m_ruleGroups[i].end())
+                {
+                    m_ruleGroups[i].erase(ruleIt);
+
+                    // recalculate all the attributes in this bucket
+                    for (BitmaskInterestAttribute* attr : m_attrGroups[i])
+                    {
+                        m_dirtyAttributes.insert(attr);
+                    }
+                }
+            }
+
+            if (rule->IsDeleted())
+            {
+                FreeRule(rule);
+            }
+        }
+
+        m_dirtyRules.clear();
+
+        for (BitmaskInterestAttribute* attr : m_dirtyAttributes)
+        {
+            InterestBitmask j = 1;
+            for (size_t i = 0; i < k_numGroups; ++i, j <<= 1)
+            {
+                auto attrIt = m_attrGroups[i].find(attr);
+                bool isMatch = !!(attr->m_bits & j);
+                if (isMatch && attrIt == m_attrGroups[i].end())
+                {
+                    m_attrGroups[i].insert(attr);
+                }
+                else if (!isMatch && attrIt != m_attrGroups[i].end())
+                {
+                    m_attrGroups[i].erase(attrIt);
+                }
+            }
+        }
+
+        for (BitmaskInterestAttribute* attr : m_dirtyAttributes)
+        {
+            auto repIt = m_resultCache.insert(attr->GetReplicaId());
+
+            InterestBitmask j = 1;
+            for (size_t i = 0; i < k_numGroups; ++i, j <<= 1)
+            {
+                if (!!(attr->m_bits & j))
+                {
+                    for (BitmaskInterestRule* rule : m_ruleGroups[i])
+                    {
+                        repIt.first->second.insert(rule->GetPeerId());
+                    }
+                }
+            }
+
+            if (attr->IsDeleted())
+            {
+                FreeAttribute(attr);
+            }
+        }
+
+        m_dirtyAttributes.clear();
+    }
+
+    void BitmaskInterestHandler::OnRulesHandlerRegistered(InterestManager* manager)
+    {
+        AZ_Assert(m_im == nullptr, "Handler is already registered with manager %p (%p)\n", m_im, manager);
+        AZ_Assert(m_rulesReplica == nullptr, "Rules replica is already created\n");
+        AZ_TracePrintf("GridMate", "Bitmask interest handler is registered\n");
+        m_im = manager;
+        m_rm = m_im->GetReplicaManager();
+        m_rm->RegisterUserContext(AZ_CRC("BitmaskInterestHandler", 0x5bf5d75b), this);
+
+        auto replica = Replica::CreateReplica("BitmaskInterestHandlerRules");
+        m_rulesReplica = CreateAndAttachReplicaChunk<BitmaskInterestChunk>(replica);
+        m_rm->AddMaster(replica);
+    }
+
+    void BitmaskInterestHandler::OnRulesHandlerUnregistered(InterestManager* manager)
+    {
+        (void)manager;
+
+        AZ_Assert(m_im == manager, "Handler was not registered with manager %p (%p)\n", manager, m_im);
+        AZ_TracePrintf("GridMate", "Bitmask interest handler is unregistered\n");
+
+        if (m_rulesReplica)
+        {
+            m_rulesReplica->m_rules.clear();
+            m_rulesReplica->m_interestHandler = nullptr;
+        }
+
+        for (auto& chunk : m_peerChunks)
+        {
+            chunk.second->m_rules.clear();
+            chunk.second->m_interestHandler = nullptr;
+        }
+
+        m_rulesReplica = nullptr;
+        m_im = nullptr;
+        m_rm->UnregisterUserContext(AZ_CRC("BitmaskInterestHandler", 0x5bf5d75b));
+        m_rm = nullptr;
+
+        m_peerChunks.clear();
+        m_localRules.clear();
+
+        for (auto& a : m_attrs)
+        {
+            delete a;
+        }
+
+        for (auto& r : m_rules)
+        {
+            delete r;
+        }
+
+        m_dirtyAttributes.clear();
+        m_dirtyRules.clear();
+
+        for (auto& group : m_attrGroups)
+        {
+            group.clear();
+        }
+
+        for (auto& group : m_ruleGroups)
+        {
+            group.clear();
+        }
+
+        m_resultCache.clear();
+    }
+    ///////////////////////////////////////////////////////////////////////////
+}

Code/Framework/GridMate/GridMate/Replica/Interest/BitmaskInterestHandler.h

@@ -0,0 +1,241 @@
+/*
+* All or portions of this file Copyright (c) Amazon.com, Inc. or its affiliates or
+* its licensors.
+*
+* For complete copyright and license terms please see the LICENSE at the root of this
+* distribution (the "License"). All use of this software is governed by the License,
+* or, if provided, by the license below or the license accompanying this file. Do not
+* remove or modify any license notices. This file is distributed on an "AS IS" BASIS,
+* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+*
+*/
+
+#ifndef GM_REPLICA_BITMASKINTERESTHANDLER_H
+#define GM_REPLICA_BITMASKINTERESTHANDLER_H
+
+#include <GridMate/Replica/RemoteProcedureCall.h>
+#include <GridMate/Replica/ReplicaChunk.h>
+#include <GridMate/Replica/Interest/RulesHandler.h>
+#include <GridMate/Serialize/UtilityMarshal.h>
+
+#include <GridMate/Containers/vector.h>
+#include <GridMate/Containers/unordered_set.h>
+#include <AzCore/std/containers/array.h>
+
+
+namespace GridMate
+{
+    class BitmaskInterestHandler;
+    using InterestBitmask = AZ::u32;
+
+    /*
+     * Base interest
+    */
+    class BitmaskInterest
+    {
+        friend class BitmaskInterestHandler;
+
+    public:
+        InterestBitmask Get() const { return m_bits; }
+
+    protected:
+        explicit BitmaskInterest(BitmaskInterestHandler* handler);
+
+        BitmaskInterestHandler* m_handler;
+        InterestBitmask m_bits;
+    };
+    ///////////////////////////////////////////////////////////////////////////
+
+
+    /*
+    * Bitmask rule
+    */
+    class BitmaskInterestRule 
+        : public InterestRule
+        , public BitmaskInterest
+    {
+        friend class BitmaskInterestHandler;
+
+    public:
+        using Ptr = AZStd::intrusive_ptr<BitmaskInterestRule>;
+
+        GM_CLASS_ALLOCATOR(BitmaskInterestRule);
+
+        void Set(InterestBitmask newBitmask);
+
+    private:
+
+        // Intrusive ptr 
+        template<class T>
+        friend struct AZStd::IntrusivePtrCountPolicy;
+        unsigned int m_refCount = 0;
+        AZ_FORCE_INLINE void add_ref() { ++m_refCount; }
+        AZ_FORCE_INLINE void release() { --m_refCount; if (!m_refCount) Destroy(); }
+        AZ_FORCE_INLINE bool IsDeleted() const { return m_refCount == 0; }
+        ///////////////////////////////////////////////////////////////////////////
+
+        BitmaskInterestRule(BitmaskInterestHandler* handler, PeerId peerId, RuleNetworkId netId)
+            : InterestRule(peerId, netId)
+            , BitmaskInterest(handler)
+        {}
+
+        void Destroy();
+    };
+    ///////////////////////////////////////////////////////////////////////////
+
+
+    /*
+    * Bitmask attribute
+    */
+    class BitmaskInterestAttribute
+        : public InterestAttribute
+        , public BitmaskInterest
+    {
+        friend class BitmaskInterestHandler;
+        template<class T> friend class InterestPtr;
+
+    public:
+        using Ptr = AZStd::intrusive_ptr<BitmaskInterestAttribute>;
+
+        GM_CLASS_ALLOCATOR(BitmaskInterestAttribute);
+
+        void Set(InterestBitmask newBitmask);
+
+    private:
+
+        // Intrusive ptr 
+        template<class T>
+        friend struct AZStd::IntrusivePtrCountPolicy;
+        unsigned int m_refCount = 0;
+        AZ_FORCE_INLINE void add_ref() { ++m_refCount; }
+        AZ_FORCE_INLINE void release() { Destroy(); }
+        AZ_FORCE_INLINE bool IsDeleted() const { return m_refCount == 0; }
+        ///////////////////////////////////////////////////////////////////////////
+
+        BitmaskInterestAttribute(BitmaskInterestHandler* handler, ReplicaId repId)
+            : InterestAttribute(repId)
+            , BitmaskInterest(handler)
+        {}
+
+        void Destroy();
+    };
+    ///////////////////////////////////////////////////////////////////////////
+
+    class BitmaskInterestChunk
+        : public ReplicaChunk
+    {
+    public:
+        GM_CLASS_ALLOCATOR(BitmaskInterestChunk);
+
+        BitmaskInterestChunk()
+            : AddRuleRpc("AddRule")
+            , RemoveRuleRpc("RemoveRule")
+            , UpdateRuleRpc("UpdateRule")
+            , AddRuleForPeerRpc("AddRuleForPeerRpc")
+            , m_interestHandler(nullptr)
+        {}
+
+        typedef AZStd::intrusive_ptr<BitmaskInterestChunk> Ptr;
+        bool IsReplicaMigratable() override { return false; }
+        bool IsBroadcast() { return true; }
+        static const char* GetChunkName() { return "BitmaskInterestChunk"; }
+
+        void OnReplicaActivate(const ReplicaContext& rc) override;
+        void OnReplicaDeactivate(const ReplicaContext& rc) override;
+
+        bool AddRuleFn(RuleNetworkId netId, InterestBitmask bits, const RpcContext& ctx);
+        bool RemoveRuleFn(RuleNetworkId netId, const RpcContext&);
+        bool UpdateRuleFn(RuleNetworkId netId, InterestBitmask bits, const RpcContext&);
+        bool AddRuleForPeerFn(RuleNetworkId netId, PeerId peerId, InterestBitmask bitmask, const RpcContext&);
+
+        Rpc<RpcArg<RuleNetworkId>, RpcArg<InterestBitmask>>::BindInterface<BitmaskInterestChunk, &BitmaskInterestChunk::AddRuleFn> AddRuleRpc;
+        Rpc<RpcArg<RuleNetworkId>>::BindInterface<BitmaskInterestChunk, &BitmaskInterestChunk::RemoveRuleFn> RemoveRuleRpc;
+        Rpc<RpcArg<RuleNetworkId>, RpcArg<InterestBitmask>>::BindInterface<BitmaskInterestChunk, &BitmaskInterestChunk::UpdateRuleFn> UpdateRuleRpc;
+
+        Rpc<RpcArg<RuleNetworkId>, RpcArg<PeerId>, RpcArg<InterestBitmask>>::BindInterface<BitmaskInterestChunk, &BitmaskInterestChunk::AddRuleForPeerFn> AddRuleForPeerRpc;
+
+        unordered_map<RuleNetworkId, BitmaskInterestRule::Ptr> m_rules;
+        BitmaskInterestHandler* m_interestHandler;
+    };
+
+    /*
+    * Rules handler
+    */
+    class BitmaskInterestHandler 
+        : public BaseRulesHandler
+    {
+        friend class BitmaskInterestRule;
+        friend class BitmaskInterestAttribute;
+        friend class BitmaskInterestChunk;
+
+    public:
+
+        GM_CLASS_ALLOCATOR(BitmaskInterestHandler);
+
+        BitmaskInterestHandler();
+
+        // Creates new bitmask rule and binds it to the peer
+        BitmaskInterestRule::Ptr CreateRule(PeerId peerId);
+
+        // Creates new bitmask attribute and binds it to the replica
+        BitmaskInterestAttribute::Ptr CreateAttribute(ReplicaId replicaId);
+
+        // Calculates rules and attributes matches
+        void Update() override;
+
+        // Returns last recalculated results
+        const InterestMatchResult& GetLastResult() override;
+
+        InterestManager* GetManager() override { return m_im; }
+    private:
+
+        // BaseRulesHandler
+        void OnRulesHandlerRegistered(InterestManager* manager) override;
+        void OnRulesHandlerUnregistered(InterestManager* manager) override;
+
+        void DestroyRule(BitmaskInterestRule* rule);
+        void FreeRule(BitmaskInterestRule* rule);
+        void UpdateRule(BitmaskInterestRule* rule);
+
+        void DestroyAttribute(BitmaskInterestAttribute* attrib);
+        void FreeAttribute(BitmaskInterestAttribute* attrib);
+        void UpdateAttribute(BitmaskInterestAttribute* attrib);
+
+
+        void OnNewRulesChunk(BitmaskInterestChunk::Ptr chunk, ReplicaPeer* peer);
+        void OnDeleteRulesChunk(BitmaskInterestChunk::Ptr chunk, ReplicaPeer* peer);
+
+        RuleNetworkId GetNewRuleNetId();
+
+        BitmaskInterestChunk::Ptr FindRulesChunkByPeerId(PeerId peerId);
+
+        typedef unordered_set<BitmaskInterestAttribute*> AttributeSet;
+        typedef unordered_set<BitmaskInterestRule*> RuleSet;
+        static const size_t k_numGroups = sizeof(InterestBitmask) * CHAR_BIT;
+
+        InterestManager* m_im;
+        ReplicaManager* m_rm;
+        
+        AZ::u32 m_lastRuleNetId;
+
+        unordered_map<PeerId, BitmaskInterestChunk::Ptr> m_peerChunks;
+
+        RuleSet m_localRules;
+
+        AttributeSet m_dirtyAttributes;
+        RuleSet m_dirtyRules;
+
+        AZStd::array<AttributeSet, k_numGroups> m_attrGroups;
+        AZStd::array<RuleSet, k_numGroups> m_ruleGroups;
+
+        InterestMatchResult m_resultCache;
+
+        BitmaskInterestChunk* m_rulesReplica;
+
+        AttributeSet m_attrs;
+        RuleSet m_rules;
+    };
+    ///////////////////////////////////////////////////////////////////////////
+}
+
+#endif

Code/Framework/GridMate/GridMate/Replica/Interest/BvDynamicTree.h

@@ -0,0 +1,878 @@
+/*
+* All or portions of this file Copyright (c) Amazon.com, Inc. or its affiliates, or 
+* a third party where indicated.
+*
+* For complete copyright and license terms please see the LICENSE at the root of this
+* distribution (the "License"). All use of this software is governed by the License,  
+* or, if provided, by the license below or the license accompanying this file. Do not
+* remove or modify any license notices. This file is distributed on an "AS IS" BASIS,
+* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.  
+*
+*/
+/*
+ * Temporary dynamic tree structure used internally by GridMate.
+ * To be replaced with a general Vis framework when that becomes available.
+ */
+
+#ifndef RR_DYNAMIC_BOUNDING_VOLUME_TREE_H
+#define RR_DYNAMIC_BOUNDING_VOLUME_TREE_H
+
+#include <AzCore/Math/Vector3.h>
+#include <AzCore/Math/Aabb.h>
+#include <AzCore/Math/Plane.h>
+
+#include <GridMate/Containers/vector.h>
+#include <AzCore/std/containers/fixed_vector.h>
+
+namespace GridMate
+{
+    namespace Internal
+    {
+        /**
+         *
+        */
+        class   DynamicTreeAabb : public AZ::Aabb
+        {
+        public:
+            GM_CLASS_ALLOCATOR(DynamicTreeAabb);
+
+            AZ_FORCE_INLINE explicit DynamicTreeAabb() {}
+            AZ_FORCE_INLINE DynamicTreeAabb(const AZ::Aabb& aabb) : AZ::Aabb(aabb) {}
+            AZ_FORCE_INLINE explicit DynamicTreeAabb(const AZ::Vector3& min,const AZ::Vector3& max) : AZ::Aabb(AZ::Aabb::CreateFromMinMax(min,max)) {}
+
+            AZ_FORCE_INLINE static DynamicTreeAabb  CreateFromFacePoints(const AZ::Vector3& a, const AZ::Vector3& b, const AZ::Vector3& c)
+            {
+                DynamicTreeAabb vol(a,a);
+                vol.AddPoint(b);
+                vol.AddPoint(c);
+                return vol;
+            }
+
+            AZ_FORCE_INLINE void                SignedExpand(const AZ::Vector3& e)
+            {
+                AZ::Vector3 zero = AZ::Vector3::CreateZero();
+                AZ::Vector3 mxE = m_max + e;
+                AZ::Vector3 miE = m_min + e;
+                m_max = AZ::Vector3::CreateSelectCmpGreater(e,zero,mxE,m_max );
+                m_min = AZ::Vector3::CreateSelectCmpGreater(e,zero,m_min,miE);
+            }
+            AZ_FORCE_INLINE int                 Classify(const AZ::Vector3& n,const float o,int s) const
+            {
+                AZ::Vector3 pi, px;
+                switch(s)
+                {
+                case    (0+0+0):    px=m_min;
+                                    pi=m_max; break;
+                case    (1+0+0):    px=AZ::Vector3(m_max.GetX(),m_min.GetY(),m_min.GetZ());
+                                    pi=AZ::Vector3(m_min.GetX(),m_max.GetY(),m_max.GetZ());break;
+                case    (0+2+0):    px=AZ::Vector3(m_min.GetX(),m_max.GetY(),m_min.GetZ());
+                                    pi=AZ::Vector3(m_max.GetX(),m_min.GetY(),m_max.GetZ());break;
+                case    (1+2+0):    px=AZ::Vector3(m_max.GetX(),m_max.GetY(),m_min.GetZ());
+                                    pi=AZ::Vector3(m_min.GetX(),m_min.GetY(),m_max.GetZ());break;
+                case    (0+0+4):    px=AZ::Vector3(m_min.GetX(),m_min.GetY(),m_max.GetZ());
+                                    pi=AZ::Vector3(m_max.GetX(),m_max.GetY(),m_min.GetZ());break;
+                case    (1+0+4):    px=AZ::Vector3(m_max.GetX(),m_min.GetY(),m_max.GetZ());
+                                    pi=AZ::Vector3(m_min.GetX(),m_max.GetY(),m_min.GetZ());break;
+                case    (0+2+4):    px=AZ::Vector3(m_min.GetX(),m_max.GetY(),m_max.GetZ());
+                                    pi=AZ::Vector3(m_max.GetX(),m_min.GetY(),m_min.GetZ());break;
+                case    (1+2+4):    px=m_max;
+                                    pi=m_min;break;
+                }
+
+                if (n.Dot(px) + o < 0.0f)
+                {
+                    return -1;
+                }
+                if (n.Dot(pi) + o > 0.0f)
+                {
+                    return  1;
+                }
+
+                return 0;
+            }
+            AZ_FORCE_INLINE float ProjectMinimum(const AZ::Vector3& v, unsigned signs) const
+            {
+                const AZ::Vector3*  b[]={&m_max,&m_min};
+                const AZ::Vector3   p(  b[(signs>>0)&1]->GetX(),b[(signs>>1)&1]->GetY(),b[(signs>>2)&1]->GetZ());
+                return p.Dot(v);
+            }
+
+            // Move the code here
+            AZ_FORCE_INLINE friend bool IntersectAabbAabb(const DynamicTreeAabb& a,const DynamicTreeAabb& b);
+            AZ_FORCE_INLINE friend bool IntersectAabbPoint(const DynamicTreeAabb& a, const AZ::Vector3& b);
+            AZ_FORCE_INLINE friend bool IntersectAabbPlane(const DynamicTreeAabb& a, const AZ::Plane& b);
+            AZ_FORCE_INLINE friend float Proximity(const DynamicTreeAabb& a, const DynamicTreeAabb& b);
+            AZ_FORCE_INLINE friend int Select(const DynamicTreeAabb& o, const DynamicTreeAabb& a, const DynamicTreeAabb& b);
+            AZ_FORCE_INLINE friend void Merge(const DynamicTreeAabb& a, const DynamicTreeAabb& b, DynamicTreeAabb& r);
+            AZ_FORCE_INLINE friend bool NotEqual(const DynamicTreeAabb& a, const DynamicTreeAabb& b);
+        private:
+            AZ_FORCE_INLINE void AddSpan(const AZ::Vector3& d, float& smi, float& smx) const
+            {
+                AZ::Vector3 vecZero = AZ::Vector3::CreateZero();
+                AZ::Vector3 mxD = m_max*d;
+                AZ::Vector3 miD = m_min*d;
+                AZ::Vector3 smiAdd = AZ::Vector3::CreateSelectCmpGreater(vecZero,d,mxD,miD);
+                AZ::Vector3 smxAdd = AZ::Vector3::CreateSelectCmpGreater(vecZero,d,miD,mxD);
+                AZ::Vector3 vecOne = AZ::Vector3::CreateOne();
+                // sum components
+                smi += smiAdd.Dot(vecOne);
+                smx += smxAdd.Dot(vecOne);
+            }
+        };
+
+        //
+        AZ_FORCE_INLINE bool IntersectAabbAabb(const DynamicTreeAabb& a, const DynamicTreeAabb& b)
+        {
+            return a.Overlaps(b);
+        }
+
+        AZ_FORCE_INLINE bool IntersectAabbPlane(const DynamicTreeAabb& a, const AZ::Plane& b)
+        {
+            //use plane normal to quickly select the nearest corner of the aabb
+            AZ::Vector3 testPoint = AZ::Vector3::CreateSelectCmpGreater(b.GetNormal(), AZ::Vector3::CreateZero(), a.GetMin(), a.GetMax());
+            //test if nearest point is inside the plane
+            return b.GetPointDist(testPoint) <= 0.0f;
+        }
+
+        //
+        AZ_FORCE_INLINE float Proximity(const DynamicTreeAabb& a, const DynamicTreeAabb& b)
+        {
+            const AZ::Vector3 d=(a.m_min+a.m_max)-(b.m_min+b.m_max);
+            // get abs and sum
+            return d.GetAbs().Dot(AZ::Vector3::CreateOne());
+        }
+
+        //
+        AZ_FORCE_INLINE int Select( const DynamicTreeAabb& o, const DynamicTreeAabb& a, const DynamicTreeAabb& b)
+        {
+            return Proximity(o,a) < Proximity(o,b);
+        }
+
+        //
+        AZ_FORCE_INLINE void Merge(const DynamicTreeAabb& a, const DynamicTreeAabb& b, DynamicTreeAabb& r)
+        {
+            r.m_min = AZ::Vector3::CreateSelectCmpGreater(b.m_min,a.m_min,a.m_min,b.m_min);
+            r.m_max = AZ::Vector3::CreateSelectCmpGreater(a.m_max,b.m_max,a.m_max,b.m_max);
+        }
+
+        //
+        AZ_FORCE_INLINE bool NotEqual(   const DynamicTreeAabb& a, const DynamicTreeAabb& b)
+        {
+            return (a.m_min != b.m_min || a.m_max != b.m_max);
+        }
+
+
+        /* NodeType             */
+        struct  DynamicTreeNode
+        {
+            GM_CLASS_ALLOCATOR(DynamicTreeNode);
+
+            DynamicTreeAabb         m_volume;
+            DynamicTreeNode*        m_parent;
+            AZ_FORCE_INLINE bool    IsLeaf() const      { return(m_childs[1]==0); }
+            AZ_FORCE_INLINE bool    IsInternal() const  { return(!IsLeaf()); }
+            union
+            {
+                DynamicTreeNode*    m_childs[2];
+                void*       m_data;
+                int         m_dataAsInt;
+            };
+        };
+    }
+
+    /**
+    * Implementation of dynamic aabb tree, based on the bullet dynamic tree (btDbvt).
+    *
+    * The BvDynamicTree class implements a fast dynamic bounding volume tree based on axis aligned bounding boxes (aabb tree).
+    * This BvDynamicTree is used for soft body collision detection and for the btDbvtBroadphase. It has a fast insert, remove and update of nodes.
+    * Unlike the BvTreeQuantized, nodes can be dynamically moved around, which allows for change in topology of the underlying data structure.
+    */
+    class   BvDynamicTree
+    {
+    public:
+        using Ptr = AZStd::intrusive_ptr<BvDynamicTree>;
+
+        GM_CLASS_ALLOCATOR(BvDynamicTree);
+
+        typedef Internal::DynamicTreeAabb   VolumeType;
+        typedef Internal::DynamicTreeNode   NodeType;
+
+        typedef vector<NodeType*>           NodeArrayType;
+        typedef vector<const NodeType*>     ConstNodeArrayType;
+
+    private:
+
+        /* Stack element    */
+        struct  sStkNN
+        {
+            const NodeType* a;
+            const NodeType* b;
+            sStkNN() {}
+            sStkNN(const NodeType* na,const NodeType* nb) : a(na), b(nb) {}
+        };
+        struct  sStkNP
+        {
+            const NodeType* node;
+            int mask;
+            sStkNP(const NodeType* n, unsigned m) : node(n), mask(m) {}
+        };
+        struct  sStkNPS
+        {
+            const NodeType* node;
+            int mask;
+            float value;
+            sStkNPS() {}
+            sStkNPS(const NodeType* n, unsigned m, const float v) : node(n), mask(m), value(v) {}
+        };
+        struct  sStkCLN
+        {
+            const NodeType* node;
+            NodeType* parent;
+            sStkCLN(const NodeType* n, NodeType* p) : node(n), parent(p) {}
+        };
+
+    public:
+        /* ICollideCollector templated collectors should implement this functions or inherit from this class */
+        struct  ICollideCollector
+        {
+            void Process(const NodeType*, const NodeType*) {}
+            void Process(const NodeType*) {}
+            void Process(const NodeType* n, const float) { Process(n); }
+            bool Descent(const NodeType*) { return true; }
+            bool AllLeaves(const NodeType*) { return true; }
+        };
+
+        /* IWriter  */
+        struct  IWriter
+        {
+            virtual ~IWriter() {}
+            virtual void Prepare(const NodeType* root,int numnodes) = 0;
+            virtual void WriteNode(const NodeType*, int index, int parent, int child0, int child1) = 0;
+            virtual void WriteLeaf(const NodeType*, int index, int parent) = 0;
+        };
+        /* IClone   */
+        struct  IClone
+        {
+            virtual ~IClone() {}
+            virtual void CloneLeaf(NodeType*) {}
+        };
+
+        // Constants
+        enum
+        {
+            SIMPLE_STACKSIZE = 64,
+            DOUBLE_STACKSIZE = SIMPLE_STACKSIZE * 2
+        };
+
+        // Methods
+        BvDynamicTree();
+        ~BvDynamicTree();
+
+        NodeType* GetRoot() const { return m_root; }
+        void Clear();
+        bool Empty() const { return 0 == m_root; }
+        int GetNumLeaves() const { return m_leaves; }
+        void OptimizeBottomUp();
+        void OptimizeTopDown(int bu_treshold = 128);
+        void OptimizeIncremental(int passes);
+        NodeType* Insert(const VolumeType& box,void* data);
+        void Update(NodeType* leaf, int lookahead=-1);
+        void Update(NodeType* leaf, VolumeType& volume);
+        bool Update(NodeType* leaf, VolumeType& volume, const AZ::Vector3& velocity, const float margin);
+        bool Update(NodeType* leaf, VolumeType& volume, const AZ::Vector3& velocity);
+        bool Update(NodeType* leaf, VolumeType& volume, const float margin);
+        void Remove(NodeType* leaf);
+        void Write(IWriter* iwriter) const;
+        void Clone(BvDynamicTree& dest, IClone* iclone=0) const;
+        static int GetMaxDepth(const NodeType* node);
+        static int CountLeaves(const NodeType* node);
+        static void ExtractLeaves(const NodeType* node, /*btAlignedObjectArray<const NodeType*>&*/vector<const NodeType*>& leaves);
+    #if DBVT_ENABLE_BENCHMARK
+        static void     Benchmark();
+    #else
+        static void     Benchmark(){}
+    #endif
+        /**
+         * Collector should inherit from ICollide
+         */
+        template<class Collector>
+        static inline void      enumNodes(  const NodeType* root, Collector& collector)
+        {
+            collector.Process(root);
+            if(root->IsInternal())
+            {
+                enumNodes(root->m_childs[0],collector);
+                enumNodes(root->m_childs[1],collector);
+            }
+        }
+        template<class Collector>
+        static void     enumLeaves( const NodeType* root,Collector& collector)
+        {
+            if(root->IsInternal())
+            {
+                enumLeaves(root->m_childs[0],collector);
+                enumLeaves(root->m_childs[1],collector);
+            }
+            else
+            {
+                collector.Process(root);
+            }
+        }
+        template<class Collector>
+        void        collideTT(  const NodeType* root0,const NodeType* root1,Collector& collector) const
+        {
+            if(root0&&root1)
+            {
+                size_t                          depth=1;
+                size_t                          treshold=DOUBLE_STACKSIZE-4;
+                vector<sStkNN>  stkStack;
+                stkStack.resize(DOUBLE_STACKSIZE);
+                stkStack[0]=sStkNN(root0,root1);
+                do  {
+                    sStkNN  p=stkStack[--depth];
+                    if(depth>treshold)
+                    {
+                        stkStack.resize(stkStack.size()*2);
+                        treshold=stkStack.size()-4;
+                    }
+                    if(p.a==p.b)
+                    {
+                        if(p.a->IsInternal())
+                        {
+                            stkStack[depth++]=sStkNN(p.a->m_childs[0],p.a->m_childs[0]);
+                            stkStack[depth++]=sStkNN(p.a->m_childs[1],p.a->m_childs[1]);
+                            stkStack[depth++]=sStkNN(p.a->m_childs[0],p.a->m_childs[1]);
+                        }
+                    }
+                    else if(IntersectAabbAabb(p.a->m_volume,p.b->m_volume))
+                    {
+                        if(p.a->IsInternal())
+                        {
+                            if(p.b->IsInternal())
+                            {
+                                stkStack[depth++]=sStkNN(p.a->m_childs[0],p.b->m_childs[0]);
+                                stkStack[depth++]=sStkNN(p.a->m_childs[1],p.b->m_childs[0]);
+                                stkStack[depth++]=sStkNN(p.a->m_childs[0],p.b->m_childs[1]);
+                                stkStack[depth++]=sStkNN(p.a->m_childs[1],p.b->m_childs[1]);
+                            }
+                            else
+                            {
+                                stkStack[depth++]=sStkNN(p.a->m_childs[0],p.b);
+                                stkStack[depth++]=sStkNN(p.a->m_childs[1],p.b);
+                            }
+                        }
+                        else
+                        {
+                            if(p.b->IsInternal())
+                            {
+                                stkStack[depth++]=sStkNN(p.a,p.b->m_childs[0]);
+                                stkStack[depth++]=sStkNN(p.a,p.b->m_childs[1]);
+                            }
+                            else
+                            {
+                                collector.Process(p.a,p.b);
+                            }
+                        }
+                    }
+                } while(depth);
+            }
+        }
+        template<class Collector>
+        void        collideTTpersistentStack(   const NodeType* root0, const NodeType* root1,Collector& collector)
+        {
+            if(root0&&root1)
+            {
+                size_t  depth=1;
+                size_t  treshold=DOUBLE_STACKSIZE-4;
+
+                m_stkStack.resize(DOUBLE_STACKSIZE);
+                m_stkStack[0]=sStkNN(root0,root1);
+                do
+                {
+                    sStkNN  p=m_stkStack[--depth];
+                    if(depth>treshold)
+                    {
+                        m_stkStack.resize(m_stkStack.size()*2);
+                        treshold=m_stkStack.size()-4;
+                    }
+                    if(p.a==p.b)
+                    {
+                        if(p.a->IsInternal())
+                        {
+                            m_stkStack[depth++]=sStkNN(p.a->m_childs[0],p.a->m_childs[0]);
+                            m_stkStack[depth++]=sStkNN(p.a->m_childs[1],p.a->m_childs[1]);
+                            m_stkStack[depth++]=sStkNN(p.a->m_childs[0],p.a->m_childs[1]);
+                        }
+                    }
+                    else if(IntersectAabbAabb(p.a->m_volume,p.b->m_volume))
+                    {
+                        if(p.a->IsInternal())
+                        {
+                            if(p.b->IsInternal())
+                            {
+                                m_stkStack[depth++]=sStkNN(p.a->m_childs[0],p.b->m_childs[0]);
+                                m_stkStack[depth++]=sStkNN(p.a->m_childs[1],p.b->m_childs[0]);
+                                m_stkStack[depth++]=sStkNN(p.a->m_childs[0],p.b->m_childs[1]);
+                                m_stkStack[depth++]=sStkNN(p.a->m_childs[1],p.b->m_childs[1]);
+                            }
+                            else
+                            {
+                                m_stkStack[depth++]=sStkNN(p.a->m_childs[0],p.b);
+                                m_stkStack[depth++]=sStkNN(p.a->m_childs[1],p.b);
+                            }
+                        }
+                        else
+                        {
+                            if(p.b->IsInternal())
+                            {
+                                m_stkStack[depth++]=sStkNN(p.a,p.b->m_childs[0]);
+                                m_stkStack[depth++]=sStkNN(p.a,p.b->m_childs[1]);
+                            }
+                            else
+                            {
+                                collector.Process(p.a,p.b);
+                            }
+                        }
+                    }
+                } while(depth);
+            }
+        }
+        template<class Collector>
+        void        collideTV(  const NodeType* root,   const VolumeType& volume,   Collector& collector) const
+        {
+            if(root)
+            {
+//              ATTRIBUTE_ALIGNED16(VolumeType)     volume(vol);
+//              btAlignedObjectArray<const NodeType*>   stack;
+                AZStd::fixed_vector<const NodeType*,SIMPLE_STACKSIZE> stack;
+                //stack.reserve(SIMPLE_STACKSIZE);
+                stack.push_back(root);
+                do  {
+                    const NodeType* n=stack[stack.size()-1];
+                    stack.pop_back();
+                    if(IntersectAabbAabb(n->m_volume,volume))
+                    {
+                        if(n->IsInternal())
+                        {
+                            stack.push_back(n->m_childs[0]);
+                            stack.push_back(n->m_childs[1]);
+                        }
+                        else
+                        {
+                            collector.Process(n);
+                        }
+                    }
+                } while(!stack.empty());
+            }
+        }
+
+        template<class Collector>
+        void collideTP(const NodeType* root, const AZ::Plane& plane, Collector& collector) const
+        {
+            if (root)
+            {
+                AZStd::fixed_vector<const NodeType*,SIMPLE_STACKSIZE> stack;
+                stack.push_back(root);
+                do
+                {
+                    const NodeType* n=stack[stack.size()-1];
+                    stack.pop_back();
+                    if (IntersectAabbPlane(n->m_volume, plane))
+                    {
+                        if(n->IsInternal())
+                        {
+                            stack.push_back(n->m_childs[0]);
+                            stack.push_back(n->m_childs[1]);
+                        }
+                        else
+                        {
+                            collector.Process(n);
+                        }
+                    }
+                } while (!stack.empty());
+            }
+        }
+
+        ///rayTest is a re-entrant ray test, and can be called in parallel as long as the btAlignedAlloc is thread-safe (uses locking etc)
+        ///rayTest is slower than rayTestInternal, because it builds a local stack, using memory allocations, and it recomputes signs/rayDirectionInverses each time
+        template<class Collector>
+        static void rayTest( const NodeType* root, const AZ::Vector3& rayFrom, const AZ::Vector3& rayTo, Collector& collector)
+        {
+            if(root)
+            {
+                AZ::Vector3 ray = rayTo-rayFrom;
+                AZ::Vector3 rayDir = ray.GetNormalized();
+
+                ///what about division by zero? --> just set rayDirection[i] to INF/1e30
+                AZ::Vector3 rayDirectionInverse = AZ::Vector3::CreateSelectCmpEqual(rayDir,AZ::Vector3::CreateZero(),AZ::Vector3(1e30),rayDir.GetReciprocal());
+
+                unsigned int signs[3];// = { rayDirectionInverse[0] < 0.0f, rayDirectionInverse[1] < 0.0f, rayDirectionInverse[2] < 0.0f };
+                signs[0] = rayDirectionInverse.GetX() < 0.0f;
+                signs[1] = rayDirectionInverse.GetY() < 0.0f;
+                signs[2] = rayDirectionInverse.GetZ() < 0.0f;
+
+                //float lambda_max = rayDir.Dot(ray);
+
+                AZ::Vector3 resultNormal;
+
+                //btAlignedObjectArray<const NodeType*> stack;
+                vector<const NodeType*> stack;
+
+                int                             depth=1;
+                int                             treshold=DOUBLE_STACKSIZE-2;
+
+                stack.resize(DOUBLE_STACKSIZE);
+                stack[0]=root;
+                AZ::Vector3 bounds[2];
+                do  {
+                    const NodeType* node=stack[--depth];
+
+                    bounds[0] = node->m_volume.GetMin();
+                    bounds[1] = node->m_volume.GetMax();
+
+                    //float tmin = 1.0f;
+                    //float lambda_min = 0.0f;
+                    // todo..
+                    unsigned int result1 = /*btRayAabb2(rayFrom,rayDirectionInverse,signs,bounds,tmin,lambda_min,lambda_max)*/0;
+#ifdef COMPARE_BTRAY_AABB2
+                    float param = 1.0f;
+                    bool result2 = /*btRayAabb(rayFrom,rayTo,node->volume.GetMin(),node->volume.GetMax(),param,resultNormal)*/0;
+                    AZ_Assert(result1 == result2, "");
+#endif //TEST_BTRAY_AABB2
+                    if(result1)
+                    {
+                        if(node->IsInternal())
+                        {
+                            if(depth>treshold)
+                            {
+                                stack.resize(stack.size()*2);
+                                treshold=stack.size()-2;
+                            }
+                            stack[depth++]=node->m_childs[0];
+                            stack[depth++]=node->m_childs[1];
+                        }
+                        else
+                        {
+                            collector.Process(node);
+                        }
+                    }
+                } while(depth);
+
+            }
+        }
+
+        ///rayTestInternal is faster than rayTest, because it uses a persistent stack (to reduce dynamic memory allocations to a minimum) and it uses precomputed signs/rayInverseDirections
+        ///rayTestInternal is used by btDbvtBroadphase to accelerate world ray casts
+        template<class Collector>
+        void        rayTestInternal(const NodeType* root, const AZ::Vector3& rayFrom, const AZ::Vector3& rayTo, const AZ::Vector3& rayDirectionInverse, unsigned int signs[3], const float lambda_max, const AZ::Vector3& aabbMin, const AZ::Vector3& aabbMax, Collector& collector) const
+        {
+            (void)rayFrom;(void)rayTo;(void)rayDirectionInverse;(void)signs;(void)lambda_max;
+            if(root)
+            {
+                AZ::Vector3 resultNormal;
+
+                int depth=1;
+                int treshold=DOUBLE_STACKSIZE-2;
+                vector<const NodeType*> stack;
+                stack.resize(DOUBLE_STACKSIZE);
+                stack[0]=root;
+                AZ::Vector3 bounds[2];
+                do
+                {
+                    const NodeType* node=stack[--depth];
+                    bounds[0] = node->m_volume.GetMin()+aabbMin;
+                    bounds[1] = node->m_volume.GetMax()+aabbMax;
+
+                    //float tmin = 1.0f;
+                    //float lambda_min = 0.0f;
+                    unsigned int result1=false;
+                    // todo...
+                    result1 = /*btRayAabb2(rayFrom,rayDirectionInverse,signs,bounds,tmin,lambda_min,lambda_max)*/false;
+                    if(result1)
+                    {
+                        if(node->IsInternal())
+                        {
+                            if(depth>treshold)
+                            {
+                                stack.resize(stack.size()*2);
+                                treshold=stack.size()-2;
+                            }
+                            stack[depth++]=node->m_childs[0];
+                            stack[depth++]=node->m_childs[1];
+                        }
+                        else
+                        {
+                            collector.Process(node);
+                        }
+                    }
+                } while(depth);
+            }
+        }
+
+        template<class Collector>
+        static void collideKDOP(const NodeType* root,   const AZ::Vector3* normals, const float* offsets, int count, Collector& collector)
+        {
+            (void)root;(void)normals;(void)offsets;(void)count;(void)collector;
+/*          if(root)
+            {
+                const int                       inside=(1<<count)-1;
+                btAlignedObjectArray<sStkNP>    stack;
+                int                             signs[sizeof(unsigned)*8];
+                btAssert(count<int (sizeof(signs)/sizeof(signs[0])));
+                for(int i=0;i<count;++i)
+                {
+                    signs[i]=   ((normals[i].x()>=0)?1:0)+
+                                ((normals[i].y()>=0)?2:0)+
+                                ((normals[i].z()>=0)?4:0);
+                }
+                stack.reserve(SIMPLE_STACKSIZE);
+                stack.push_back(sStkNP(root,0));
+                do  {
+                    sStkNP  se=stack[stack.size()-1];
+                    bool    out=false;
+                    stack.pop_back();
+                    for(int i=0,j=1;(!out)&&(i<count);++i,j<<=1)
+                    {
+                        if(0==(se.mask&j))
+                        {
+                            const int   side=se.node->volume.Classify(normals[i],offsets[i],signs[i]);
+                            switch(side)
+                            {
+                            case    -1: out=true;break;
+                            case    +1: se.mask|=j;break;
+                            }
+                        }
+                    }
+                    if(!out)
+                    {
+                        if((se.mask!=inside)&&(se.node->isinternal()))
+                        {
+                            stack.push_back(sStkNP(se.node->childs[0],se.mask));
+                            stack.push_back(sStkNP(se.node->childs[1],se.mask));
+                        }
+                        else
+                        {
+                            if(policy.AllLeaves(se.node)) enumLeaves(se.node,policy);
+                        }
+                    }
+                } while(!stack.empty());
+            }*/
+        }
+        template<class Collector>
+        static void collideOCL( const NodeType* root,   const AZ::Vector3* normals, const float* offsets, const AZ::Vector3& sortaxis, int count, Collector& collector, bool fullsort=true)
+        {
+            (void)root;(void)normals;(void)offsets;(void)sortaxis;(void)count;(void)offsets;(void)collector;(void)fullsort;
+/*              if(root)
+                {
+                    const unsigned                  srtsgns=(sortaxis[0]>=0?1:0)+
+                        (sortaxis[1]>=0?2:0)+
+                        (sortaxis[2]>=0?4:0);
+                    const int                       inside=(1<<count)-1;
+                    btAlignedObjectArray<sStkNPS>   stock;
+                    btAlignedObjectArray<int>       ifree;
+                    btAlignedObjectArray<int>       stack;
+                    int                             signs[sizeof(unsigned)*8];
+                    btAssert(count<int (sizeof(signs)/sizeof(signs[0])));
+                    for(int i=0;i<count;++i)
+                    {
+                        signs[i]=   ((normals[i].x()>=0)?1:0)+
+                            ((normals[i].y()>=0)?2:0)+
+                            ((normals[i].z()>=0)?4:0);
+                    }
+                    stock.reserve(SIMPLE_STACKSIZE);
+                    stack.reserve(SIMPLE_STACKSIZE);
+                    ifree.reserve(SIMPLE_STACKSIZE);
+                    stack.push_back(allocate(ifree,stock,sStkNPS(root,0,root->volume.ProjectMinimum(sortaxis,srtsgns))));
+                    do  {
+                        const int   id=stack[stack.size()-1];
+                        sStkNPS     se=stock[id];
+                        stack.pop_back();ifree.push_back(id);
+                        if(se.mask!=inside)
+                        {
+                            bool    out=false;
+                            for(int i=0,j=1;(!out)&&(i<count);++i,j<<=1)
+                            {
+                                if(0==(se.mask&j))
+                                {
+                                    const int   side=se.node->volume.Classify(normals[i],offsets[i],signs[i]);
+                                    switch(side)
+                                    {
+                                    case    -1: out=true;break;
+                                    case    +1: se.mask|=j;break;
+                                    }
+                                }
+                            }
+                            if(out) continue;
+                        }
+                        if(policy.Descent(se.node))
+                        {
+                            if(se.node->isinternal())
+                            {
+                                const NodeType* pns[]={ se.node->childs[0],se.node->childs[1]};
+                                sStkNPS     nes[]={ sStkNPS(pns[0],se.mask,pns[0]->volume.ProjectMinimum(sortaxis,srtsgns)),
+                                    sStkNPS(pns[1],se.mask,pns[1]->volume.ProjectMinimum(sortaxis,srtsgns))};
+                                const int   q=nes[0].value<nes[1].value?1:0;
+                                int         j=stack.size();
+                                if(fsort&&(j>0))
+                                {
+                                    // Insert 0
+                                    j=nearest(&stack[0],&stock[0],nes[q].value,0,stack.size());
+                                    stack.push_back(0);
+#if DBVT_USE_MEMMOVE
+                                    memmove(&stack[j+1],&stack[j],sizeof(int)*(stack.size()-j-1));
+#else
+                                    for(int k=stack.size()-1;k>j;--k) stack[k]=stack[k-1];
+#endif
+                                    stack[j]=allocate(ifree,stock,nes[q]);
+                                    // Insert 1
+                                    j=nearest(&stack[0],&stock[0],nes[1-q].value,j,stack.size());
+                                    stack.push_back(0);
+#if DBVT_USE_MEMMOVE
+                                    memmove(&stack[j+1],&stack[j],sizeof(int)*(stack.size()-j-1));
+#else
+                                    for(int k=stack.size()-1;k>j;--k) stack[k]=stack[k-1];
+#endif
+                                    stack[j]=allocate(ifree,stock,nes[1-q]);
+                                }
+                                else
+                                {
+                                    stack.push_back(allocate(ifree,stock,nes[q]));
+                                    stack.push_back(allocate(ifree,stock,nes[1-q]));
+                                }
+                            }
+                            else
+                            {
+                                policy.Process(se.node,se.value);
+                            }
+                        }
+                    } while(stack.size());
+                }*/
+        }
+
+        template<class Collector>
+        static void collideTU(const NodeType* root, Collector& collector)
+        {
+            (void)root;(void)collector;
+/*          if(root)
+            {
+                btAlignedObjectArray<const NodeType*>   stack;
+                stack.reserve(SIMPLE_STACKSIZE);
+                stack.push_back(root);
+                do  {
+                    const NodeType* n=stack[stack.size()-1];
+                    stack.pop_back();
+                    if(policy.Descent(n))
+                    {
+                        if(n->isinternal())
+                        { stack.push_back(n->childs[0]);stack.push_back(n->childs[1]); }
+                        else
+                        { policy.Process(n); }
+                    }
+                } while(stack.size()>0);
+            }*/
+        }
+
+    private:
+        BvDynamicTree(const BvDynamicTree&) {}
+
+        // Helpers
+        //static AZ_FORCE_INLINE int    nearest(const int* i,const BvDynamicTree::sStkNPS* a,const float& v,int l,int h)
+        //{
+        //  int m=0;
+        //  while(l<h)
+        //  {
+        //      m=(l+h)>>1;
+        //      if(a[i[m]].value>=v) l=m+1; else h=m;
+        //  }
+        //  return h;
+        //}
+        //static AZ_FORCE_INLINE int    allocate( int_fixed_stack_type& ifree, stknps_fixed_stack_type& stock, const sStkNPS& value)
+        //{
+        //  int i;
+        //  if( !ifree.empty() )
+        //  {
+        //      i=ifree[ifree.size()-1];
+        //      ifree.pop_back();
+        //      stock[i]=value;
+        //  }
+        //  else
+        //  {
+        //      i=stock.size();
+        //      stock.push_back(value);
+        //  }
+        //  return i;
+        //}
+        //
+
+        AZ_FORCE_INLINE void deletenode( NodeType* node)
+        {
+            //btAlignedFree(pdbvt->m_free);
+            delete m_free;
+            m_free=node;
+        }
+
+        void recursedeletenode( NodeType* node)
+        {
+            if(!node->IsLeaf())
+            {
+                recursedeletenode(node->m_childs[0]);
+                recursedeletenode(node->m_childs[1]);
+            }
+
+            if( node == m_root ) m_root=0;
+            deletenode(node);
+        }
+
+
+        AZ_FORCE_INLINE NodeType*   createnode( NodeType* parent, void* data)
+        {
+            NodeType*   node;
+            if(m_free)
+            { node=m_free;m_free=0; }
+            else
+            { node = aznew NodeType(); }
+            node->m_parent      =   parent;
+            node->m_data        =   data;
+            node->m_childs[1]   =   0;
+            return node;
+        }
+        AZ_FORCE_INLINE NodeType*   createnode( BvDynamicTree::NodeType* parent, const VolumeType& volume, void* data)
+        {
+            NodeType*   node = createnode(parent,data);
+            node->m_volume=volume;
+            return node;
+        }
+        //
+        AZ_FORCE_INLINE NodeType*   createnode( BvDynamicTree::NodeType* parent, const VolumeType& volume0, const VolumeType& volume1, void* data)
+        {
+            NodeType*   node = createnode(parent,data);
+            Merge(volume0,volume1,node->m_volume);
+            return node;
+        }
+        void        insertleaf( NodeType* root, NodeType* leaf);
+        NodeType*   removeleaf( NodeType* leaf);
+        void        fetchleaves(NodeType* root,NodeArrayType& leaves,int depth=-1);
+        void        split(const NodeArrayType& leaves,NodeArrayType& left,NodeArrayType& right,const AZ::Vector3& org,const AZ::Vector3& axis);
+        VolumeType  bounds(const NodeArrayType& leaves);
+        void        bottomup( NodeArrayType& leaves );
+        NodeType*   topdown(NodeArrayType& leaves,int bu_treshold);
+        AZ_FORCE_INLINE NodeType*   sort(NodeType* n,NodeType*& r);
+
+        NodeType*       m_root;
+        NodeType*       m_free;
+        int             m_lkhd;
+        int             m_leaves;
+        unsigned        m_opath;
+
+        //btAlignedObjectArray<sStkNN>  m_stkStack;
+        // Profile and choose static or dynamic vector.
+        typedef AZStd::fixed_vector<sStkNN,DOUBLE_STACKSIZE> stknn_fixed_stack_type;
+        typedef AZStd::fixed_vector<int,SIMPLE_STACKSIZE> int_fixed_stack_type;
+        typedef AZStd::fixed_vector<sStkNPS,SIMPLE_STACKSIZE> stknps_fixed_stack_type;
+
+        stknn_fixed_stack_type  m_stkStack;
+    };
+}
+
+#endif // RR_DYNAMIC_BOUNDING_VOLUME_TREE_H
+#pragma once

Code/Framework/GridMate/GridMate/Replica/Interest/InterestDefs.h

@@ -0,0 +1,136 @@
+/*
+* All or portions of this file Copyright (c) Amazon.com, Inc. or its affiliates or
+* its licensors.
+*
+* For complete copyright and license terms please see the LICENSE at the root of this
+* distribution (the "License"). All use of this software is governed by the License,
+* or, if provided, by the license below or the license accompanying this file. Do not
+* remove or modify any license notices. This file is distributed on an "AS IS" BASIS,
+* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+*
+*/
+#ifndef GM_REPLICA_INTERESTDEFS_H
+#define GM_REPLICA_INTERESTDEFS_H
+
+#include <AzCore/base.h>
+#include <GridMate/Replica/ReplicaCommon.h>
+#include <GridMate/Containers/vector.h>
+#include <GridMate/Containers/unordered_set.h>
+#include <GridMate/Containers/unordered_map.h>
+#include <AzCore/std/sort.h>
+
+namespace GridMate
+{
+    /**
+    * Bitmask used internally in InterestManager to check which handler is responsible for a given interest match
+    */
+    using InterestHandlerSlot = AZ::u32;
+
+    /**
+    * Rule identifier (unique within the session)
+    */
+    using RuleNetworkId = AZ::u64;
+    ///////////////////////////////////////////////////////////////////////////
+
+    using InterestPeerSet = unordered_set<PeerId>;
+
+    /**
+     * InterestMatchResult: a structure to gather new matches from handlers.
+     * Passed to handler within matching context when handler's Match method is invoked.
+     * User must fill the structure with changes that handler recalculated.
+     *
+     * Specifically, the changes should have all the replicas that had their list of associated peers modified.
+     * Each entry replica - new full list of associated peers.
+     */
+    class InterestMatchResult : public unordered_map<ReplicaId, InterestPeerSet>
+    {
+    public:
+        using unordered_map::unordered_map;
+
+        /*
+        * An expensive debug trace helper, prints sorted mapping between replica id's and associated peers.
+        */
+        void PrintMatchResult(const char* name) const;
+    };
+    ///////////////////////////////////////////////////////////////////////////
+
+    /**
+    *  Base class for interest rules
+    */
+    class InterestRule
+    {
+    public:
+        explicit InterestRule(PeerId peerId, RuleNetworkId netId)
+            : m_peerId(peerId)
+            , m_netId(netId)
+        {}
+
+        PeerId GetPeerId() const { return m_peerId; }
+        RuleNetworkId GetNetworkId() const { return m_netId; }
+
+    protected:
+        PeerId m_peerId; ///< the peer this rule is bound to
+        RuleNetworkId m_netId; ///< network id
+    };
+    ///////////////////////////////////////////////////////////////////////////
+
+
+    /**
+    *  Base class for interest attributes
+    */
+    class InterestAttribute
+    {
+    public:
+        explicit InterestAttribute(ReplicaId replicaId)
+            : m_replicaId(replicaId)
+        {}
+
+        ReplicaId GetReplicaId() const { return m_replicaId; }
+
+    protected:
+        ReplicaId m_replicaId; ///< Replica id this attribute is bound to
+    };
+    ///////////////////////////////////////////////////////////////////////////
+
+#if !defined(AZ_DEBUG_BUILD)
+    AZ_INLINE void InterestMatchResult::PrintMatchResult(const char*) const {}
+#else
+    AZ_INLINE void InterestMatchResult::PrintMatchResult(const char* name) const
+    {
+        if (size() == 0)
+        {
+            AZ_TracePrintf("GridMate", "InterestMatchResult %s empty \n", name);
+            return;
+        }
+
+        AZStd::vector<value_type> sorted;
+        for (auto& r : *this)
+        {
+            sorted.push_back(r);
+        }
+
+        auto sortByReplicaId = [](const value_type& one, const value_type& another)
+        {
+            return one.first < another.first;
+        };
+        AZStd::sort(sorted.begin(), sorted.end(), sortByReplicaId);
+
+        AZ_TracePrintf("GridMate", "InterestMatchResult %s \n", name);
+        for (auto& match : sorted)
+        {
+            auto repId = match.first;
+            AZ_TracePrintf("GridMate", "\t\t\t for repId %d ", repId);
+
+            // unsorted list of peers
+            for (auto& peerId : match.second)
+            {
+                AZ_TracePrintf("", "peer %d", peerId);
+            }
+
+            AZ_TracePrintf("", "\n");
+        }
+    }
+#endif
+} // GridMate
+
+#endif // GM_REPLICA_INTERESTDEFS_H

Code/Framework/GridMate/GridMate/Replica/Interest/InterestEvents.h

@@ -0,0 +1,57 @@
+/*
+* All or portions of this file Copyright (c) Amazon.com, Inc. or its affiliates or
+* its licensors.
+*
+* For complete copyright and license terms please see the LICENSE at the root of this
+* distribution (the "License"). All use of this software is governed by the License,
+* or, if provided, by the license below or the license accompanying this file. Do not
+* remove or modify any license notices. This file is distributed on an "AS IS" BASIS,
+* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+*
+*/
+
+#ifndef GM_REPLICA_INTERESTEVENTS_H
+#define GM_REPLICA_INTERESTEVENTS_H
+
+#if defined(GM_INTEREST_MANAGER)
+
+#include <AzCore/base.h>
+#include <GridMate/Replica/ReplicaCommon.h>
+
+#include <GridMate/containers/unordered_set.h>
+#include <GridMate/containers/unordered_map.h>
+#include <GridMate/EBus.h>
+
+namespace GridMate
+{
+    /**
+    *  EBus for interest manager's events.
+    *  Notifies subscribers about new interest matches and new mismatches happened.
+    */
+    class InterestManagerEvents
+        : public AZ::EBusTraits
+    {
+    public:
+        static const AZ::EBusAddressPolicy AddressPolicy = AZ::EBusAddressPolicy::ById;
+        typedef AZStd::recursive_mutex MutexType;
+        typedef void* BusIdType;
+        typedef SysContAlloc AllocatorType;
+
+        virtual ~InterestManagerEvents() {}
+
+        /**
+        * Called when new pair of replica and peer matched their interest
+        */
+        virtual void OnInterestMatched(ReplicaId replicaId, PeerId peerId) { (void) replicaId; (void) peerId; }
+
+        /**
+        * Called when pair of replica and peer mismatched interest (only called if the pair was previously matching)
+        */
+        virtual void OnInterestUnmatched(ReplicaId replicaId, PeerId peerId) { (void) replicaId; (void) peerId; }
+    };
+
+    typedef AZ::EBus<InterestManagerEvents> InterestManagerEventsBus;
+}
+
+#endif // GM_INTEREST_MANAGER
+#endif

Code/Framework/GridMate/GridMate/Replica/Interest/InterestManager.cpp

@@ -0,0 +1,247 @@
+/*
+* All or portions of this file Copyright (c) Amazon.com, Inc. or its affiliates or
+* its licensors.
+*
+* For complete copyright and license terms please see the LICENSE at the root of this
+* distribution (the "License"). All use of this software is governed by the License,
+* or, if provided, by the license below or the license accompanying this file. Do not
+* remove or modify any license notices. This file is distributed on an "AS IS" BASIS,
+* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+*
+*/
+
+#include <GridMate/Replica/Interest/InterestManager.h>
+#include <GridMate/Replica/Interest/RulesHandler.h>
+#include <GridMate/Replica/Interest/InterestQueryResult.h>
+
+#include <GridMate/Replica/ReplicaMgr.h>
+
+#include <AzCore/std/containers/array.h>
+
+namespace GridMate
+{
+    static const unsigned k_maxHandlers = sizeof(GridMate::InterestHandlerSlot) * CHAR_BIT;
+
+    /**
+    * Hashing utils
+    */
+    struct ReplicaHashByPeer
+    {
+        AZ_FORCE_INLINE AZStd::size_t operator()(const ReplicaTarget* t) const
+        {
+            static_assert(sizeof(AZStd::size_t) >= sizeof(ReplicaPeer*), "Types sizes mismatch");
+            return reinterpret_cast<AZStd::size_t>(t->GetPeer());
+        }
+    };
+
+    struct ReplicaEqualToByPeer 
+    {
+        AZ_FORCE_INLINE bool operator()(const ReplicaTarget* left, const ReplicaTarget* right) const
+        {
+            return left->GetPeer() == right->GetPeer();
+        }
+    };
+
+    struct ReplicaHashByPeerId
+    {
+        AZ_FORCE_INLINE AZStd::size_t operator()(PeerId peerId) const
+        {
+            static_assert(sizeof(AZStd::size_t) >= sizeof(PeerId), "Types sizes mismatch");
+            return static_cast<AZStd::size_t>(peerId);
+        }
+    };
+
+    struct ReplicaEqualToByPeerId
+    {
+        AZ_FORCE_INLINE bool operator()(PeerId peerId, const ReplicaTarget* right) const
+        {
+            return peerId == right->GetPeer()->GetId();
+        }
+    };
+    ///////////////////////////////////////////////////////////////////////////
+
+    /**
+    * InterestManager
+    */
+    InterestManager::InterestManager()
+        : m_rm(nullptr)
+        , m_freeSlots(~0u)
+    {
+    }
+
+    void InterestManager::Init(const InterestManagerDesc& desc)
+    {
+        m_rm = desc.m_rm;
+        AZ_Assert(m_rm, "Invalid replica manager");
+    }
+
+    bool InterestManager::IsReady() const
+    {
+        return m_rm != nullptr;
+    }
+
+    InterestManager::~InterestManager()
+    {
+        while (!m_handlers.empty())
+        {
+            m_handlers.back()->OnRulesHandlerUnregistered(this);
+            m_handlers.pop_back();
+        }
+    }
+
+    void InterestManager::RegisterHandler(BaseRulesHandler* handler)
+    {
+        AZ_Assert(handler, "Invalid rules handler");
+
+        for (BaseRulesHandler* h : m_handlers)
+        {
+            if (h == handler)
+            {
+                AZ_TracePrintf("GridMate", "Rules handler %p is already registered", handler);
+                return;
+            }
+        }
+
+        InterestHandlerSlot slot = GetNewSlot();
+        if (!slot)
+        {
+            AZ_TracePrintf("GridMate", "Too many rules handlers, max=%u\n", k_maxHandlers);
+            return;
+        }
+
+        handler->m_slot = slot;
+        m_handlers.push_back(handler);
+        handler->OnRulesHandlerRegistered(this);
+    }
+
+    void InterestManager::UnregisterHandler(BaseRulesHandler* handler)
+    {
+        AZ_Assert(handler, "Invalid rules handler");
+
+        for (auto it = m_handlers.begin(); it != m_handlers.end(); ++it)
+        {
+            if (*it == handler)
+            {
+                handler->OnRulesHandlerUnregistered(this);
+                m_handlers.erase(it);
+                return;
+            }
+        }
+
+        AZ_Assert(false, "Handler was not registered");
+    }
+
+    void InterestManager::Update()
+    {
+        // Updating all handlers
+        for (BaseRulesHandler* handler : m_handlers)
+        {
+            handler->Update();
+        }
+
+        // merging results from every handler
+        for (BaseRulesHandler* handler : m_handlers)
+        {
+            const InterestMatchResult& result = handler->GetLastResult();
+
+            for (auto& match : result)
+            {
+                ReplicaPtr replica = m_rm->FindReplica(match.first);
+                if (!replica) // replica was destroyed: ignoring this match
+                {
+                    continue;
+                }
+
+                unordered_set<ReplicaTarget*, ReplicaHashByPeer, ReplicaEqualToByPeer> targets;
+
+                for (ReplicaTarget& targetObj : replica->m_targets)
+                {
+                    targets.insert(&targetObj);
+                    if (!match.second.count(targetObj.GetPeer()->GetId()))
+                    {
+                        targetObj.m_slotMask &= ~handler->m_slot;
+                        if (!targetObj.m_slotMask)
+                        {
+                            targetObj.m_flags |= ReplicaTarget::TargetRemoved;
+                            m_rm->OnReplicaChanged(replica);
+                        }
+                    }
+                }
+
+                for (const PeerId& peerId : match.second)
+                {
+                    ReplicaTarget* rt = nullptr;
+
+                    auto it = targets.find_as(peerId, ReplicaHashByPeerId(), ReplicaEqualToByPeerId());
+                    if (it == targets.end())
+                    {
+                        ReplicaPeer* peer = m_rm->FindPeer(peerId);
+
+                        if (!ShouldForward(replica.get(), peer))
+                        {
+                            continue;
+                        }
+
+                        rt = ReplicaTarget::AddReplicaTarget(peer, replica.get());
+                        rt->SetNew(true);
+                        m_rm->OnReplicaChanged(replica);
+                    }
+                    else
+                    {
+                        rt = *it;
+                    }
+
+                    rt->m_slotMask |= handler->m_slot;
+                    rt->m_flags &= ~ReplicaTarget::TargetRemoved;
+                }
+            }
+        }
+    }
+
+
+    bool InterestManager::ShouldForward(Replica* replica, ReplicaPeer* peer) const
+    {
+        if (!peer) // invalid peer
+        {
+            return false;
+        }
+
+        if (replica->IsMaster()) // own the replica?
+        {
+            return true;
+        }
+
+        if (m_rm->GetLocalPeerId() == peer->GetId() || peer->GetId() == replica->m_upstreamHop->GetId()) // forwarding to local peer or to owner?
+        {
+            return false;
+        }
+
+        if (m_rm->IsSyncHost() && !(replica->m_upstreamHop->GetMode() == Mode_Peer && peer->GetMode() == Mode_Peer)) // we are host and replica' owner and target are not connected
+        {
+            return true;
+        }
+
+        return false;
+    }
+
+    InterestHandlerSlot InterestManager::GetNewSlot()
+    {
+        InterestHandlerSlot s = m_freeSlots;
+        for (unsigned i = 0; s && i < k_maxHandlers; ++i, s >>= 1)
+        {
+            if (s & 1)
+            {
+                InterestHandlerSlot slot = (1 << i);
+                m_freeSlots &= ~slot;
+                return slot;
+            }
+        }
+        return 0;
+    }
+
+    void InterestManager::FreeSlot(InterestHandlerSlot slot)
+    {
+        m_freeSlots |= slot;
+    }
+    ///////////////////////////////////////////////////////////////////////////
+}

Code/Framework/GridMate/GridMate/Replica/Interest/InterestManager.h

@@ -0,0 +1,95 @@
+/*
+* All or portions of this file Copyright (c) Amazon.com, Inc. or its affiliates or
+* its licensors.
+*
+* For complete copyright and license terms please see the LICENSE at the root of this
+* distribution (the "License"). All use of this software is governed by the License,
+* or, if provided, by the license below or the license accompanying this file. Do not
+* remove or modify any license notices. This file is distributed on an "AS IS" BASIS,
+* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+*
+*/
+#ifndef GM_REPLICA_INTERESTMANAGER_H
+#define GM_REPLICA_INTERESTMANAGER_H
+
+#include <GridMate/Containers/list.h>
+
+#include <GridMate/Replica/Interest/InterestDefs.h>
+
+namespace GridMate
+{
+    class BaseRulesHandler;
+
+    /**
+    *  Interest manager initialization parameters
+    */
+    struct InterestManagerDesc
+    {
+        ReplicaManager* m_rm; ///< Replica manager instance
+
+        InterestManagerDesc()
+            : m_rm(nullptr)
+        {
+
+        }
+    };
+
+    /**
+    *  InterestManager: responsible for matching of replicas and peers pairs based on rules and attribute provided.
+    *  InterestManager allows registration of up to 32 custom rules handler. Each rules handler is responsible of matching attributes
+    *  and rules that user provides. InterestManager is responsible for merging results of matching from every registered handler and
+    *  maintaining valid forwarding targets cache on every Replica.
+    */
+    class InterestManager
+    {
+    public:
+
+        GM_CLASS_ALLOCATOR(InterestManager);
+
+        InterestManager();
+        ~InterestManager();
+
+        /**
+        * Initialize manager with a descriptor
+        */
+        void Init(const InterestManagerDesc& desc);
+
+        /**
+        * Returns true if InterestManager is initialized and is ready to use
+        */
+        bool IsReady() const;
+
+        /**
+        * Register new handler with a given type and instance
+        */
+        void RegisterHandler(BaseRulesHandler* handler);
+
+        /**
+        * Unregister handler
+        */
+        void UnregisterHandler(BaseRulesHandler* handler);
+
+        /**
+        * Call to update current replica->peers cache
+        */
+        void Update();
+
+        /**
+        *  Returns replica manager IM is bount to
+        */
+        ReplicaManager* GetReplicaManager() { return m_rm; }
+    private:
+        InterestManager(const InterestManager&) = delete;
+        InterestManager& operator=(const InterestManager&) = delete;
+
+        InterestHandlerSlot GetNewSlot();
+        void FreeSlot(InterestHandlerSlot slot);
+        bool ShouldForward(Replica* replica, ReplicaPeer* peer) const;
+
+        ReplicaManager* m_rm;
+        vector<BaseRulesHandler*> m_handlers;
+        InterestHandlerSlot m_freeSlots;
+    };
+} // namespace GridMate
+
+#endif // GM_REPLICA_INTERESTMANAGER_H

Code/Framework/GridMate/GridMate/Replica/Interest/InterestQueryResult.cpp

@@ -0,0 +1,29 @@
+/*
+* All or portions of this file Copyright (c) Amazon.com, Inc. or its affiliates or
+* its licensors.
+*
+* For complete copyright and license terms please see the LICENSE at the root of this
+* distribution (the "License"). All use of this software is governed by the License,
+* or, if provided, by the license below or the license accompanying this file. Do not
+* remove or modify any license notices. This file is distributed on an "AS IS" BASIS,
+* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+*
+*/
+/*
+#include <GridMate/Replica/Interest/InterestQueryResult.h>
+
+namespace GridMate
+{
+    InterestQueryResult::InterestQueryResult()
+    {
+
+    }
+
+    InterestQueryResult::PeerList& InterestQueryResult::Insert(ReplicaId repId)
+    {
+        auto it = m_matches.insert_key(repId);
+        return it.first->second;
+    }
+} // namespace GridMate
+
+*/

Code/Framework/GridMate/GridMate/Replica/Interest/InterestQueryResult.h

@@ -0,0 +1,29 @@
+/*
+* All or portions of this file Copyright (c) Amazon.com, Inc. or its affiliates or
+* its licensors.
+*
+* For complete copyright and license terms please see the LICENSE at the root of this
+* distribution (the "License"). All use of this software is governed by the License,
+* or, if provided, by the license below or the license accompanying this file. Do not
+* remove or modify any license notices. This file is distributed on an "AS IS" BASIS,
+* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+*
+*/
+#ifndef GM_REPLICA_INTERESTQUERYRESULT_H
+#define GM_REPLICA_INTERESTQUERYRESULT_H
+/*
+#include <AzCore/base.h>
+
+#include <GridMate/containers/vector.h>
+#include <GridMate/containers/unordered_map.h>
+#include <GridMate/Replica/Interest/InterestDefs.h>
+#include <GridMate/Replica/ReplicaCommon.h>
+
+namespace GridMate
+{
+
+    using InterestPeerList = vector<PeerId>;
+    using InterestQueryResult = unordered_map<ReplicaId, InterestPeerList>;
+} // GridMate
+*/
+#endif // GM_REPLICA_INTERESTQUERYRESULT_H

Code/Framework/GridMate/GridMate/Replica/Interest/ProximityInterestHandler.cpp

@@ -0,0 +1,597 @@
+/*
+* All or portions of this file Copyright (c) Amazon.com, Inc. or its affiliates or
+* its licensors.
+*
+* For complete copyright and license terms please see the LICENSE at the root of this
+* distribution (the "License"). All use of this software is governed by the License,
+* or, if provided, by the license below or the license accompanying this file. Do not
+* remove or modify any license notices. This file is distributed on an "AS IS" BASIS,
+* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+*
+*/
+
+#include <GridMate/Replica/Interest/ProximityInterestHandler.h>
+
+#include <GridMate/Replica/Replica.h>
+#include <GridMate/Replica/ReplicaFunctions.h>
+#include <GridMate/Replica/ReplicaMgr.h>
+
+#include <GridMate/Replica/Interest/InterestManager.h>
+#include <GridMate/Replica/Interest/BvDynamicTree.h>
+
+// for highly verbose internal debugging
+//#define INTERNAL_DEBUG_PROXIMITY
+
+namespace GridMate
+{
+
+    void ProximityInterestChunk::OnReplicaActivate(const ReplicaContext& rc)
+    {
+        m_interestHandler = static_cast<ProximityInterestHandler*>(rc.m_rm->GetUserContext(AZ_CRC("ProximityInterestHandler", 0x3a90b3e4)));
+        AZ_Warning("GridMate", m_interestHandler, "No proximity interest handler in the user context");
+
+        if (m_interestHandler)
+        {
+            m_interestHandler->OnNewRulesChunk(this, rc.m_peer);
+        }
+    }
+
+    void ProximityInterestChunk::OnReplicaDeactivate(const ReplicaContext& rc)
+    {
+        if (rc.m_peer && m_interestHandler)
+        {
+            m_interestHandler->OnDeleteRulesChunk(this, rc.m_peer);
+        }
+    }
+
+    bool ProximityInterestChunk::AddRuleFn(RuleNetworkId netId, AZ::Aabb bbox, const RpcContext& ctx)
+    {
+        if (IsProxy())
+        {
+            auto rulePtr = m_interestHandler->CreateRule(ctx.m_sourcePeer);
+            rulePtr->Set(bbox);
+            m_rules.insert(AZStd::make_pair(netId, rulePtr));
+        }
+
+        return true;
+    }
+
+    bool ProximityInterestChunk::RemoveRuleFn(RuleNetworkId netId, const RpcContext&)
+    {
+        if (IsProxy())
+        {
+            m_rules.erase(netId);
+        }
+
+        return true;
+    }
+
+    bool ProximityInterestChunk::UpdateRuleFn(RuleNetworkId netId, AZ::Aabb bbox, const RpcContext&)
+    {
+        if (IsProxy())
+        {
+            auto it = m_rules.find(netId);
+            if (it != m_rules.end())
+            {
+                it->second->Set(bbox);
+            }
+        }
+
+        return true;
+    }
+
+    bool ProximityInterestChunk::AddRuleForPeerFn(RuleNetworkId netId, PeerId peerId, AZ::Aabb bbox, const RpcContext&)
+    {
+        ProximityInterestChunk* peerChunk = m_interestHandler->FindRulesChunkByPeerId(peerId);
+        if (peerChunk)
+        {
+            auto it = peerChunk->m_rules.find(netId);
+            if (it == peerChunk->m_rules.end())
+            {
+                auto rulePtr = m_interestHandler->CreateRule(peerId);
+                peerChunk->m_rules.insert(AZStd::make_pair(netId, rulePtr));
+                rulePtr->Set(bbox);
+            }
+        }
+        return false;
+    }
+    ///////////////////////////////////////////////////////////////////////////
+
+
+    /*
+    * ProximityInterest
+    */
+    ProximityInterest::ProximityInterest(ProximityInterestHandler* handler)
+        : m_handler(handler)
+          , m_bbox(AZ::Aabb::CreateNull())
+    {
+        AZ_Assert(m_handler, "Invalid interest handler");
+    }
+    ///////////////////////////////////////////////////////////////////////////
+
+
+    /*
+    * ProximityInterestRule
+    */
+    void ProximityInterestRule::Set(const AZ::Aabb& bbox)
+    {
+        m_bbox = bbox;
+        m_handler->UpdateRule(this);
+    }
+
+    void ProximityInterestRule::Destroy()
+    {
+        m_handler->DestroyRule(this);
+    }
+    ///////////////////////////////////////////////////////////////////////////
+
+
+    /*
+    * ProximityInterestAttribute
+    */
+    void ProximityInterestAttribute::Set(const AZ::Aabb& bbox)
+    {
+        m_bbox = bbox;
+        m_handler->UpdateAttribute(this);
+    }
+
+    void ProximityInterestAttribute::Destroy()
+    {
+        m_handler->DestroyAttribute(this);
+    }
+    ///////////////////////////////////////////////////////////////////////////
+
+
+    /*
+    * ProximityInterestHandler
+    */
+    ProximityInterestHandler::ProximityInterestHandler()
+        : m_im(nullptr)
+        , m_rm(nullptr)
+        , m_lastRuleNetId(0)
+        , m_rulesReplica(nullptr)
+    {
+        m_attributeWorld = AZStd::make_unique<SpatialIndex>();
+        AZ_Assert(m_attributeWorld, "Out of memory");
+    }
+
+    ProximityInterestRule::Ptr ProximityInterestHandler::CreateRule(PeerId peerId)
+    {
+        ProximityInterestRule* rulePtr = aznew ProximityInterestRule(this, peerId, GetNewRuleNetId());
+        if (m_rm && peerId == m_rm->GetLocalPeerId())
+        {
+            m_rulesReplica->AddRuleRpc(rulePtr->GetNetworkId(), rulePtr->Get());
+        }
+
+        CreateAndInsertIntoSpatialStructure(rulePtr);
+
+        return rulePtr;
+    }
+
+    ProximityInterestAttribute::Ptr ProximityInterestHandler::CreateAttribute(ReplicaId replicaId)
+    {
+        auto newAttribute = aznew ProximityInterestAttribute(this, replicaId);
+        AZ_Assert(newAttribute, "Out of memory");
+
+        CreateAndInsertIntoSpatialStructure(newAttribute);
+
+        return newAttribute;
+    }
+
+    void ProximityInterestHandler::FreeRule(ProximityInterestRule* rule)
+    {
+        //TODO: should be pool-allocated
+        delete rule;
+    }
+
+    void ProximityInterestHandler::DestroyRule(ProximityInterestRule* rule)
+    {
+        if (m_rm && rule->GetPeerId() == m_rm->GetLocalPeerId())
+        {
+            m_rulesReplica->RemoveRuleRpc(rule->GetNetworkId());
+        }
+
+        MarkAttributesDirtyInRule(rule);
+
+        rule->m_bbox = AZ::Aabb::CreateNull();
+        m_removedRules.insert(rule);
+        m_localRules.erase(rule);
+    }
+
+    void ProximityInterestHandler::UpdateRule(ProximityInterestRule* rule)
+    {
+        if (m_rm && rule->GetPeerId() == m_rm->GetLocalPeerId())
+        {
+            m_rulesReplica->UpdateRuleRpc(rule->GetNetworkId(), rule->Get());
+        }
+
+        m_dirtyRules.insert(rule);
+    }
+
+    void ProximityInterestHandler::FreeAttribute(ProximityInterestAttribute* attrib)
+    {
+        delete attrib;
+    }
+
+    void ProximityInterestHandler::DestroyAttribute(ProximityInterestAttribute* attrib)
+    {
+        RemoveFromSpatialStructure(attrib);
+
+        m_attributes.erase(attrib);
+        m_removedAttributes.insert(attrib);
+    }
+
+    void ProximityInterestHandler::RemoveFromSpatialStructure(ProximityInterestAttribute* attribute)
+    {
+        attribute->m_bbox = AZ::Aabb::CreateNull();
+        m_attributeWorld->Remove(attribute->GetNode());
+        attribute->SetNode(nullptr);
+    }
+
+    void ProximityInterestHandler::UpdateAttribute(ProximityInterestAttribute* attrib)
+    {
+        auto node = attrib->GetNode();
+        AZ_Assert(node, "Attribute wasn't created correctly");
+        node->m_volume = attrib->Get();
+        m_attributeWorld->Update(node);
+
+        m_dirtyAttributes.insert(attrib);
+    }
+
+    void ProximityInterestHandler::OnNewRulesChunk(ProximityInterestChunk* chunk, ReplicaPeer* peer)
+    {
+        if (chunk != m_rulesReplica) // non-local
+        {
+            m_peerChunks.insert(AZStd::make_pair(peer->GetId(), chunk));
+
+            for (auto& rule : m_localRules)
+            {
+                chunk->AddRuleForPeerRpc(rule->GetNetworkId(), rule->GetPeerId(), rule->Get());
+            }
+        }
+    }
+
+    void ProximityInterestHandler::OnDeleteRulesChunk(ProximityInterestChunk* chunk, ReplicaPeer* peer)
+    {
+        (void)chunk;
+        m_peerChunks.erase(peer->GetId());
+    }
+
+    RuleNetworkId ProximityInterestHandler::GetNewRuleNetId()
+    {
+        ++m_lastRuleNetId;
+
+        if (m_rulesReplica)
+        {
+            return m_rulesReplica->GetReplicaId() | (static_cast<AZ::u64>(m_lastRuleNetId) << 32);
+        }
+
+        return (static_cast<AZ::u64>(m_lastRuleNetId) << 32);
+    }
+
+    ProximityInterestChunk* ProximityInterestHandler::FindRulesChunkByPeerId(PeerId peerId)
+    {
+        auto it = m_peerChunks.find(peerId);
+        if (it == m_peerChunks.end())
+        {
+            return nullptr;
+        }
+
+        return it->second;
+    }
+
+    const InterestMatchResult& ProximityInterestHandler::GetLastResult()
+    {
+        return m_resultCache;
+    }
+
+    ProximityInterestHandler::RuleSet& ProximityInterestHandler::GetAffectedRules()
+    {
+        /*
+         * The expectation that lots of attributes will change frequently,
+         * so there is no point in trying to optimize cases
+         * where only a few attributes have changed.
+         */
+        if (m_dirtyAttributes.empty() && !m_dirtyRules.empty())
+        {
+            return m_dirtyRules;
+        }
+
+        /*
+         * Assuming all rules might have been affected.
+         *
+         * There is an optimization chance here if the number of rules is large, as in 1,000+ rules.
+         * To handle such scale we would need another spatial structure for rules.
+         */
+        return m_localRules;
+    }
+
+    void ProximityInterestHandler::GetAttributesWithinRule(ProximityInterestRule* rule, SpatialIndex::NodeCollector& nodes)
+    {
+        m_attributeWorld->Query(rule->Get(), nodes);
+    }
+
+    void ProximityInterestHandler::ClearDirtyState()
+    {
+        m_dirtyAttributes.clear();
+        m_dirtyRules.clear();
+    }
+
+    void ProximityInterestHandler::CreateAndInsertIntoSpatialStructure(ProximityInterestAttribute* attribute)
+    {
+        m_attributes.insert(attribute);
+        SpatialIndex::Node* node = m_attributeWorld->Insert(attribute->Get(), attribute);
+        attribute->SetNode(node);
+    }
+
+    void ProximityInterestHandler::CreateAndInsertIntoSpatialStructure(ProximityInterestRule* rule)
+    {
+        m_localRules.insert(rule);
+    }
+
+    void ProximityInterestHandler::UpdateInternal(InterestMatchResult& result)
+    {
+        /*
+         * The goal is to return all dirty attributes that were either dirty because:
+         * 1) they changed which rules have apply to
+         * 2) rules have changed and no longer apply to those attributes
+         * and thus resulted in different peer(s) associated with a given replica.
+         */
+
+        const RuleSet& rules = GetAffectedRules();
+
+        for (auto& dirtyAttribute : m_dirtyAttributes)
+        {
+            result.insert(dirtyAttribute->GetReplicaId());
+        }
+
+        /*
+        * The exectation is to have a lot more attributes than rules.
+        * The amount of rules should grow linear with amount of peers,
+        * so it should be OK to iterate through all rules each update.
+        */
+        for (auto& rule : rules)
+        {
+            CheckChangesForRule(rule, result);
+        }
+
+        for (auto& removedRule : m_removedRules)
+        {
+            FreeRule(removedRule);
+        }
+        m_removedRules.clear();
+
+        // mark removed attribute as having no peers
+        for (auto& removedAttribute : m_removedAttributes)
+        {
+            result.insert(removedAttribute->GetReplicaId());
+            FreeAttribute(removedAttribute);
+        }
+        m_removedAttributes.clear();
+    }
+
+    void ProximityInterestHandler::CheckChangesForRule(ProximityInterestRule* rule, InterestMatchResult& result)
+    {
+        SpatialIndex::NodeCollector collector;
+        GetAttributesWithinRule(rule, collector);
+
+        auto peerId = rule->GetPeerId();
+        for (ProximityInterestAttribute* attr : collector.GetNodes())
+        {
+            AZ_Assert(attr, "bad node?");
+
+            auto findIt = result.find(attr->GetReplicaId());
+            if (findIt != result.end())
+            {
+                findIt->second.insert(peerId);
+            }
+            else
+            {
+                auto resultIt = result.insert(attr->GetReplicaId());
+                AZ_Assert(resultIt.second, "Successfully inserted");
+                resultIt.first->second.insert(peerId);
+            }
+        }
+    }
+
+    void ProximityInterestHandler::MarkAttributesDirtyInRule(ProximityInterestRule* rule)
+    {
+        SpatialIndex::NodeCollector collector;
+        GetAttributesWithinRule(rule, collector);
+
+        for (ProximityInterestAttribute* attr : collector.GetNodes())
+        {
+            AZ_Assert(attr, "bad node?");
+
+            UpdateAttribute(attr);
+        }
+    }
+
+    void ProximityInterestHandler::ProduceChanges(const InterestMatchResult& before, const InterestMatchResult& after)
+    {
+        m_resultCache.clear();
+
+#if defined(INTERNAL_DEBUG_PROXIMITY)
+        before.PrintMatchResult("before");
+        after.PrintMatchResult("after");
+#endif
+
+        /*
+         * 'after' contains only the stuff that might have changed
+         */
+        for (auto& possiblyDirty : after)
+        {
+            ReplicaId repId = possiblyDirty.first;
+            const InterestPeerSet& peerSet = possiblyDirty.second;
+
+            auto foundInBefore = before.find(repId);
+            if (foundInBefore != before.end())
+            {
+                if (!HasSamePeers(foundInBefore->second, peerSet))
+                {
+                    // was in the last calculation but has a different peer set now
+                    m_resultCache.insert(AZStd::make_pair(repId, peerSet));
+                }
+            }
+            else
+            {
+                // since it wasn't present during last calculation
+                m_resultCache.insert(AZStd::make_pair(repId, peerSet));
+            }
+        }
+
+        // Mark attributes (replicas) for removal that have not moved but a rule (clients) no longer sees it
+        for (auto& possiblyDirty : before)
+        {
+            ReplicaId repId = possiblyDirty.first;
+
+            const auto foundInAfter = after.find(repId);
+            /*
+             * If the prior state was a replica A present on peer X: "A{X}", and now A should no longer be present on any peer: "A{}"
+             * then by the rules of InterestHandlers interacting with InterestManager, we should return in @m_resultCache the following:
+             *
+             * A{} - indicating that replica A must be removed all peers.
+             *
+             * On the next pass, the prior state would be: "A{}" and the current state would be "A{}" as well. At that point, we have
+             * already sent the update to remove A from X, so @m_resultCache should no longer mention A at all.
+             */
+            if (foundInAfter == after.end() && !possiblyDirty.second.empty() /* "not A{}" see the above comment */)
+            {
+                m_resultCache.insert(AZStd::make_pair(repId, InterestPeerSet()));
+            }
+        }
+
+#if defined(INTERNAL_DEBUG_PROXIMITY)
+        m_resultCache.PrintMatchResult("changes");
+#endif
+
+    }
+
+    bool ProximityInterestHandler::HasSamePeers(const InterestPeerSet& one, const InterestPeerSet& another)
+    {
+        if (one.size() != another.size())
+        {
+            return false;
+        }
+
+        for (auto& peerFromOne : one)
+        {
+            if (another.find(peerFromOne) == another.end())
+            {
+                return false;
+            }
+        }
+
+        // Safe to assume it's the same sets since all entries are unique in a peer sets
+        return true;
+    }
+
+    void ProximityInterestHandler::Update()
+    {
+        InterestMatchResult newResult;
+
+        UpdateInternal(newResult);
+        ProduceChanges(m_lastResult, newResult);
+
+        m_lastResult = std::move(newResult);
+        ClearDirtyState();
+    }
+
+    void ProximityInterestHandler::OnRulesHandlerRegistered(InterestManager* manager)
+    {
+        AZ_Assert(m_im == nullptr, "Handler is already registered with manager %p (%p)\n", m_im, manager);
+        AZ_Assert(m_rulesReplica == nullptr, "Rules replica is already created\n");
+        AZ_TracePrintf("GridMate", "Proximity interest handler is registered\n");
+        m_im = manager;
+        m_rm = m_im->GetReplicaManager();
+        m_rm->RegisterUserContext(AZ_CRC("ProximityInterestHandler", 0x3a90b3e4), this);
+
+        auto replica = Replica::CreateReplica("ProximityInterestHandlerRules");
+        m_rulesReplica = CreateAndAttachReplicaChunk<ProximityInterestChunk>(replica);
+        m_rm->AddMaster(replica);
+    }
+
+    void ProximityInterestHandler::OnRulesHandlerUnregistered(InterestManager* manager)
+    {
+        (void)manager;
+        AZ_Assert(m_im == manager, "Handler was not registered with manager %p (%p)\n", manager, m_im);
+        AZ_TracePrintf("GridMate", "Proximity interest handler is unregistered\n");
+        m_rulesReplica = nullptr;
+        m_im = nullptr;
+        m_rm->UnregisterUserContext(AZ_CRC("ProximityInterestHandler", 0x3a90b3e4));
+        m_rm = nullptr;
+
+        for (auto& chunk : m_peerChunks)
+        {
+            chunk.second->m_interestHandler = nullptr;
+        }
+        m_peerChunks.clear();
+
+        ClearDirtyState();
+        DestroyAll();
+
+        m_resultCache.clear();
+    }
+
+    void ProximityInterestHandler::DestroyAll()
+    {
+        for (ProximityInterestRule* rule : m_localRules)
+        {
+            FreeRule(rule);
+        }
+        m_localRules.clear();
+
+        for (ProximityInterestAttribute* attr : m_attributes)
+        {
+            FreeAttribute(attr);
+        }
+        m_attributes.clear();
+
+        for (auto& removedRule : m_removedRules)
+        {
+            FreeRule(removedRule);
+        }
+        m_removedRules.clear();
+
+        for (auto& removedAttribute : m_removedAttributes)
+        {
+            FreeAttribute(removedAttribute);
+        }
+        m_removedAttributes.clear();
+    }
+
+    ///////////////////////////////////////////////////////////////////////////
+    ProximityInterestHandler::~ProximityInterestHandler()
+    {
+        /*
+         * If a handler was registered with a InterestManager, then InterestManager ought to have called OnRulesHandlerUnregistered
+         * but this is a safety pre-caution.
+         */
+        DestroyAll();
+    }
+
+    SpatialIndex::SpatialIndex()
+    {
+        m_tree.reset(aznew GridMate::BvDynamicTree());
+    }
+
+    void SpatialIndex::Remove(Node* node)
+    {
+        m_tree->Remove(node);
+    }
+
+    void SpatialIndex::Update(Node* node)
+    {
+        m_tree->Update(node);
+    }
+
+    SpatialIndex::Node* SpatialIndex::Insert(const AZ::Aabb& get, ProximityInterestAttribute* attribute)
+    {
+        return m_tree->Insert(get, attribute);
+    }
+
+    void SpatialIndex::Query(const AZ::Aabb& shape, NodeCollector& nodes)
+    {
+        m_tree->collideTV(m_tree->GetRoot(), shape, nodes);
+    }
+}

Code/Framework/GridMate/GridMate/Replica/Interest/ProximityInterestHandler.h

@@ -0,0 +1,314 @@
+/*
+* All or portions of this file Copyright (c) Amazon.com, Inc. or its affiliates or
+* its licensors.
+*
+* For complete copyright and license terms please see the LICENSE at the root of this
+* distribution (the "License"). All use of this software is governed by the License,
+* or, if provided, by the license below or the license accompanying this file. Do not
+* remove or modify any license notices. This file is distributed on an "AS IS" BASIS,
+* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+*
+*/
+
+#ifndef GM_REPLICA_PROXIMITYINTERESTHANDLER_H
+#define GM_REPLICA_PROXIMITYINTERESTHANDLER_H
+
+#include <GridMate/Replica/RemoteProcedureCall.h>
+#include <GridMate/Replica/ReplicaChunk.h>
+#include <GridMate/Replica/Interest/RulesHandler.h>
+#include <GridMate/Replica/Interest/BvDynamicTree.h>
+#include <GridMate/Serialize/UtilityMarshal.h>
+
+#include <AzCore/Math/Aabb.h>
+#include <AzCore/std/smart_ptr/unique_ptr.h>
+
+namespace GridMate
+{
+    class ProximityInterestHandler;
+    class ProximityInterestAttribute;
+
+    /*
+    * Base interest
+    */
+    class ProximityInterest
+    {
+        friend class ProximityInterestHandler;
+
+    public:
+        const AZ::Aabb& Get() const { return m_bbox; }
+
+    protected:
+        explicit ProximityInterest(ProximityInterestHandler* handler);
+
+        ProximityInterestHandler* m_handler;
+        AZ::Aabb m_bbox;
+    };
+    ///////////////////////////////////////////////////////////////////////////
+
+
+    /*
+    * Proximity rule
+    */
+    class ProximityInterestRule
+        : public InterestRule
+        , public ProximityInterest
+    {
+        friend class ProximityInterestHandler;
+
+    public:
+        using Ptr = AZStd::intrusive_ptr<ProximityInterestRule>;
+
+        GM_CLASS_ALLOCATOR(ProximityInterestRule);
+
+        void Set(const AZ::Aabb& bbox);
+
+    private:
+
+        // Intrusive ptr
+        template<class T>
+        friend struct AZStd::IntrusivePtrCountPolicy;
+        unsigned int m_refCount = 0;
+        AZ_FORCE_INLINE void add_ref() { ++m_refCount; }
+        AZ_FORCE_INLINE void release() { --m_refCount; if (!m_refCount) Destroy(); }
+        AZ_FORCE_INLINE bool IsDeleted() const { return m_refCount == 0; }
+        ///////////////////////////////////////////////////////////////////////////
+
+        ProximityInterestRule(ProximityInterestHandler* handler, PeerId peerId, RuleNetworkId netId)
+            : InterestRule(peerId, netId)
+            , ProximityInterest(handler)
+        {}
+
+        void Destroy();
+    };
+    ///////////////////////////////////////////////////////////////////////////
+
+    class SpatialIndex
+    {
+    public:
+        typedef Internal::DynamicTreeNode Node;
+
+        class NodeCollector
+        {
+            typedef AZStd::vector<ProximityInterestAttribute*> Type;
+
+        public:
+            void Process(const Internal::DynamicTreeNode* node)
+            {
+                m_nodes.push_back(reinterpret_cast<ProximityInterestAttribute*>(node->m_data));
+            }
+
+            const Type& GetNodes() const
+            {
+                return m_nodes;
+            }
+
+        private:
+            Type m_nodes;
+        };
+
+        SpatialIndex();
+        ~SpatialIndex() = default;
+
+        AZ_FORCE_INLINE void Remove(Node* node);
+        AZ_FORCE_INLINE void Update(Node* node);
+        AZ_FORCE_INLINE Node* Insert(const AZ::Aabb& get, ProximityInterestAttribute* attribute);
+        AZ_FORCE_INLINE void Query(const AZ::Aabb& get, NodeCollector& nodes);
+
+    private:
+        AZStd::unique_ptr<BvDynamicTree> m_tree;
+    };
+
+    /*
+    * Proximity attribute
+    */
+    class ProximityInterestAttribute
+        : public InterestAttribute
+        , public ProximityInterest
+    {
+        friend class ProximityInterestHandler;
+        template<class T> friend class InterestPtr;
+
+    public:
+        using Ptr = AZStd::intrusive_ptr<ProximityInterestAttribute>;
+
+        GM_CLASS_ALLOCATOR(ProximityInterestAttribute);
+
+        void Set(const AZ::Aabb& bbox);
+
+    private:
+
+        // Intrusive ptr
+        template<class T>
+        friend struct AZStd::IntrusivePtrCountPolicy;
+        unsigned int m_refCount = 0;
+        AZ_FORCE_INLINE void add_ref() { ++m_refCount; }
+        AZ_FORCE_INLINE void release() { Destroy(); }
+        AZ_FORCE_INLINE bool IsDeleted() const { return m_refCount == 0; }
+        ///////////////////////////////////////////////////////////////////////////
+
+        ProximityInterestAttribute(ProximityInterestHandler* handler, ReplicaId repId)
+            : InterestAttribute(repId)
+            , ProximityInterest(handler)
+            , m_worldNode(nullptr)
+        {}
+
+        void Destroy();
+
+        void SetNode(SpatialIndex::Node* node) { m_worldNode = node; }
+        SpatialIndex::Node* GetNode() const { return m_worldNode; }
+        SpatialIndex::Node* m_worldNode; ///< non-owning pointer
+    };
+    ///////////////////////////////////////////////////////////////////////////
+
+    class ProximityInterestChunk
+        : public ReplicaChunk
+    {
+    public:
+        GM_CLASS_ALLOCATOR(ProximityInterestChunk);
+
+        // ReplicaChunk
+        typedef AZStd::intrusive_ptr<ProximityInterestChunk> Ptr;
+        bool IsReplicaMigratable() override { return false; }
+        bool IsBroadcast() override { return true; }
+        static const char* GetChunkName() { return "ProximityInterestChunk"; }
+
+        ProximityInterestChunk()
+            : AddRuleRpc("AddRule")
+            , RemoveRuleRpc("RemoveRule")
+            , UpdateRuleRpc("UpdateRule")
+            , AddRuleForPeerRpc("AddRuleForPeerRpc")
+            , m_interestHandler(nullptr)
+        {
+        }
+
+        void OnReplicaActivate(const ReplicaContext& rc) override;
+        void OnReplicaDeactivate(const ReplicaContext& rc) override;
+
+        bool AddRuleFn(RuleNetworkId netId, AZ::Aabb bbox, const RpcContext& ctx);
+        bool RemoveRuleFn(RuleNetworkId netId, const RpcContext&);
+        bool UpdateRuleFn(RuleNetworkId netId, AZ::Aabb bbox, const RpcContext&);
+        bool AddRuleForPeerFn(RuleNetworkId netId, PeerId peerId, AZ::Aabb bbox, const RpcContext&);
+
+        Rpc<RpcArg<RuleNetworkId>, RpcArg<AZ::Aabb>>::BindInterface<ProximityInterestChunk, &ProximityInterestChunk::AddRuleFn> AddRuleRpc;
+        Rpc<RpcArg<RuleNetworkId>>::BindInterface<ProximityInterestChunk, &ProximityInterestChunk::RemoveRuleFn> RemoveRuleRpc;
+        Rpc<RpcArg<RuleNetworkId>, RpcArg<AZ::Aabb>>::BindInterface<ProximityInterestChunk, &ProximityInterestChunk::UpdateRuleFn> UpdateRuleRpc;
+
+        Rpc<RpcArg<RuleNetworkId>, RpcArg<PeerId>, RpcArg<AZ::Aabb>>::BindInterface<ProximityInterestChunk, &ProximityInterestChunk::AddRuleForPeerFn> AddRuleForPeerRpc;
+
+        unordered_map<RuleNetworkId, ProximityInterestRule::Ptr> m_rules;
+        ProximityInterestHandler* m_interestHandler;
+    };
+
+    /*
+    * Rules handler
+    */
+    class ProximityInterestHandler
+        : public BaseRulesHandler
+    {
+        friend class ProximityInterestRule;
+        friend class ProximityInterestAttribute;
+        friend class ProximityInterestChunk;
+
+    public:
+
+        typedef unordered_set<ProximityInterestAttribute*> AttributeSet;
+        typedef unordered_set<ProximityInterestRule*> RuleSet;
+
+        GM_CLASS_ALLOCATOR(ProximityInterestHandler);
+
+        ProximityInterestHandler();
+        ~ProximityInterestHandler();
+
+        /*
+         * Creates new proximity rule and binds it to the peer.
+         * Note: the lifetime of the created rule is tied to the lifetime of this handler.
+         */
+        ProximityInterestRule::Ptr CreateRule(PeerId peerId);
+
+        /*
+         * Creates new proximity attribute and binds it to the replica.
+         * Note: the lifetime of the created attribute is tied to the lifetime of this handler.
+         */
+        ProximityInterestAttribute::Ptr CreateAttribute(ReplicaId replicaId);
+
+        // Calculates rules and attributes matches
+        void Update() override;
+
+        // Returns last recalculated results
+        const InterestMatchResult& GetLastResult() override;
+
+        // Returns the manager it's bound to
+        InterestManager* GetManager() override { return m_im; }
+
+        // Rules that this handler is aware of
+        const RuleSet& GetLocalRules() const { return m_localRules; }
+
+    private:
+
+        // BaseRulesHandler
+        void OnRulesHandlerRegistered(InterestManager* manager) override;
+        void OnRulesHandlerUnregistered(InterestManager* manager) override;
+
+        void DestroyRule(ProximityInterestRule* rule);
+        void FreeRule(ProximityInterestRule* rule);
+        void UpdateRule(ProximityInterestRule* rule);
+
+        void DestroyAttribute(ProximityInterestAttribute* attrib);
+        void FreeAttribute(ProximityInterestAttribute* attrib);
+        void UpdateAttribute(ProximityInterestAttribute* attrib);
+
+        void OnNewRulesChunk(ProximityInterestChunk* chunk, ReplicaPeer* peer);
+        void OnDeleteRulesChunk(ProximityInterestChunk* chunk, ReplicaPeer* peer);
+
+        RuleNetworkId GetNewRuleNetId();
+
+        ProximityInterestChunk* FindRulesChunkByPeerId(PeerId peerId);
+
+        void DestroyAll();
+
+        InterestManager* m_im;
+        ReplicaManager* m_rm;
+
+        AZ::u32 m_lastRuleNetId;
+
+        unordered_map<PeerId, ProximityInterestChunk*> m_peerChunks;
+
+        RuleSet m_localRules;
+        RuleSet m_removedRules;
+        RuleSet m_dirtyRules;
+
+        AttributeSet m_attributes;
+        AttributeSet m_removedAttributes;
+        AttributeSet m_dirtyAttributes;
+
+        ProximityInterestChunk* m_rulesReplica;
+
+        // collection of all known attributes
+        AZStd::unique_ptr<SpatialIndex> m_attributeWorld;
+
+        InterestMatchResult m_resultCache;
+
+        ///////////////////////////////////////////////////////////////////////////////////////////////////
+        // internal processing helpers
+        AZ_FORCE_INLINE RuleSet& GetAffectedRules();
+        AZ_FORCE_INLINE void GetAttributesWithinRule(ProximityInterestRule* rule, SpatialIndex::NodeCollector& nodes);
+        AZ_FORCE_INLINE void ClearDirtyState();
+
+        AZ_FORCE_INLINE void CreateAndInsertIntoSpatialStructure(ProximityInterestAttribute* attribute);
+        AZ_FORCE_INLINE void RemoveFromSpatialStructure(ProximityInterestAttribute* attribute);
+        AZ_FORCE_INLINE void CreateAndInsertIntoSpatialStructure(ProximityInterestRule* rule);
+
+        void UpdateInternal(InterestMatchResult& result);
+        void CheckChangesForRule(ProximityInterestRule* rule, InterestMatchResult& result);
+        void MarkAttributesDirtyInRule(ProximityInterestRule* rule);
+
+        static bool HasSamePeers(const InterestPeerSet& one, const InterestPeerSet& another);
+        void ProduceChanges(const InterestMatchResult& before, const InterestMatchResult& after);
+
+        InterestMatchResult m_lastResult;
+        ///////////////////////////////////////////////////////////////////////////////////////////////////
+    };
+    ///////////////////////////////////////////////////////////////////////////
+}
+
+#endif // GM_REPLICA_PROXIMITYINTERESTHANDLER_H

Code/Framework/GridMate/GridMate/Replica/Interest/RulesHandler.h

@@ -0,0 +1,69 @@
+/*
+* All or portions of this file Copyright (c) Amazon.com, Inc. or its affiliates or
+* its licensors.
+*
+* For complete copyright and license terms please see the LICENSE at the root of this
+* distribution (the "License"). All use of this software is governed by the License,
+* or, if provided, by the license below or the license accompanying this file. Do not
+* remove or modify any license notices. This file is distributed on an "AS IS" BASIS,
+* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+*
+*/
+#ifndef GM_REPLICA_RULES_HANDLER_H
+#define GM_REPLICA_RULES_HANDLER_H
+
+#include <GridMate/Replica/ReplicaCommon.h>
+
+#include <GridMate/Replica/Interest/InterestDefs.h>
+
+namespace GridMate
+{
+    class InterestManager;
+
+    /**
+    *  BaseRulesHandler: base handler class
+    *  RulesHandler's job is to provide InterestManager with matching pairs of attributes and rules.
+    */
+    class BaseRulesHandler
+    {
+    public:
+        BaseRulesHandler()
+            : m_slot(0)
+        {}
+
+        virtual ~BaseRulesHandler() { };
+
+        /**
+        *  Ticked by interest manager to retrieve new matches or mismatches of interests
+        */
+        virtual void Update() = 0;
+
+        /**
+        *  Returns result of a previous update
+        *  This only returns changes that happened on the previous tick not the whole world state
+        */
+        virtual const InterestMatchResult& GetLastResult() = 0;
+
+        /**
+        *  Called by InterestManager when the given handler instance is registered
+        */
+        virtual void OnRulesHandlerRegistered(InterestManager* manager) = 0;
+
+        /**
+        *  Called by InterestManager when the given handler is unregistered
+        */
+        virtual void OnRulesHandlerUnregistered(InterestManager* manager) = 0;
+
+        /**
+        *  Returns interest mananger this handler is bound to, or nullptr if it's unbound
+        */
+        virtual InterestManager* GetManager() = 0;
+
+    private:
+        friend class InterestManager;
+
+        InterestHandlerSlot m_slot;
+    };
+} // namespace GridMate
+
+#endif // GM_REPLICA_RULES_HANDLER_H

Code/Framework/GridMate/GridMate/Replica/Interpolators.h

@@ -0,0 +1,369 @@
+/*
+* All or portions of this file Copyright (c) Amazon.com, Inc. or its affiliates or
+* its licensors.
+*
+* For complete copyright and license terms please see the LICENSE at the root of this
+* distribution (the "License"). All use of this software is governed by the License,
+* or, if provided, by the license below or the license accompanying this file. Do not
+* remove or modify any license notices. This file is distributed on an "AS IS" BASIS,
+* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+*
+*/
+#ifndef GM_INTERPOLATOR_H
+#define GM_INTERPOLATOR_H
+
+#include <AzCore/Math/MathUtils.h>
+#include <AzCore/Debug/Trace.h>
+#include <AzCore/std/chrono/chrono.h>
+#include <AzCore/std/containers/array.h>
+
+namespace GridMate
+{
+    template<typename T>
+    struct SampleInfo
+    {
+        T m_v;
+        unsigned int m_t;
+        bool m_cantBreak;
+    };
+
+    template<typename T>
+    struct SimpleValueInterpolator
+    {
+        /// \param time is [0.0,1.0]
+        static T Interpolate(const T& from, const T& to, float time) { return T(from + (to - from) * time); }
+    };
+
+    //-----------------------------------------------------------------------------
+    // Interpolators
+    //-----------------------------------------------------------------------------
+    //-----------------------------------------------------------------------------
+    // PointSample
+    //-----------------------------------------------------------------------------
+    template<typename T, int k_maxSamples = 8>
+    class PointSample
+    {
+    public:
+        struct Sample
+        {
+            T m_v;
+            unsigned int m_t;
+        };
+
+
+        PointSample()
+            : m_curIdx(-1)
+            , m_count(0) { }
+
+        void AddSample(const T& sample, unsigned int time)
+        {
+            // discard old data points
+            if (m_count > 0 && m_samples[m_curIdx].m_t > time)
+            {
+                return;
+            }
+            // only add new point if time moved forward
+            if (m_count == 0 || time > m_samples[m_curIdx].m_t)
+            {
+                m_curIdx = (m_curIdx + 1) % k_maxSamples;
+                m_count = AZ::GetMin(m_count + 1, k_maxSamples);
+            }
+            m_samples[m_curIdx].m_v = sample;
+            m_samples[m_curIdx].m_t = time;
+        }
+
+        T GetInterpolatedValue(unsigned int time) const
+        {
+            AZ_Assert(m_count > 0, "No samples available.");
+            int first = (m_curIdx + k_maxSamples - m_count + 1) % k_maxSamples;
+            int beyond = (m_curIdx + 1) % k_maxSamples;
+            for (int i = (first + 1) % k_maxSamples; i != beyond; i = (i + 1) % k_maxSamples)
+            {
+                if (m_samples[i].m_t > time)
+                {
+                    return m_samples[first].m_v;
+                }
+                first = i;
+            }
+            return m_samples[first].m_v;
+        }
+
+        T GetLastValue() const
+        {
+            AZ_Assert(m_count > 0, "No samples available.");
+            return m_samples[m_curIdx].m_v;
+        }
+
+        void Break() { }
+        void Clear() { m_curIdx = -1; m_count = 0; }
+
+        // Debug info
+        int GetSampleCount() const { return m_count; }
+        SampleInfo<T> GetSampleInfo(int i) const
+        {
+            AZ_Assert(i >= 0 && i < m_count, "Out of bounds.");
+            int _i = (m_curIdx + k_maxSamples - m_count + 1 + i) % k_maxSamples;
+            SampleInfo<T> info;
+            info.m_v = m_samples[_i].m_v;
+            info.m_t = m_samples[_i].m_t;
+            info.m_cantBreak = false;
+            return info;
+        }
+
+    private:
+        AZStd::array<Sample, k_maxSamples> m_samples;
+        int m_curIdx, m_count;
+    };
+    //-----------------------------------------------------------------------------
+
+    //-----------------------------------------------------------------------------
+    // LinearInterp
+    //-----------------------------------------------------------------------------
+    template<typename T, int k_maxSamples = 8, typename Interpolator = SimpleValueInterpolator<T> >
+    class LinearInterp
+    {
+    public:
+        struct Sample
+        {
+            T m_v;
+            unsigned int m_t;
+            bool m_cantBreak;
+        };
+
+        LinearInterp()
+            : m_curIdx(-1)
+            , m_count(0)
+            , m_cantBreak(false) { }
+
+        void AddSample(const T& sample, unsigned int time)
+        {
+            // discard old data points
+            if (m_count > 0 && m_samples[m_curIdx].m_t > time)
+            {
+                return;
+            }
+            // only add new point if time moved forward
+            if (m_count == 0 || time > m_samples[m_curIdx].m_t)
+            {
+                m_curIdx = (m_curIdx + 1) % k_maxSamples;
+                m_count = AZ::GetMin(m_count + 1, k_maxSamples);
+            }
+            m_samples[m_curIdx].m_v = sample;
+            m_samples[m_curIdx].m_t = time;
+            m_samples[m_curIdx].m_cantBreak = m_cantBreak;
+            m_cantBreak = false;
+        }
+
+        T GetInterpolatedValue(unsigned int time) const
+        {
+            AZ_Assert(m_count > 0, "No samples available.");
+            int first = (m_curIdx + k_maxSamples - m_count + 1) % k_maxSamples;
+            //int last = (curIdx + maxSamples - count - 1) % maxSamples;
+            int beyond = (m_curIdx + 1) % k_maxSamples;
+            if (time < m_samples[first].m_t)
+            {
+                return m_samples[first].m_v;
+            }
+            float interval, t;
+            T v;
+            for (int second = (first + 1) % k_maxSamples; second != beyond; second = (second + 1) % k_maxSamples)
+            {
+                if (m_samples[second].m_t > time)
+                {
+                    // interpolation
+                    if (m_samples[second].m_cantBreak)
+                    {
+                        v = m_samples[first].m_v;
+                    }
+                    else
+                    {
+                        interval = float(m_samples[second].m_t - m_samples[first].m_t);
+                        AZ_Assert(interval > 0.001f, "non-incrementing timestamps!");
+                        t = float(time - m_samples[first].m_t) / interval;
+                        AZ_Assert(t >= 0.f && t < 1.f, "we should be interpolating!");
+                        v = Interpolator::Interpolate(m_samples[first].m_v, m_samples[second].m_v, t);
+                    }
+                    return v;
+                }
+                first = second;
+            }
+            // return last value
+            return m_samples[first].m_v;
+        }
+
+        const T& GetLastValue() const
+        {
+            AZ_Assert(m_count > 0, "No samples available.");
+            return m_samples[m_curIdx].m_v;
+        }
+
+        const Sample& GetLastSample() const
+        {
+            AZ_Assert(m_count > 0, "No samples available.");
+            return m_samples[m_curIdx];
+        }
+
+        const Sample& GetFirstSample() const
+        {
+            AZ_Assert(m_count > 0, "No samples available.");
+            int first = (m_curIdx + k_maxSamples - m_count + 1) % k_maxSamples;
+            return m_samples[first];
+        }
+
+        void Break() { m_cantBreak = true; }
+        void Clear() { m_curIdx = -1; m_count = 0; m_cantBreak = false; }
+
+        // Debug info
+        int GetSampleCount() const { return m_count; }
+        SampleInfo<T> GetSampleInfo(int i) const
+        {
+            AZ_Assert(i >= 0 && i < m_count, "Out of bounds.");
+            int _i = (m_curIdx + k_maxSamples - m_count + 1 + i) % k_maxSamples;
+            SampleInfo<T> info;
+            info.m_v = m_samples[_i].m_v;
+            info.m_t = m_samples[_i].m_t;
+            info.m_cantBreak = m_samples[_i].m_cantBreak;
+            return info;
+        }
+
+    private:
+        AZStd::array<Sample, k_maxSamples> m_samples;
+        int m_curIdx, m_count;
+        bool m_cantBreak;
+    };
+    //-----------------------------------------------------------------------------
+
+    //-----------------------------------------------------------------------------
+    // LinearInterpExtrap
+    // Interpolates/Extrapolates using the 2 closest samples
+    //-----------------------------------------------------------------------------
+    template<typename T, int k_maxSamples = 8, typename Interpolator = SimpleValueInterpolator<T> >
+    class LinearInterpExtrap
+    {
+    public:
+        struct Sample
+        {
+            T m_v;
+            unsigned int m_t;
+            bool m_cantBreak;
+        };
+
+        LinearInterpExtrap()
+            : m_curIdx(-1)
+            , m_count(0)
+            , m_cantBreak(false) { }
+
+        void AddSample(const T& sample, unsigned int time)
+        {
+            // discard old data points
+            if (m_count > 0 && m_samples[m_curIdx].m_t > time)
+            {
+                return;
+            }
+            // only add new point if time moved forward
+            if (m_count == 0 || time > m_samples[m_curIdx].m_t)
+            {
+                m_curIdx = (m_curIdx + 1) % k_maxSamples;
+                m_count = AZ::GetMin(m_count + 1, k_maxSamples);
+            }
+            m_samples[m_curIdx].m_v = sample;
+            m_samples[m_curIdx].m_t = time;
+            m_samples[m_curIdx].m_cantBreak = m_cantBreak;
+            m_cantBreak = false;
+        }
+
+        T GetInterpolatedValue(unsigned int time) const
+        {
+            AZ_Assert(m_count > 0, "No samples available.");
+            if (m_count == 1 || k_maxSamples == 1)
+            {
+                return m_samples[0].m_v;
+            }
+            int first = (m_curIdx + k_maxSamples - m_count + 1) % k_maxSamples;
+            int beyond = (m_curIdx + 1) % k_maxSamples;
+            if (time < m_samples[first].m_t)
+            {
+                return m_samples[first].m_v;
+            }
+            int second, previous = 0;
+            float interval, t;
+            T v;
+            for (second = (first + 1) % k_maxSamples; second != beyond; second = (second + 1) % k_maxSamples)
+            {
+                if (m_samples[second].m_t > time)
+                {
+                    // interpolation
+                    if (m_samples[second].m_cantBreak)
+                    {
+                        v = m_samples[first].m_v;
+                    }
+                    else
+                    {
+                        interval = static_cast<float>(m_samples[second].m_t - m_samples[first].m_t);
+                        AZ_Assert(interval > 0.001f, "non-incrementing timestamps!");
+                        t = (time - m_samples[first].m_t) / interval;
+                        AZ_Assert(t >= 0.f && t < 1.f, "we should be interpolating!");
+                        v = Interpolator::Interpolate(m_samples[first].m_v, m_samples[second].m_v, t);
+                    }
+                    return v;
+                }
+                previous = first;
+                first = second;
+            }
+            // extrapolation
+            second = first;
+            first = previous;
+            if (m_samples[second].m_cantBreak)
+            {
+                v = m_samples[second].m_v;
+            }
+            else
+            {
+                interval = static_cast<float>(m_samples[second].m_t - m_samples[first].m_t);
+                AZ_Assert(interval > 0.001f, "non-incrementing timestamps!");
+                t = (time - m_samples[first].m_t) / interval;
+                AZ_Assert(t > 0.99f, "we should be extrapolating!");
+                v = Interpolator::Interpolate(m_samples[first].m_v, m_samples[second].m_v, t);
+            }
+            return v;
+        }
+
+        void Break()
+        {
+            m_cantBreak = true;
+        }
+
+        T GetLastValue() const
+        {
+            AZ_Assert(m_count > 0, "No samples available.");
+            return m_samples[m_curIdx].m_v;
+        }
+
+        void Clear()
+        {
+            m_count = 0;
+            m_curIdx = -1;
+            m_cantBreak = false;
+        }
+
+        // Debug info
+        int GetSampleCount() const { return m_count; }
+        SampleInfo<T> GetSampleInfo(int i) const
+        {
+            AZ_Assert(i >= 0 && i < m_count, "Out of bounds.");
+            int _i = (m_curIdx + k_maxSamples - m_count + 1 + i) % k_maxSamples;
+            SampleInfo<T> info;
+            info.m_v = m_samples[_i].m_v;
+            info.m_t = m_samples[_i].m_t;
+            info.m_cantBreak = m_samples[_i].m_cantBreak;
+            return info;
+        }
+    private:
+        AZStd::array<Sample, k_maxSamples> m_samples;
+        int m_curIdx, m_count;
+        bool m_cantBreak;
+    };
+    //-----------------------------------------------------------------------------
+} // namespace GridMate
+
+#endif // GM_INTERPOLATOR_H

Code/Framework/GridMate/GridMate/Replica/MigrationSequence.cpp

@@ -0,0 +1,420 @@
+/*
+* All or portions of this file Copyright (c) Amazon.com, Inc. or its affiliates or
+* its licensors.
+*
+* For complete copyright and license terms please see the LICENSE at the root of this
+* distribution (the "License"). All use of this software is governed by the License,
+* or, if provided, by the license below or the license accompanying this file. Do not
+* remove or modify any license notices. This file is distributed on an "AS IS" BASIS,
+* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+*
+*/
+#include <GridMate/Replica/MigrationSequence.h>
+#include <GridMate/Replica/ReplicaStatus.h>
+
+namespace GridMate
+{
+    namespace ReplicaInternal
+    {
+        //-----------------------------------------------------------------------------
+        //-----------------------------------------------------------------------------
+        MigrationSequence::MigrationSequence(Replica* replica, PeerId newOwnerId)
+            : m_replica(replica)
+            , m_newOwnerId(newOwnerId)
+        {
+            ReplicaMgrCallbackBus::Handler::BusConnect(replica->GetReplicaManager()->GetGridMate());
+
+            m_replicaMgr = replica->GetReplicaManager();
+
+            if (replica->IsMaster() && newOwnerId != m_replicaMgr->GetLocalPeerId())
+            {
+                m_sm.SetStateHandler(AZ_HSM_STATE_NAME(MST_TOP), AZ::HSM::StateHandler(this, &MigrationSequence::DefaultHandler), AZ::HSM::InvalidStateId, MST_MIGRATING);
+            }
+            else
+            {
+                m_sm.SetStateHandler(AZ_HSM_STATE_NAME(MST_TOP), AZ::HSM::StateHandler(this, &MigrationSequence::DefaultHandler), AZ::HSM::InvalidStateId, MST_CHANGE_ROUTING_ONLY);
+            }
+            m_sm.SetStateHandler(AZ_HSM_STATE_NAME(MST_MIGRATING), AZ::HSM::StateHandler(this, &MigrationSequence::OnStateMigrating), MST_TOP, MST_FLUSH_UPSTREAM);
+            m_sm.SetStateHandler(AZ_HSM_STATE_NAME(MST_FLUSH_UPSTREAM), AZ::HSM::StateHandler(this, &MigrationSequence::OnStateFlushUpstream), MST_MIGRATING);
+            m_sm.SetStateHandler(AZ_HSM_STATE_NAME(MST_FLUSH_DOWNSTREAM), AZ::HSM::StateHandler(this, &MigrationSequence::OnStateFlushDownstream), MST_MIGRATING);
+            m_sm.SetStateHandler(AZ_HSM_STATE_NAME(MST_CHANGE_ROUTING_FOR_MIGRATION), AZ::HSM::StateHandler(this, &MigrationSequence::OnStateChangeRouting), MST_MIGRATING);
+            m_sm.SetStateHandler(AZ_HSM_STATE_NAME(MST_HANDOFF_REPLICA), AZ::HSM::StateHandler(this, &MigrationSequence::OnStateHandoffReplica), MST_MIGRATING);
+            m_sm.SetStateHandler(AZ_HSM_STATE_NAME(MST_ROLLBACK), AZ::HSM::StateHandler(this, &MigrationSequence::OnStateRollback), MST_TOP);
+            m_sm.SetStateHandler(AZ_HSM_STATE_NAME(MST_ABORT), AZ::HSM::StateHandler(this, &MigrationSequence::OnStateAbort), MST_TOP);
+            m_sm.SetStateHandler(AZ_HSM_STATE_NAME(MST_CHANGE_ROUTING_ONLY), AZ::HSM::StateHandler(this, &MigrationSequence::OnStateChangeRoutingOnly), MST_TOP);
+            m_sm.SetStateHandler(AZ_HSM_STATE_NAME(MST_IDLE), AZ::HSM::StateHandler(this, &MigrationSequence::DefaultHandler), MST_TOP);
+
+            m_sm.Start();
+        }
+        //-----------------------------------------------------------------------------
+        void MigrationSequence::Update()
+        {
+            m_sm.Dispatch(ME_UPDATE);
+        }
+        //-----------------------------------------------------------------------------
+        bool MigrationSequence::IsDone() const
+        {
+            return m_sm.IsInState(MST_IDLE);
+        }
+        //-----------------------------------------------------------------------------
+        void MigrationSequence::ModifyNewOwner(PeerId newOwnerId)
+        {
+            AZ::HSM::Event ev;
+            ev.id = ME_MODIFY_NEW_OWNER;
+            ev.userData = &newOwnerId;
+        }
+        //-----------------------------------------------------------------------------
+        bool MigrationSequence::OnStateMigrating(AZ::HSM& sm, const AZ::HSM::Event& event)
+        {
+            switch (event.id)
+            {
+            case AZ::HSM::EnterEventId:
+                //AZ_TracePrintf("GridMate", "Replica 0x%x entering migration state MST_MIGRATING.\n", m_replica->GetRepId());
+                m_pendingAcks.clear();
+                return true;
+            case AZ::HSM::ExitEventId:
+                return true;
+            case ME_PEER_REMOVED:
+                if (*static_cast<PeerId*>(event.userData) == m_newOwnerId)
+                {
+                    sm.Transition(MST_ABORT);
+                    return true;
+                }
+            // else fall through
+            case ME_PEER_ACK:
+                m_pendingAcks.erase(*static_cast<PeerId*>(event.userData));
+                return true;
+            case ME_MODIFY_NEW_OWNER:
+                m_newOwnerId = *static_cast<PeerId*>(event.userData);
+                return true;
+            default:
+                break;
+            }
+            return false;
+        }
+        //-----------------------------------------------------------------------------
+        bool MigrationSequence::OnStateFlushUpstream(AZ::HSM& sm, const AZ::HSM::Event& event)
+        {
+            switch (event.id)
+            {
+            case AZ::HSM::EnterEventId:
+                //AZ_TracePrintf("GridMate", "Replica 0x%x entering migration state MST_FLUSH_UPSTREAM.\n", m_replica->GetRepId());
+                for (auto& peerReplica : m_replicaMgr->m_peerReplicas)
+                {
+                    if (peerReplica.second->m_peerId.Get() != m_replicaMgr->GetLocalPeerId())
+                    {
+                        m_pendingAcks.insert(peerReplica.second->m_peerId.Get());
+                    }
+                }
+                AZStd::static_pointer_cast<ReplicaStatus>(m_replica->m_replicaStatus)->SetUpstreamSuspended(true);
+                m_timestamp = m_replicaMgr->GetTime().m_realTime;
+                AZStd::static_pointer_cast<ReplicaStatus>(m_replica->m_replicaStatus)->MigrationSuspendUpstream(m_replicaMgr->GetLocalPeerId(), m_timestamp);
+                return true;
+            case ME_PEER_REMOVED:
+                if (*static_cast<PeerId*>(event.userData) == m_newOwnerId)
+                {
+                    sm.Transition(MST_ABORT);
+                    return true;
+                }
+            // else fall through
+            case ME_PEER_ACK:
+                m_pendingAcks.erase(*static_cast<PeerId*>(event.userData));
+                if (m_pendingAcks.empty())
+                {
+                    sm.Transition(MST_FLUSH_DOWNSTREAM);
+                }
+                return true;
+            default:
+                break;
+            }
+            return false;
+        }
+        //-----------------------------------------------------------------------------
+        bool MigrationSequence::OnStateFlushDownstream(AZ::HSM& sm, const AZ::HSM::Event& event)
+        {
+            switch (event.id)
+            {
+            case AZ::HSM::EnterEventId:
+                //AZ_TracePrintf("GridMate", "Replica 0x%x entering migration state MST_FLUSH_DOWNSTREAM.\n", m_replica->GetRepId());
+                return true;
+            case ME_UPDATE:
+                for (auto& peerReplica : m_replicaMgr->m_peerReplicas)
+                {
+                    if (peerReplica.second->m_peerId.Get() != m_replicaMgr->GetLocalPeerId())
+                    {
+                        m_pendingAcks.insert(peerReplica.second->m_peerId.Get());
+                    }
+                }
+                m_timestamp = m_replicaMgr->GetTime().m_realTime;
+                AZStd::static_pointer_cast<ReplicaStatus>(m_replica->m_replicaStatus)->MigrationRequestDownstreamAck(m_replicaMgr->GetLocalPeerId(), m_timestamp);
+
+                // Demote the replica so no more updates are made to it
+                m_replicaMgr->ChangeReplicaOwnership(m_replica, m_replica->GetMyContext(), false);
+
+                // Move the replica to its new routing peer (which effectively disables outbound replication)
+                // This will be done in the tick to force a frame delay between demoting the replica
+                // and actually moving it to allow one last outbound send.
+                sm.Transition(MST_CHANGE_ROUTING_FOR_MIGRATION);
+                return true;
+            default:
+                break;
+            }
+            return false;
+        }
+        //-----------------------------------------------------------------------------
+        bool MigrationSequence::OnStateChangeRouting(AZ::HSM& sm, const AZ::HSM::Event& event)
+        {
+            switch (event.id)
+            {
+            case AZ::HSM::EnterEventId:
+                //AZ_TracePrintf("GridMate", "Replica 0x%x entering migration state MST_CHANGE_ROUTING_FOR_MIGRATION.\n", m_replica->GetRepId());
+                return true;
+            case ME_UPDATE:
+            {
+                if (m_replica->IsSuspendDownstream()) // wait until downstream suspention command is sent to everyone
+                {
+                    return true;
+                }
+                if (UpdateReplicaRouting())
+                {
+                    m_replicaMgr->UpdateReplicaTargets(m_replica);
+                    sm.Transition(MST_HANDOFF_REPLICA);
+                }
+                else
+                {
+                    AZ_Warning("GridMate", false, "Replica Migration: Can't find new next hop for the replica! Aborting migration.");
+                    sm.Transition(MST_ROLLBACK);
+                }
+                return true;
+            }
+            case ME_PEER_REMOVED:
+                if (*static_cast<PeerId*>(event.userData) == m_newOwnerId)
+                {
+                    sm.Transition(MST_ROLLBACK);
+                    return true;
+                }
+                return false;
+            default:
+                break;
+            }
+            return false;
+        }
+        //-----------------------------------------------------------------------------
+        bool MigrationSequence::OnStateHandoffReplica(AZ::HSM& sm, const AZ::HSM::Event& event)
+        {
+            switch (event.id)
+            {
+            case AZ::HSM::EnterEventId:
+                //AZ_TracePrintf("GridMate", "Replica 0x%x entering migration state MST_HANDOFF_REPLICA.\n", m_replica->GetRepId());
+                return true;
+            case ME_PEER_REMOVED:
+                if (*static_cast<PeerId*>(event.userData) == m_newOwnerId)
+                {
+                    sm.Transition(MST_ROLLBACK);
+                    return true;
+                }
+            case ME_PEER_ACK:
+                m_pendingAcks.erase(*static_cast<PeerId*>(event.userData));
+                break;
+            case ME_UPDATE:
+                break;
+            default:
+                return false;
+            }
+
+            // If we received all the necessary acks, it's time to actually
+            // handoff the replica and complete the migration.
+            // This is done via an OOB message to all the peers
+            if (m_pendingAcks.empty())
+            {
+                m_replicaMgr->AnnounceReplicaMigrated(m_replica->GetRepId(), m_newOwnerId);
+                sm.Transition(MST_IDLE);
+            }
+            return true;
+        }
+        //-----------------------------------------------------------------------------
+        bool MigrationSequence::OnStateAbort(AZ::HSM& sm, const AZ::HSM::Event& event)
+        {
+            switch (event.id)
+            {
+            case AZ::HSM::EnterEventId:
+                //AZ_TracePrintf("GridMate", "Replica 0x%x entering migration state MST_ABORT.\n", m_replica->GetRepId());
+                return true;
+            case ME_UPDATE:
+                AZStd::static_pointer_cast<ReplicaStatus>(m_replica->m_replicaStatus)->SetUpstreamSuspended(false);
+                sm.Transition(MST_IDLE);
+                return true;
+            case ME_PEER_REMOVED:
+            case ME_PEER_ACK:
+                return true;
+            case ME_MODIFY_NEW_OWNER:
+                m_newOwnerId = *static_cast<PeerId*>(event.userData);
+                sm.Transition(MST_MIGRATING);
+                return true;
+            default:
+                break;
+            }
+            return false;
+        }
+        //-----------------------------------------------------------------------------
+        bool MigrationSequence::OnStateRollback(AZ::HSM& sm, const AZ::HSM::Event& event)
+        {
+            switch (event.id)
+            {
+            case AZ::HSM::EnterEventId:
+                //AZ_TracePrintf("GridMate", "Replica 0x%x entering migration state MST_ROLLBACK.\n", m_replica->GetRepId());
+                return true;
+            case ME_UPDATE:
+                m_replicaMgr->ChangeReplicaOwnership(m_replica, m_replica->GetMyContext(), true);
+                AZStd::static_pointer_cast<ReplicaStatus>(m_replica->m_replicaStatus)->SetUpstreamSuspended(false);
+                if (m_replica->m_upstreamHop != &m_replicaMgr->m_self)
+                {
+                    m_replica->m_upstreamHop->Remove(m_replica);
+                    m_replicaMgr->m_self.Add(m_replica);
+                }
+                sm.Transition(MST_IDLE);
+                return true;
+            case ME_PEER_REMOVED:
+            case ME_PEER_ACK:
+                return true;
+            default:
+                break;
+            }
+            return false;
+        }
+        //-----------------------------------------------------------------------------
+        bool MigrationSequence::OnStateChangeRoutingOnly(AZ::HSM& sm, const AZ::HSM::Event& event)
+        {
+            switch (event.id)
+            {
+            case AZ::HSM::EnterEventId:
+                //AZ_TracePrintf("GridMate", "Replica 0x%x entering migration state MST_CHANGE_ROUTING_ONLY.\n", m_replica->GetRepId());
+                return true;
+            case ME_UPDATE:
+            {
+                UpdateReplicaRouting();
+                sm.Transition(MST_IDLE);
+                return true;
+            }
+            default:
+                break;
+            }
+            return false;
+        }
+        //-----------------------------------------------------------------------------
+        bool MigrationSequence::DefaultHandler(AZ::HSM& sm, const AZ::HSM::Event& event)
+        {
+            (void)sm;
+            switch (event.id)
+            {
+            case AZ::HSM::EnterEventId:
+                //AZ_TracePrintf("GridMate", "Replica 0x%x entering migration state MST_IDLE.\n", m_replica->GetRepId());
+                return true;
+            case ME_MODIFY_NEW_OWNER:
+                m_newOwnerId = *static_cast<PeerId*>(event.userData);
+                if (m_replica->IsMaster() && m_newOwnerId != m_replicaMgr->m_self.GetId())
+                {
+                    sm.Transition(MST_MIGRATING);
+                }
+                else
+                {
+                    sm.Transition(MST_CHANGE_ROUTING_ONLY);
+                }
+                return true;
+            case ME_REPLICA_REMOVED:
+                if (sm.GetCurrentState() != MST_IDLE)
+                {
+                    sm.Transition(MST_IDLE);
+                }
+                return true;
+            default:
+                break;
+            }
+            return true;
+        }
+        //-----------------------------------------------------------------------------
+        void MigrationSequence::OnDeactivateReplica(ReplicaId replicaId, ReplicaManager* pMgr)
+        {
+            if (pMgr == m_replicaMgr && replicaId == m_replica->GetRepId())
+            {
+                m_sm.Dispatch(ME_REPLICA_REMOVED);
+            }
+        }
+        //-----------------------------------------------------------------------------
+        void MigrationSequence::OnPeerRemoved(PeerId peerId, ReplicaManager* pMgr)
+        {
+            if (pMgr == m_replicaMgr)
+            {
+                AZ::HSM::Event ev;
+                ev.id = ME_PEER_REMOVED;
+                ev.userData = &peerId;
+                m_sm.Dispatch(ev);
+            }
+        }
+        //-----------------------------------------------------------------------------
+        void MigrationSequence::OnReceivedAckUpstreamSuspended(PeerId from, AZ::u32 requestTime)
+        {
+            if (requestTime == m_timestamp)
+            {
+                //AZ_TracePrintf("GridMate", "Accepted upstream suspend ack response requested at %u for 0x%x from 0x%x.\n", requestTime, m_replica->GetRepId(), from);
+                AZ::HSM::Event ev;
+                ev.id = ME_PEER_ACK;
+                ev.userData = &from;
+                m_sm.Dispatch(ev);
+            }
+        }
+        //-----------------------------------------------------------------------------
+        void MigrationSequence::OnReceivedAckDownstream(PeerId from, AZ::u32 requestTime)
+        {
+            if (requestTime == m_timestamp)
+            {
+                //AZ_TracePrintf("GridMate", "Accepted downstream ack response requested at %u for 0x%x from 0x%x.\n", requestTime, m_replica->GetRepId(), from);
+                AZ::HSM::Event ev;
+                ev.id = ME_PEER_ACK;
+                ev.userData = &from;
+                m_sm.Dispatch(ev);
+            }
+        }
+        //-----------------------------------------------------------------------------
+        bool MigrationSequence::UpdateReplicaRouting()
+        {
+            // If we have a direct connection to the owner, then
+            // then the next hop is the owner peer, otherwise it is the host
+            ReplicaPeer* nextHop = nullptr;
+            if (m_newOwnerId == m_replicaMgr->GetLocalPeerId())
+            {
+                nextHop = &m_replicaMgr->m_self;
+            }
+            else
+            {
+                AZStd::lock_guard<AZStd::recursive_mutex> lock(m_replicaMgr->m_mutexRemotePeers);
+                for (auto& route : m_replicaMgr->m_remotePeers)
+                {
+                    if (route->GetId() == m_newOwnerId)
+                    {
+                        nextHop = route;
+                        break;
+                    }
+                    if (route->IsSyncHost())
+                    {
+                        nextHop = route;
+                    }
+                }
+            }
+            if (nextHop)
+            {
+                if (nextHop != m_replica->m_upstreamHop)
+                {
+                    if (m_replica->m_upstreamHop)
+                    {
+                        m_replica->m_upstreamHop->Remove(m_replica);
+                    }
+                    nextHop->Add(m_replica);
+                }
+                return true;
+            }
+            return false;
+        }
+        //-----------------------------------------------------------------------------
+    }   // namespace ReplicaInternal
+}   // namespace GridMate

Code/Framework/GridMate/GridMate/Replica/MigrationSequence.h

@@ -0,0 +1,93 @@
+/*
+* All or portions of this file Copyright (c) Amazon.com, Inc. or its affiliates or
+* its licensors.
+*
+* For complete copyright and license terms please see the LICENSE at the root of this
+* distribution (the "License"). All use of this software is governed by the License,
+* or, if provided, by the license below or the license accompanying this file. Do not
+* remove or modify any license notices. This file is distributed on an "AS IS" BASIS,
+* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+*
+*/
+#ifndef GM_REPLICA_MIGRATION_SEQUENCE_H
+#define GM_REPLICA_MIGRATION_SEQUENCE_H
+
+#include <GridMate/Replica/Replica.h>
+#include <GridMate/Replica/ReplicaMgr.h>
+#include <GridMate/Containers/unordered_set.h>
+#include <AzCore/State/HSM.h>
+
+namespace GridMate
+{
+    namespace ReplicaInternal
+    {
+        class MigrationSequence
+            : public ReplicaMgrCallbackBus::Handler
+        {
+        public:
+            enum MigrationState
+            {
+                MST_TOP,
+                MST_MIGRATING,
+                MST_FLUSH_UPSTREAM,
+                MST_FLUSH_DOWNSTREAM,
+                MST_CHANGE_ROUTING_FOR_MIGRATION,
+                MST_HANDOFF_REPLICA,
+                MST_ROLLBACK,
+                MST_ABORT,
+                MST_CHANGE_ROUTING_ONLY,
+                MST_IDLE,
+            };
+
+            enum MigrationEvent
+            {
+                ME_UPDATE,
+                ME_REPLICA_REMOVED,
+                ME_PEER_REMOVED,
+                ME_PEER_ACK,
+                ME_MODIFY_NEW_OWNER,
+            };
+
+            GM_CLASS_ALLOCATOR(MigrationSequence);
+
+            MigrationSequence(Replica* replica, PeerId newOwnerId);
+
+            void Update();
+            bool IsDone() const;
+            void ModifyNewOwner(PeerId newOwnerId);
+
+            bool OnStateMigrating(AZ::HSM& sm, const AZ::HSM::Event& event);
+            bool OnStateFlushUpstream(AZ::HSM& sm, const AZ::HSM::Event& event);
+            bool OnStateFlushDownstream(AZ::HSM& sm, const AZ::HSM::Event& event);
+            bool OnStateChangeRouting(AZ::HSM& sm, const AZ::HSM::Event& event);
+            bool OnStateHandoffReplica(AZ::HSM& sm, const AZ::HSM::Event& event);
+            bool OnStateAbort(AZ::HSM& sm, const AZ::HSM::Event& event);
+            bool OnStateRollback(AZ::HSM& sm, const AZ::HSM::Event& event);
+            bool OnStateChangeRoutingOnly(AZ::HSM& sm, const AZ::HSM::Event& event);
+            bool DefaultHandler(AZ::HSM& sm, const AZ::HSM::Event& event);
+
+            ///////////////////////////////////////////////////////////////////
+            // ReplicaMgrCallbackBus
+            virtual void OnDeactivateReplica(ReplicaId replicaId, ReplicaManager* pMgr) override;
+            virtual void OnPeerRemoved(PeerId peerId, ReplicaManager* pMgr) override;
+            ///////////////////////////////////////////////////////////////////
+
+            void OnReceivedAckUpstreamSuspended(PeerId from, AZ::u32 requestTime);
+            void OnReceivedAckDownstream(PeerId from, AZ::u32 requestTime);
+
+            bool UpdateReplicaRouting();
+
+            Replica* m_replica;
+            PeerId m_newOwnerId;
+            ReplicaManager* m_replicaMgr;
+            unsigned int m_timestamp;
+
+            AZ::HSM m_sm;
+
+            typedef unordered_set<PeerId> PeerAckTracker;
+            PeerAckTracker m_pendingAcks;
+        };
+    }   // namespace ReplicaInternal
+}   // namespace GridMate
+
+#endif // GM_REPLICA_MIGRATION_SEQUENCE_H

Code/Framework/GridMate/GridMate/Replica/RemoteProcedureCall.cpp

@@ -0,0 +1,57 @@
+/*
+* All or portions of this file Copyright (c) Amazon.com, Inc. or its affiliates or
+* its licensors.
+*
+* For complete copyright and license terms please see the LICENSE at the root of this
+* distribution (the "License"). All use of this software is governed by the License,
+* or, if provided, by the license below or the license accompanying this file. Do not
+* remove or modify any license notices. This file is distributed on an "AS IS" BASIS,
+* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+*
+*/
+
+#include <AzCore/Debug/Profiler.h>
+
+#include <GridMate/Replica/RemoteProcedureCall.h>
+#include <GridMate/Replica/Replica.h>
+#include <GridMate/Replica/ReplicaChunkDescriptor.h>
+#include <GridMate/Replica/ReplicaDrillerEvents.h>
+#include <GridMate/Replica/ReplicaMgr.h>
+#include <GridMate/Replica/ReplicaUtils.h>
+
+namespace GridMate
+{
+    RpcBase::RpcBase(const char* debugName)
+        : m_replicaChunk(nullptr)
+    {
+        ReplicaChunkInitContext* initContext = ReplicaChunkDescriptorTable::Get().GetCurrentReplicaChunkInitContext();
+        AZ_Assert(initContext, "Replica construction stack is NOT pushed on the stack! Call Replica::Desriptor::Push() before construction!");
+        ReplicaChunkDescriptor* descriptor = initContext->m_descriptor;
+        AZ_Assert(descriptor, "Replica's descriptor is NOT pushed on the stack! Call Replica::Desriptor::Push() before construction!");
+        descriptor->RegisterRPC(debugName, this);
+    }
+
+    void RpcBase::Queue(GridMate::Internal::RpcRequest* rpc)
+    {
+        m_replicaChunk->QueueRPCRequest(rpc);
+    }
+
+    void RpcBase::OnRpcRequest(GridMate::Internal::RpcRequest* rpc) const
+    {
+        EBUS_EVENT(Debug::ReplicaDrillerBus, OnRequestRpc, m_replicaChunk, rpc);
+    }
+
+    void RpcBase::OnRpcInvoke(GridMate::Internal::RpcRequest* rpc) const
+    {
+        EBUS_EVENT(Debug::ReplicaDrillerBus, OnInvokeRpc, m_replicaChunk, rpc);
+    }
+
+    PeerId RpcBase::GetSourcePeerId()
+    {
+        if (m_replicaChunk->GetReplicaManager())
+        {
+            return m_replicaChunk->GetReplicaManager()->GetLocalPeerId();
+        }
+        return InvalidReplicaPeerId;
+    }
+}

Code/Framework/GridMate/GridMate/Replica/RemoteProcedureCall.h

@@ -0,0 +1,576 @@
+/*
+* All or portions of this file Copyright (c) Amazon.com, Inc. or its affiliates or
+* its licensors.
+*
+* For complete copyright and license terms please see the LICENSE at the root of this
+* distribution (the "License"). All use of this software is governed by the License,
+* or, if provided, by the license below or the license accompanying this file. Do not
+* remove or modify any license notices. This file is distributed on an "AS IS" BASIS,
+* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+*
+*/
+#ifndef GM_REPLICA_RPC_H
+#define GM_REPLICA_RPC_H
+
+#include <AzCore/std/function/function_fwd.h>
+#include <AzCore/Preprocessor/Sequences.h>
+#include <AzCore/std/typetraits/is_base_of.h>
+#include <AzCore/std/typetraits/remove_const.h>
+#include <AzCore/std/typetraits/remove_reference.h>
+
+#include <GridMate/Replica/Replica.h>
+#include <GridMate/Replica/ReplicaChunk.h>
+#include <GridMate/Replica/ReplicaChunkDescriptor.h>
+#include <GridMate/Replica/ReplicaCommon.h>
+#include <GridMate/Replica/ReplicaDefs.h>
+
+#include <GridMate/Serialize/Buffer.h>
+#include <GridMate/Serialize/DataMarshal.h>
+
+namespace GridMate
+{
+    class ReplicaMarshalTaskBase;
+
+    namespace Internal
+    {
+        struct RpcRequest;
+        struct InterfaceResolver;
+    }
+
+    struct RpcDefaultTraits
+    {
+        static const bool s_isReliable = true;
+        static const bool s_isPostAttached = true;
+        static const bool s_alwaysForwardSourcePeer = false;
+        static const bool s_allowNonAuthoritativeRequests = true;
+        static const bool s_allowNonAuthoritativeRequestRelay = true;
+    };
+
+    struct RpcAuthoritativeTraits : public RpcDefaultTraits
+    {
+        static const bool s_isReliable = true;
+        static const bool s_isPostAttached = true;
+        static const bool s_allowNonAuthoritativeRequests = false;
+    };
+
+    struct RpcUnreliable
+        : public RpcDefaultTraits
+    {
+        static const bool s_isReliable = false;
+    };
+
+    //--------------------------------------------------------------------------
+    // RpcContext
+    //--------------------------------------------------------------------------
+    struct RpcContext
+    {
+        unsigned int m_realTime;
+        unsigned int m_localTime;
+        AZ::u32 m_timestamp;
+
+        // The source peer is derived from the connection id, unless s_alwaysForwardSourcePeer is set for the RPC.
+        // This will be the 'expected' peer for a direct connection, but if forwarding is involved between
+        // peers, then the s_alwaysForwardSourcePeer should be set to ensure the source value is propagated
+        // across the network.
+        PeerId m_sourcePeer;
+
+        RpcContext(unsigned int realTime = 0,
+            unsigned int localTime = 0,
+            unsigned int timestamp = 0,
+            PeerId sourcePeer = InvalidReplicaPeerId)
+            : m_realTime(realTime)
+            , m_localTime(localTime)
+            , m_timestamp(timestamp)
+            , m_sourcePeer(sourcePeer)
+        { }
+    };
+    //--------------------------------------------------------------------------
+
+    /**
+    * RPC argument container, used like Rpc<RpcArg<...>, RpcArg<...>>::BindInterface<Class, Method>...
+    */
+    struct RpcArgBase { };
+    template<class T, class M = Marshaler<typename AZStd::remove_const<typename AZStd::remove_reference<T>::type>::type> >
+    struct RpcArg
+        : public RpcArgBase
+    {
+        typedef T Type;
+        typedef M MarshalerType;
+    };
+
+    /**
+    * RPC base class
+    */
+    class RpcBase
+    {
+        friend Replica;
+        friend ReplicaChunkBase;
+        friend ReplicaChunkDescriptor;
+        friend ReplicaMarshalTaskBase;
+        friend Internal::InterfaceResolver;
+
+    protected:
+        RpcBase(const char* debugName);
+    public:
+        virtual ~RpcBase() { }
+
+    protected:
+        virtual void Marshal(WriteBuffer& wb, Internal::RpcRequest* request) = 0;
+        virtual Internal::RpcRequest* Unmarshal(ReadBuffer& rb) = 0;
+        virtual bool Invoke(Internal::RpcRequest* rpc) const = 0;
+        virtual bool IsPostAttached() const = 0;                //Requires Data Sets updated before executing RPC
+        virtual bool IsAllowNonAuthoritativeRequests() const = 0;
+        virtual bool IsAllowNonAuthoritativeRequestsRelay() const = 0;
+
+        PeerId GetSourcePeerId();
+
+        void Queue(Internal::RpcRequest* context);
+        void OnRpcRequest(Internal::RpcRequest* rpc) const;
+        void OnRpcInvoke(Internal::RpcRequest* rpc) const;
+
+        ReplicaChunkBase* m_replicaChunk;
+    };
+
+    namespace Internal
+    {
+        // -- Internal RPC data ---------------------------
+        struct RpcRequest
+            : public RpcContext
+        {
+            GM_CLASS_ALLOCATOR(RpcRequest);
+            bool m_authoritative;
+            bool m_processed;
+            bool m_relayed;
+            bool m_reliable; // need to save reliability state as unreliable rpcs might be promoted to reliable (e.g. when unreliable rpc, called before reliable on the same frame)
+            RpcBase* m_rpc;
+
+            RpcRequest(RpcBase* rpc, unsigned int timestamp = 0, unsigned int realTime = 0, unsigned int localTime = 0, PeerId sourcePeer = InvalidReplicaPeerId)
+                : RpcContext(realTime, localTime, timestamp, sourcePeer)
+                , m_authoritative(false)
+                , m_processed(false)
+                , m_relayed(false)
+                , m_reliable(true)
+                , m_rpc(rpc)
+            {
+                AZ_Assert(m_rpc, "We require a valid RPCBase pointer!");
+            }
+
+            virtual ~RpcRequest() { }
+        };
+        // ------------------------------------------------
+
+        struct InterfaceResolver
+        {
+            template<class C, class T>
+            static C* GetInterface(T* object)
+            {
+                static_assert(AZStd::is_base_of<ReplicaChunkInterface, C>::value, "Class must inherit from ReplicaChunkInterface");
+                AZ_Assert(object, "Invalid handler");
+                AZ_Assert(object->m_replicaChunk, "Invalid replica chunk");
+                return static_cast<C*>(object->m_replicaChunk->GetHandler());
+            }
+        };
+    }
+
+    namespace Internal
+    {
+        // -- Static integer sequence ---------------------
+        // Used to unpack all values from a tuple at once
+        template<size_t ...>
+        struct StaticSequence { };
+
+        template<size_t N, size_t ... Indices>
+        struct GenerateStaticSequence
+            : GenerateStaticSequence<N - 1, N - 1, Indices...> { };
+
+        template<size_t ... Indices>
+        struct GenerateStaticSequence<0, Indices...>
+        {
+            typedef StaticSequence<Indices...> Type;
+        };
+        // ------------------------------------------------
+
+        // -- Convenience structures for extracting a type without wrapping it in RpcArg<T,M> ---------------------
+        template<typename T, typename = void>
+        struct RpcTypeExtraction;
+        template<typename T>
+        struct RpcTypeExtraction<T, typename AZStd::Utils::enable_if_c<AZStd::is_base_of<RpcArgBase, T>::value>::type>
+        {
+            using Type = typename T::Type;
+            using MarshalerType = typename T::MarshalerType;
+        };
+
+        template<typename T>
+        struct RpcTypeExtraction<T, typename AZStd::Utils::enable_if_c<!AZStd::is_base_of<RpcArgBase, T>::value>::type>
+        {
+            using Type = T;
+            using MarshalerType = Marshaler<T>;
+        };
+        // ------------------------------------------------
+
+        // -- Storage for Marshalers ---------------------
+        template<class ... Args>
+        struct VariadicMarshaler;
+
+        template<>
+        struct VariadicMarshaler<>
+        {
+            GM_CLASS_ALLOCATOR(VariadicMarshaler);
+
+            VariadicMarshaler() { }
+        };
+
+        template<class This, class ... Rest>
+        struct VariadicMarshaler<This, Rest...>
+            : public VariadicMarshaler<Rest...>
+        {
+            GM_CLASS_ALLOCATOR(VariadicMarshaler);
+
+            using ParentType = VariadicMarshaler<Rest...>;
+            using MarshalerType = typename RpcTypeExtraction<This>::MarshalerType;
+
+            VariadicMarshaler()
+                : ParentType() { }
+
+            template<typename ThisArg, typename ... RestArgs>
+            VariadicMarshaler(ThisArg&& marshaler, RestArgs&& ... args)
+                : ParentType(args ...)
+                , m_marshaler(marshaler) { }
+
+            MarshalerType m_marshaler;
+        };
+        // ------------------------------------------------
+
+        // -- Storage for RPC -----------------------------
+        template<class ... Args>
+        struct VariadicStorage;
+
+        template<>
+        struct VariadicStorage<>
+            : public Internal::RpcRequest
+        {
+            GM_CLASS_ALLOCATOR(VariadicStorage);
+
+            using MarshalerTuple = VariadicMarshaler<>;
+            using StaticArgSequence = Internal::GenerateStaticSequence<0>::Type;
+
+            VariadicStorage(RpcBase* rpc)
+                : RpcRequest(rpc) { }
+            VariadicStorage(RpcBase* rpc, const RpcContext& ctx)
+                : RpcRequest(rpc, ctx.m_timestamp, ctx.m_realTime, ctx.m_localTime, ctx.m_sourcePeer) { }
+
+            bool Marshal(WriteBuffer&, const VariadicMarshaler<>&) const
+            {
+                return true;
+            }
+            bool Unmarshal(ReadBuffer&, const VariadicMarshaler<>&)
+            {
+                return true;
+            }
+        };
+
+        template<class This, class ... Rest>
+        struct VariadicStorage<This, Rest...>
+            : public VariadicStorage<Rest...>
+        {
+            GM_CLASS_ALLOCATOR(VariadicStorage);
+
+            using ParentType = VariadicStorage<Rest...>;
+            using ExtractedType = typename RpcTypeExtraction<This>::Type;
+            using ThisType = typename AZStd::remove_const<typename AZStd::remove_reference<ExtractedType>::type>::type;
+            using MarshalerTuple = VariadicMarshaler<This, Rest...>;
+            using StaticArgSequence = typename Internal::GenerateStaticSequence<sizeof ... (Rest) +1>::Type;
+
+            VariadicStorage(RpcBase* rpc)
+                : ParentType(rpc) { }
+
+            template<typename ThisArg, typename ... RestArgs>
+            VariadicStorage(RpcBase* rpc, const RpcContext& ctx, ThisArg&& val, RestArgs&& ... args)
+                : ParentType(rpc, ctx, args ...)
+                , m_val(val) { }
+
+            void Marshal(WriteBuffer& wb, MarshalerTuple& marshaler)
+            {
+                ParentType::Marshal(wb, marshaler);
+
+                wb.Write(m_val, marshaler.m_marshaler);
+            }
+
+            bool Unmarshal(ReadBuffer& rb, MarshalerTuple& marshaler)
+            {
+                if (!ParentType::Unmarshal(rb, marshaler))
+                {
+                    return false;
+                }
+
+                return rb.Read(m_val, marshaler.m_marshaler);
+            }
+
+            template<size_t Index, class ... Types>
+            struct ExtractIndexContainerType;
+
+            template<class First, class ... Types>
+            struct ExtractIndexContainerType<0, First, Types...>
+            {
+                typedef VariadicStorage<First, Types...> ContainerType;
+            };
+
+            template<size_t Index, class First, class ... Types>
+            struct ExtractIndexContainerType<Index, First, Types...>
+                : ExtractIndexContainerType<Index - 1, Types...>
+            { };
+
+            template<size_t Index, class First, class ... Types>
+            struct ExtractIndexReturnType
+            {
+                typedef typename ExtractIndexContainerType<Index, This, Rest...>::ContainerType ContainerType;
+                typedef typename ContainerType::ThisType ThisType;
+            };
+
+            ThisType m_val;
+        };
+        // ------------------------------------------------
+
+        // -- GetStorageFromIndex -------------------------
+        template<size_t index, typename Storage, class ... Args>
+        const typename Storage::template ExtractIndexReturnType<index, Args...>::ThisType & GetStorageFromIndex(const Storage * storage)
+        {
+            typedef typename Storage::template ExtractIndexContainerType<index, Args...>::ContainerType ContainerType;
+
+            // Cast to Nth parent type to extract the contained value
+            return static_cast<const ContainerType*>(storage)->m_val;
+        }
+        // ------------------------------------------------
+
+        // -- Base binding class for common functionality --
+        template<class Traits, class TypeTuple, class ForwardHandler>
+        class RpcBindBase
+            : public RpcBase
+        {
+        public:
+            typedef typename TypeTuple::MarshalerTuple MarshalerSetType;
+
+            template<typename ... LocalArgs>
+            RpcBindBase(const char* debugName, LocalArgs&& ... marshalers)
+                : RpcBase(debugName)
+                , m_marshalers(AZStd::forward<LocalArgs>(marshalers) ...) { }
+
+            virtual bool InvokeImpl(Internal::RpcRequest* request) const = 0;
+
+            bool IsPostAttached() const override { return Traits::s_isPostAttached; }
+            bool IsAllowNonAuthoritativeRequests() const override { return Traits::s_allowNonAuthoritativeRequests; }
+            bool IsAllowNonAuthoritativeRequestsRelay() const override { return Traits::s_allowNonAuthoritativeRequestRelay; }
+
+            template<typename ... LocalArgs>
+            void operator()(LocalArgs&& ... args)
+            {
+                AZ_Assert(m_replicaChunk, "Cannot call an RPC that is not bound to a ReplicaChunk");
+                Replica* replica = m_replicaChunk->GetReplica();
+                ReplicaContext rc = replica ? replica->GetMyContext() : ReplicaContext(nullptr, TimeContext());
+                PeerId sourcePeerId = GetSourcePeerId();
+                bool shouldQueue = true;
+                bool processed = false;
+                bool isMaster = m_replicaChunk->IsMaster();
+                if (isMaster)
+                {
+                    // We are authoritative so execute the RPC immediately, forwarding the args along
+                    RpcRequest localRequest(this, rc.m_realTime, rc.m_realTime, rc.m_localTime);
+                    OnRpcRequest(&localRequest);
+                    OnRpcInvoke(&localRequest);
+                    bool isReplicaActive = replica && replica->IsActive(); // cache this because it could get changed during the RPC call.
+                    bool isForward = static_cast<ForwardHandler*>(this)->Forward(RpcContext(rc.m_realTime, rc.m_localTime, rc.m_realTime, sourcePeerId), AZStd::forward<LocalArgs>(args) ...);
+                    shouldQueue = isForward && isReplicaActive;
+                    processed = true;
+                }
+                if (shouldQueue)
+                {
+                    TypeTuple* storage = aznew TypeTuple(this, RpcContext(rc.m_realTime, rc.m_realTime, rc.m_localTime, sourcePeerId), AZStd::forward<LocalArgs>(args) ...);
+                    storage->m_authoritative = isMaster;
+                    storage->m_processed = processed;
+                    storage->m_reliable = Traits::s_isReliable;
+                    OnRpcRequest(storage);
+                    Queue(storage);
+                }
+            }
+
+            bool Invoke(Internal::RpcRequest* rpc) const override
+            {
+                OnRpcInvoke(rpc);
+                return InvokeImpl(rpc);
+            }
+
+        protected:
+            void Marshal(WriteBuffer& wb, Internal::RpcRequest* request) override
+            {
+                TypeTuple* storage = static_cast<TypeTuple*>(request);
+
+                wb.Write(storage->m_timestamp);
+                wb.Write(storage->m_authoritative);
+                if (Traits::s_alwaysForwardSourcePeer)
+                {
+                    wb.Write(storage->m_sourcePeer);
+                }
+
+                // Pass the marshal onto the storage which unwraps the marshaling of each RPC value
+                storage->Marshal(wb, m_marshalers);
+            }
+
+            Internal::RpcRequest* Unmarshal(ReadBuffer& rb) override
+            {
+                TypeTuple* storage = aznew TypeTuple(this);
+
+                rb.Read(storage->m_timestamp);
+                rb.Read(storage->m_authoritative);
+                if (Traits::s_alwaysForwardSourcePeer)
+                {
+                    rb.Read(storage->m_sourcePeer);
+                }
+
+                // Pass the unmarshal onto the storage which unwraps the marshaling of each RPC value
+                if (!storage->Unmarshal(rb, m_marshalers))
+                {
+                    delete storage;
+                    return nullptr;
+                }
+                return storage;
+            }
+
+            MarshalerSetType m_marshalers; // Set of marshalers, one for each type specified in the RPC
+        };
+        // -----------------------------------------
+
+        template<class ... Args>
+        struct RpcBindArgsWrapper
+        {
+            /**
+            * Bind a class method to the RPC
+            */
+            template<class Traits, class TypeTuple, class ThisResolver, class C, bool (C::* FuncPtr)(typename Args::Type..., const RpcContext&)>
+            class BindInterface
+                : public RpcBindBase<Traits, TypeTuple, BindInterface<Traits, TypeTuple, ThisResolver, C, FuncPtr> >
+            {
+            public:
+                template<typename ... LocalArgs>
+                BindInterface(const char* debugName, LocalArgs&& ... marshalers)
+                    : RpcBindBase<Traits, TypeTuple, BindInterface<Traits, TypeTuple, ThisResolver, C, FuncPtr> >(debugName, AZStd::forward<LocalArgs>(marshalers) ...) { }
+
+                // Invoke from storage
+                bool InvokeImpl(Internal::RpcRequest* request) const override
+                {
+                    TypeTuple* storage = static_cast<TypeTuple*>(request);
+                    return InvokeWithArgs(storage, typename TypeTuple::StaticArgSequence());
+                }
+
+                // Forward from a direct in-place call
+                template<typename ... LocalArgs>
+                bool Forward(const RpcContext& context, LocalArgs&& ... args)
+                {
+                    C* c = ThisResolver::template GetInterface<C>(this);
+                    if (c)
+                    {
+                        return (*c.*FuncPtr)(AZStd::forward<LocalArgs>(args) ..., context);
+                    }
+
+                    return false;
+                }
+
+            protected:
+                // Invoke and unwrap the args
+                template<size_t ... Indices>
+                bool InvokeWithArgs(TypeTuple* storage, Internal::StaticSequence<Indices...>) const
+                {
+                    C* c = ThisResolver::template GetInterface<C>(this);
+                    if (c)
+                    {
+                        return (*c.*FuncPtr)(GetStorageFromIndex<Indices, TypeTuple, Args...>(storage) ..., *storage);
+                    }
+                     
+                    return false;
+                }
+            };
+
+            /**
+            * Bind a class method to the RPC (0 args version)
+            */
+            template<class Traits, class TypeTuple, class ThisResolver, class C, bool (C::* FuncPtr)(const RpcContext&)>
+            class BindInterface0
+                : public RpcBindBase<Traits, TypeTuple, BindInterface0<Traits, TypeTuple, ThisResolver, C, FuncPtr> >
+            {
+            public:
+                BindInterface0(const char* debugName)
+                    : RpcBindBase<Traits, TypeTuple, BindInterface0<Traits, TypeTuple, ThisResolver, C, FuncPtr> >(debugName) { }
+
+                // Invoke from storage
+                bool InvokeImpl(Internal::RpcRequest* request) const override
+                {
+                    C* c = ThisResolver::template GetInterface<C>(this);
+                    if (c)
+                    {
+                        TypeTuple* storage = static_cast<TypeTuple*>(request);
+                        return (*c.*FuncPtr)(*storage);
+                    }
+                     
+                    return false;
+                }
+
+                // Forward from a direct in-place call
+                bool Forward(const RpcContext& context)
+                {
+                    C* c = ThisResolver::template GetInterface<C>(this);
+                    if (c)
+                    {
+                        return (*c.*FuncPtr)(context);
+                    }
+
+                    return false;
+                }
+            };
+
+            RpcBindArgsWrapper() = delete;
+        };
+    }
+
+    /**
+    * Public interface for declaring an RPC
+    */
+    template<typename ... Args>
+    class Rpc;
+
+    /**
+    * 0 args specialisation
+    */
+    template<>
+    class Rpc<>
+    {
+        typedef Internal::VariadicStorage<> TypeTuple;
+
+    public:
+        template<class C, bool (C::* FuncPtr)(const RpcContext&), class Traits = RpcDefaultTraits>
+        using BindInterface = Internal::RpcBindArgsWrapper<>::BindInterface0<Traits, TypeTuple, Internal::InterfaceResolver, C, FuncPtr>;
+
+        Rpc() = delete;
+    };
+
+    /**
+    * Any args specialisation
+    */
+    template<typename ... Args>
+    class Rpc
+    {
+        typedef Internal::VariadicStorage<Args...> TypeTuple;
+
+    public:
+        template<class C, bool (C::* FuncPtr)(typename Args::Type..., const RpcContext&), class Traits = RpcDefaultTraits>
+        struct BindInterface
+            : public Internal::RpcBindArgsWrapper<Args...>::template BindInterface<Traits, TypeTuple, Internal::InterfaceResolver, C, FuncPtr>
+        {
+            template<typename ... LocalArgs>
+            BindInterface(const char* debugName, LocalArgs&& ... marshalers)
+                : Internal::RpcBindArgsWrapper<Args...>::template BindInterface<Traits, TypeTuple, Internal::InterfaceResolver, C, FuncPtr>(debugName, AZStd::forward<LocalArgs>(marshalers) ...) { }
+        };
+
+        Rpc() = delete;
+    };
+} // namespace GridMate
+
+#endif // GM_REPLICA_RPC_H

Code/Framework/GridMate/GridMate/Replica/Replica.cpp

@@ -0,0 +1,807 @@
+/*
+* All or portions of this file Copyright (c) Amazon.com, Inc. or its affiliates or
+* its licensors.
+*
+* For complete copyright and license terms please see the LICENSE at the root of this
+* distribution (the "License"). All use of this software is governed by the License,
+* or, if provided, by the license below or the license accompanying this file. Do not
+* remove or modify any license notices. This file is distributed on an "AS IS" BASIS,
+* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+*
+*/
+
+#include <AzCore/Debug/Profiler.h>
+
+#include <GridMate/Replica/DataSet.h>
+#include <GridMate/Replica/MigrationSequence.h>
+#include <GridMate/Replica/RemoteProcedureCall.h>
+#include <GridMate/Replica/Replica.h>
+#include <GridMate/Replica/ReplicaChunk.h>
+#include <GridMate/Replica/ReplicaChunkDescriptor.h>
+#include <GridMate/Replica/ReplicaDrillerEvents.h>
+#include <GridMate/Replica/ReplicaFunctions.h>
+#include <GridMate/Replica/ReplicaMgr.h>
+#include <GridMate/Replica/ReplicaStatus.h>
+#include <GridMate/Replica/ReplicaUtils.h>
+#include <GridMate/Serialize/CompressionMarshal.h>
+
+namespace GridMate
+{
+    //-----------------------------------------------------------------------------
+    Replica* Replica::CreateReplica(const char* replicaName)
+    {
+        return aznew Replica(replicaName);
+    }
+    //-----------------------------------------------------------------------------
+    Replica::Replica(const char* replicaName)
+        : m_refCount(0)
+        , m_myId(InvalidReplicaId)
+        , m_flags(0)
+        , m_createTime(0)
+        , m_manager(nullptr)
+        , m_upstreamHop(nullptr)
+        , m_replicaStatus(nullptr)
+        , m_priority(0)
+        , m_revision(1)
+    {
+        AZ_PROFILE_TIMER("GridMate");
+
+        m_upstreamHop = nullptr;
+        m_dirtyHook.m_next = m_dirtyHook.m_prev = nullptr;
+
+#if GM_REPLICA_HAS_DEBUG_NAME == 0
+        replicaName = nullptr;
+#endif
+        InternalCreateInitialChunks(replicaName);
+
+        EBUS_EVENT(Debug::ReplicaDrillerBus, OnCreateReplica, this);
+    }
+    //-----------------------------------------------------------------------------
+    Replica::~Replica()
+    {
+        AZ_Assert(m_refCount == 0, "Attempting to free replica with non-zero refCount(%d)!", m_refCount);
+    }
+    //-----------------------------------------------------------------------------
+    void Replica::release()
+    {
+        AZ_Assert(m_refCount > 0, "Reference count logic error, trying to remove reference when refcount is 0");
+        if (m_refCount == 1)
+        {
+            {
+                ReplicaPtr reference = this;
+
+                // PreDestruct - This is to run any destruction code that can call out to user code.
+                // We temporarily hold a refcount here to prevent a double deletion if user code creates
+                // and deletes another ref counted container, since that will cause the refcount
+                // to change from 0 -> 1 -> 0 again.
+                PreDestruct();
+            }
+
+            AZ_Assert(m_refCount == 1, "Attempting to hold on to replica refcount while deleting: refCount(%d)!", m_refCount);
+
+            --m_refCount;
+            delete this;
+        }
+        else
+        {
+            --m_refCount;
+        }
+    }
+    //-----------------------------------------------------------------------------
+    void Replica::PreDestruct()
+    {
+        AZ_PROFILE_TIMER("GridMate");
+
+        for (auto chunk : m_chunks)
+        {
+            if (chunk)
+            {
+                chunk->DetachedFromReplica();
+            }
+        }
+        m_chunks.clear();
+
+        EBUS_EVENT(Debug::ReplicaDrillerBus, OnDestroyReplica, this);
+    }
+    //-----------------------------------------------------------------------------
+    void Replica::Destroy()
+    {
+        AZ_Assert(IsMaster(), "We don't own replica 0x%x!", GetRepId());
+        if (m_manager)
+        {
+            m_manager->Destroy(this);
+        }
+    }
+    //-----------------------------------------------------------------------------
+    void Replica::InternalCreateInitialChunks(const char* replicaName)
+    {
+        ReplicaStatus* statusChunk = CreateReplicaChunk<ReplicaStatus>();
+
+        if (replicaName)
+        {
+            statusChunk->SetDebugName(replicaName);
+        }
+
+        statusChunk->SetUpstreamSuspended(false);
+        m_replicaStatus = statusChunk;
+        AttachReplicaChunk(statusChunk);
+    }
+    //-----------------------------------------------------------------------------
+    PeerId Replica::GetPeerId() const
+    {
+        PeerId peerId(InvalidReplicaPeerId);
+
+        if (m_manager != nullptr)
+        {
+            peerId = m_manager->m_cfg.m_myPeerId;
+        }
+
+        return peerId;
+    }
+    //-----------------------------------------------------------------------------
+    bool Replica::AttachReplicaChunk(const ReplicaChunkPtr& chunk)
+    {
+        AZ_PROFILE_TIMER("GridMate");
+
+        // Check for duplicate attach
+        if (!chunk->GetReplica())
+        {
+            // Chunks cannot be attached while active
+            if (!IsActive())
+            {
+                if (m_chunks.size() < GM_MAX_CHUNKS_PER_REPLICA)
+                {
+                    m_chunks.push_back(chunk);
+                    OnReplicaPriorityUpdated(chunk.get());
+                    chunk->AttachedToReplica(this);
+                    AZ_Assert(chunk->GetReplica() == this, "Must be bound to the same replica");
+
+                    return true;
+                }
+                else
+                {
+                    AZ_Warning("GridMate", false, "Cannot attach chunk %s because GM_MAX_CHUNKS_PER_REPLICA has been exceeded.", chunk->GetDescriptor()->GetChunkName());
+                }
+            }
+            else
+            {
+                AZ_Warning("GridMate", false, "Cannot attach chunk %s while replica is active.", chunk->GetDescriptor()->GetChunkName());
+            }
+        }
+        else
+        {
+            AZ_Warning("GridMate", false, "Cannot attach chunk %s because it is already attached to a replica.", chunk->GetDescriptor()->GetChunkName());
+        }
+
+        return false;
+    }
+    //-----------------------------------------------------------------------------
+    bool Replica::DetachReplicaChunk(const ReplicaChunkPtr& chunk)
+    {
+        AZ_PROFILE_TIMER("GridMate");
+
+        if (!IsActive())
+        {
+            for (auto iter = m_chunks.begin(); iter != m_chunks.end(); ++iter)
+            {
+                if ((*iter) == chunk)
+                {
+                    (*iter)->DetachedFromReplica();
+                    m_chunks.erase(iter);
+                    OnReplicaPriorityUpdated(chunk.get());
+                    return true;
+                }
+            }
+        }
+        else
+        {
+            AZ_Warning("GridMate", false, "Cannot detach chunk %s because the replica is active.", chunk->GetDescriptor()->GetChunkName());
+        }
+        return false;
+    }
+    //-----------------------------------------------------------------------------
+    const char* Replica::GetDebugName() const
+    {
+        return static_cast<ReplicaStatus*>(m_replicaStatus.get())->GetDebugName();
+    }
+    //-----------------------------------------------------------------------------
+    ReplicaContext Replica::GetMyContext() const
+    {
+        ReplicaContext rc(nullptr, TimeContext());
+        if (m_manager)
+        {
+            m_manager->GetReplicaContext(this, rc);
+        }
+        return rc;
+    }
+    //-----------------------------------------------------------------------------
+    void Replica::UpdateReplica(const ReplicaContext& rc)
+    {
+        AZ_PROFILE_TIMER("GridMate");
+
+        for (auto chunk : m_chunks)
+        {
+            if (chunk)
+            {
+                chunk->InternalUpdateChunk(rc);
+            }
+        }
+    }
+    //-----------------------------------------------------------------------------
+    void Replica::UpdateFromReplica(const ReplicaContext& rc)
+    {
+        AZ_PROFILE_TIMER("GridMate");
+
+        for (auto chunk : m_chunks)
+        {
+            if (chunk)
+            {
+                chunk->InternalUpdateFromChunk(rc);
+            }
+        }
+    }
+    //-----------------------------------------------------------------------------
+    bool Replica::AcceptChangeOwnership(PeerId requestor, const ReplicaContext& rc)
+    {
+        AZ_PROFILE_TIMER("GridMate");
+
+        for (auto chunk : m_chunks)
+        {
+            if (chunk)
+            {
+                bool accepted = chunk->AcceptChangeOwnership(requestor, rc);
+                if (!accepted)
+                {
+                    return false;
+                }
+            }
+        }
+        return true;
+    }
+    //-----------------------------------------------------------------------------
+    void Replica::OnActivate(const ReplicaContext& rc)
+    {
+        EBUS_EVENT(Debug::ReplicaDrillerBus, OnActivateReplica, this);
+
+        for (auto chunk : m_chunks)
+        {
+            if (chunk)
+            {
+                {
+                    GM_PROFILE_USER_CALLBACK("OnReplicaActivate");
+                    chunk->OnReplicaActivate(rc);
+                }
+                EBUS_EVENT(Debug::ReplicaDrillerBus, OnActivateReplicaChunk, chunk.get());
+            }
+        }
+    }
+    //-----------------------------------------------------------------------------
+    void Replica::OnDeactivate(const ReplicaContext& rc)
+    {
+        AZ_PROFILE_TIMER("GridMate");
+
+        EBUS_EVENT_ID(rc.m_rm->GetGridMate(), ReplicaMgrCallbackBus, OnDeactivateReplica, GetRepId(), rc.m_rm);
+        EBUS_EVENT(Debug::ReplicaDrillerBus, OnDeactivateReplica, this);
+
+        for (auto chunk : m_chunks)
+        {
+            if (chunk)
+            {
+                {
+                    GM_PROFILE_USER_CALLBACK("OnReplicaDeactivate");
+                    chunk->OnReplicaDeactivate(rc);
+                }
+                EBUS_EVENT(Debug::ReplicaDrillerBus, OnDeactivateReplicaChunk, chunk.get());
+            }
+        }
+    }
+    //-----------------------------------------------------------------------------
+    void Replica::OnChangeOwnership(const ReplicaContext& rc)
+    {
+        AZ_PROFILE_TIMER("GridMate");
+
+        for (auto chunk : m_chunks)
+        {
+            if (chunk)
+            {
+                GM_PROFILE_USER_CALLBACK("OnReplicaChangeOwnership");
+                chunk->OnReplicaChangeOwnership(rc);
+            }
+        }
+    }
+    //-----------------------------------------------------------------------------
+    void Replica::RequestChangeOwnership(PeerId newOwner)
+    {
+        if (newOwner == InvalidReplicaPeerId)
+        {
+            newOwner = m_manager->GetLocalPeerId();
+        }
+        AZStd::static_pointer_cast<ReplicaStatus>(m_replicaStatus)->RequestOwnership(newOwner);
+    }
+    //-----------------------------------------------------------------------------
+    bool Replica::RequestOwnershipFn(PeerId requestor, const RpcContext& rpcContext)
+    {
+        (void) rpcContext;
+
+        AZ_PROFILE_TIMER("GridMate");
+
+        if (IsActive())
+        {
+            if (IsMaster())
+            {
+                EBUS_EVENT(Debug::ReplicaDrillerBus, OnRequestReplicaChangeOwnership, this, requestor);
+
+                if (IsMigratable() && requestor != m_manager->GetLocalPeerId())
+                {
+                    bool accepted;
+                    {
+                        GM_PROFILE_USER_CALLBACK("AcceptChangeOwnership");
+                        accepted = AcceptChangeOwnership(requestor, GetMyContext());
+                    }
+                    if (accepted)
+                    {
+                        m_manager->MigrateReplica(this, requestor);
+                    }
+                }
+            }
+        }
+        return false;
+    }
+    //-----------------------------------------------------------------------------
+    bool Replica::MigrationSuspendUpstreamFn(PeerId ownerId, AZ::u32 requestTime, const RpcContext& rpcContext)
+    {
+        (void) rpcContext;
+        if (IsProxy())
+        {
+            //AZ_TracePrintf("GridMate", "Received upstream suspend ack requested at %u for 0x%x from 0x%x.\n", requestTime, GetRepId(), ownerId);
+            m_manager->AckUpstreamSuspended(GetRepId(), ownerId, requestTime);
+        }
+        else
+        {
+            //AZ_TracePrintf("GridMate", "Sending upstream suspend ack requested at %u for 0x%x.\n", requestTime, GetRepId(), ownerId);
+        }
+        return true;
+    }
+    //-----------------------------------------------------------------------------
+    bool Replica::MigrationRequestDownstreamAckFn(PeerId ownerId, AZ::u32 requestTime, const RpcContext& rpcContext)
+    {
+        (void) rpcContext;
+        if (IsProxy())
+        {
+            //AZ_TracePrintf("GridMate", "Received downstream ack requested at %u for 0x%x from 0x%x .\n", requestTime, GetRepId(), ownerId);
+            m_manager->AckDownstream(GetRepId(), ownerId, requestTime);
+        }
+        else
+        {
+            m_flags |= Rep_SuspendDownstream;
+            //AZ_TracePrintf("GridMate", "Sending downstream ack request at %u for 0x%x from 0x%x .\n", requestTime, GetRepId(), ownerId);
+        }
+        return true;
+    }
+    //-----------------------------------------------------------------------------
+    void Replica::InitReplica(ReplicaManager* manager)
+    {
+        m_manager = manager;
+    }
+    //-----------------------------------------------------------------------------
+    void Replica::Activate(const ReplicaContext& rc)
+    {
+        AZ_PROFILE_TIMER("GridMate");
+
+        // Resolve whether we're migratable or not from the chunks
+        // present when we're attached to the network.
+        // If there are no chunks (excluding the system chunk) then
+        // we can't migrate, as the destination wont know what
+        // to do with an empty replica.
+        if (!m_chunks.empty())
+        {
+            bool migratable = true;
+            for (auto chunk : m_chunks)
+            {
+                if (chunk && !chunk->IsReplicaMigratable())
+                {
+                    migratable = false;
+                    break;
+                }
+            }
+            SetMigratable(migratable);
+        }
+
+        m_flags |= Rep_Active;
+
+        OnActivate(rc);
+    }
+    //-----------------------------------------------------------------------------
+    void Replica::Deactivate(const ReplicaContext& rc)
+    {
+        AZ_PROFILE_TIMER("GridMate");
+
+        if (IsActive())
+        {
+            OnDeactivate(rc);
+            m_flags &= ~Rep_Active;
+            m_manager->CancelTasks(this);
+        }
+        m_manager = nullptr;
+    }
+    //-----------------------------------------------------------------------------
+    void Replica::SetSyncStage(bool b)
+    {
+        AZ_Assert(!IsActive(), "Synchronization category can only be set before a replica is registered!");
+        m_flags = b ? m_flags | Rep_SyncStage : m_flags & ~Rep_SyncStage;
+    }
+    //-----------------------------------------------------------------------------
+    void Replica::SetMigratable(bool migratable)
+    {
+        AZ_Assert(!IsActive(), "Migration capabilities can only be set before a replica is registered!");
+        m_flags = migratable ? m_flags | Rep_CanMigrate : m_flags & ~Rep_CanMigrate;
+    }
+    //-----------------------------------------------------------------------------
+    bool Replica::IsSuspendDownstream() const
+    {
+        return !!(m_flags & Rep_SuspendDownstream);
+    }
+    //-----------------------------------------------------------------------------
+    bool Replica::ProcessRPCs(const ReplicaContext& rc)
+    {
+        AZ_PROFILE_TIMER("GridMate");
+
+        bool isProcessed = true;
+        for (auto chunk : m_chunks)
+        {
+            if (chunk)
+            {
+                isProcessed &= chunk->ProcessRPCs(rc);
+            }
+        }
+
+        if (!isProcessed) // have some rpcs left that might require forwarding to other peers so marking replica dirty for next marshaling
+        {
+            rc.m_rm->OnReplicaChanged(this);
+        }
+
+        return isProcessed;
+    }
+    //-----------------------------------------------------------------------------
+    void Replica::ClearPendingRPCs()
+    {
+        for (auto chunk : m_chunks)
+        {
+            if (chunk)
+            {
+                chunk->ClearPendingRPCs();
+            }
+        }
+    }
+    //-----------------------------------------------------------------------------
+    void Replica::OnReplicaPriorityUpdated(ReplicaChunkBase* modifiedChunk)
+    {
+        (void)modifiedChunk;
+        ReplicaPriority maxRepPri = 0;
+        for (const auto& chunk : m_chunks)
+        {
+            if (chunk)
+            {
+                maxRepPri = AZStd::GetMax(maxRepPri, chunk->GetPriority());
+            }
+        }
+
+        m_priority = maxRepPri;
+
+        if (m_manager)
+        {
+            m_manager->OnReplicaPriorityUpdated(this);
+        }
+    }
+    //-----------------------------------------------------------------------------
+    void Replica::MarkRPCsAsRelayed()
+    {
+        for (auto chunk : m_chunks)
+        {
+            if (chunk)
+            {
+                chunk->MarkRPCsAsRelayed();
+            }
+        }
+    }
+    //-----------------------------------------------------------------------------
+    void Replica::SetRepId(ReplicaId id)
+    {
+        m_myId = id;
+    }
+    //-----------------------------------------------------------------------------
+    PrepareDataResult Replica::PrepareData(EndianType endianType, AZ::u32 marshalFlags)
+    {
+        //AZ_PROFILE_TIMER("GridMate");
+
+        PrepareDataResult pdr(false, false, false, false);
+        bool dataSetChange = false;
+        for (auto chunk : m_chunks)
+        {
+            if (chunk)
+            {
+                PrepareDataResult chunkPDR = chunk->PrepareData(endianType, marshalFlags);
+
+                pdr.m_isDownstreamReliableDirty |= chunkPDR.m_isDownstreamReliableDirty;
+                pdr.m_isDownstreamUnreliableDirty |= chunkPDR.m_isDownstreamUnreliableDirty;
+                pdr.m_isUpstreamReliableDirty |= chunkPDR.m_isUpstreamReliableDirty;
+                pdr.m_isUpstreamUnreliableDirty |= chunkPDR.m_isUpstreamUnreliableDirty;
+                dataSetChange |= chunk->m_reliableDirtyBits.any() | chunk->m_unreliableDirtyBits.any();
+            }
+        }
+
+        if(dataSetChange)
+        {
+            m_revision++;    //If any chunk's dataset changed increase the replica revision.
+        }
+
+        return pdr;
+    }
+    //-----------------------------------------------------------------------------
+    void Replica::Marshal(MarshalContext& mc)
+    {
+        //AZ_PROFILE_TIMER("GridMate");
+
+        // We are going to replace the outBuffer with a temporary chunk buffer for each chunk,
+        // hold on to the original so we can restore it later and write the chunk buffers into
+        WriteBuffer* outBuffer = mc.m_outBuffer;
+
+        mc.m_outBuffer = nullptr;
+
+        AZStd::bitset<GM_MAX_CHUNKS_PER_REPLICA> chunkManifest;
+
+        struct ChunkInfo
+        {
+            ChunkInfo(EndianType endianness)
+                : m_length(endianness)
+                , m_payload(endianness, 0)
+            {
+            }
+
+            WriteBufferStatic<5> m_length;  // length will never need more than 5 bytes.
+            WriteBufferDynamic m_payload;
+        };
+
+        PackedSize payloadLen = 0;
+        AZStd::fixed_vector<ChunkInfo, GM_MAX_CHUNKS_PER_REPLICA> chunkBuffers;
+        for (size_t iChunk = 0; iChunk < m_chunks.size(); ++iChunk)
+        {
+            ReplicaChunkPtr chunk = m_chunks[iChunk];
+
+            if (!chunk)
+            {
+                continue;
+            }
+
+            if (!(mc.m_marshalFlags & ReplicaMarshalFlags::ForceDirty)
+                && !chunk->IsDirty(mc.m_marshalFlags)
+                && !(ReplicaTarget::IsAckEnabled() && (mc.m_peerLatestVersionAckd < chunk->m_revision )) )
+            {
+                /*
+                 * New operation such as NewProxy are optimized to send chunks that are not currently dirty but
+                 * have values are no longer the default constructor values.
+                 */
+                if ((mc.m_marshalFlags & ReplicaMarshalFlags::NewProxy) != ReplicaMarshalFlags::NewProxy)
+                {
+                    continue;
+                }
+            }
+
+            if (!chunk->ShouldSendToPeer(mc.m_peer))
+            {
+                continue;
+            }
+
+            // Add the chunk to the manifest and prepare its buffer
+            chunkManifest.set(iChunk);
+            chunkBuffers.push_back(ChunkInfo(outBuffer->GetEndianType()));
+            ChunkInfo& chunkInfo = chunkBuffers.back();
+            chunkInfo.m_payload.Init(128);
+            mc.m_outBuffer = &chunkInfo.m_payload;
+
+            EBUS_EVENT(Debug::ReplicaDrillerBus, OnSendReplicaChunkBegin, chunk.get(), static_cast<AZ::u32>(iChunk), mc.m_rm->GetLocalPeerId(), mc.m_peer->GetId());
+            PackedSize writeOffset = mc.m_outBuffer->GetExactSize();
+            // Write the ctor data if we need to
+            if (mc.m_marshalFlags & ReplicaMarshalFlags::IncludeCtorData)
+            {
+                mc.m_outBuffer->Write(chunk->GetDescriptor()->GetChunkTypeId());
+                chunk->GetDescriptor()->MarshalCtorData(chunk.get(), *mc.m_outBuffer);
+            }
+
+            // Marshal the chunk data
+            chunk->Marshal(mc, static_cast<AZ::u32>(iChunk));
+            EBUS_EVENT(Debug::ReplicaDrillerBus, OnSendReplicaChunkEnd, chunk.get(), static_cast<AZ::u32>(iChunk), mc.m_outBuffer->Get() + writeOffset.GetBytes(), mc.m_outBuffer->Size() - writeOffset.GetBytes());
+
+            // Precompute the chunk payload length and add to overall replica payload length
+            PackedSize chunkLen = chunkInfo.m_payload.GetExactSize();
+            chunkInfo.m_length.Write(chunkLen);
+
+            payloadLen += chunkLen + chunkInfo.m_length.GetExactSize();
+        }
+
+        if (!chunkBuffers.empty())
+        {
+            mc.m_outBuffer = outBuffer;
+            mc.m_outBuffer->Write(GetRepId());
+
+            WriteBufferStatic<VlqU64Marshaler::MaxEncodingBytes> chunkManifestBuffer(mc.m_outBuffer->GetEndianType());
+            chunkManifestBuffer.Write(chunkManifest.to_ullong(), VlqU64Marshaler());
+
+            payloadLen += chunkManifestBuffer.Size();
+
+            mc.m_outBuffer->Write(payloadLen);
+
+            mc.m_outBuffer->WriteRaw(chunkManifestBuffer.Get(), chunkManifestBuffer.Size());
+
+            for (ChunkInfo& chunkInfo : chunkBuffers)
+            {
+                mc.m_outBuffer->WriteRaw(chunkInfo.m_length.Get(), chunkInfo.m_length.GetExactSize());
+                mc.m_outBuffer->WriteRaw(chunkInfo.m_payload.Get(), chunkInfo.m_payload.GetExactSize());
+            }
+        }
+    }
+    //-----------------------------------------------------------------------------
+    bool Replica::Unmarshal(UnmarshalContext& mc)
+    {
+        AZ_PROFILE_TIMER("GridMate");
+
+        UnmarshalContext chunkContext(mc);
+        ReadBuffer& buffer = *mc.m_iBuf;
+
+        // Add new chunks or update existing ones
+        AZStd::bitset<GM_MAX_CHUNKS_PER_REPLICA> chunkManifest;
+        if (buffer.Read(*reinterpret_cast<AZ::u64*>(chunkManifest.data()), VlqU64Marshaler()))
+        {
+            for (size_t iChunk = 0; iChunk < GM_MAX_CHUNKS_PER_REPLICA && chunkManifest.any(); ++iChunk)
+            {
+                if (chunkManifest.test(iChunk))
+                {
+                    chunkManifest.reset(iChunk);
+
+                    PackedSize chunkSize;
+                    if (!buffer.Read(chunkSize))
+                    {
+                        return false;
+                    }
+
+                    // Generate a buffer bound to the size of the chunk
+                    ReadBuffer innerBuffer = buffer.ReadInnerBuffer(chunkSize);
+                    if (innerBuffer.IsValid())
+                    {
+                        chunkContext.m_iBuf = &innerBuffer;
+                    }
+                    else
+                    {
+                        AZ_Warning("GridMate", false, "We're going to read too much data to unmarshal properly");
+                        return false;
+                    }
+
+                    ReplicaChunkPtr chunk = m_chunks.size() > iChunk ? m_chunks[iChunk] : nullptr;
+                    if (mc.m_hasCtorData)
+                    {
+                        ReplicaChunkClassId repChunkClassId;
+                        if (!innerBuffer.Read(repChunkClassId))
+                        {
+                            return false;
+                        }
+
+                        if (!chunk)
+                        {
+                            chunk = CreateReplicaChunkFromStream(repChunkClassId, chunkContext);
+                            AZ_Warning("GridMate", chunk, "Received unknown replica chunk type 0x%x at index %d, discarding %d bytes and %d bits.",
+                                repChunkClassId, static_cast<int>(iChunk), static_cast<int>(innerBuffer.Left().GetBytes()),
+                                static_cast<int>(innerBuffer.Left().GetAdditionalBits()));
+                        }
+                        else
+                        {
+                            chunk->GetDescriptor()->DiscardCtorStream(chunkContext);
+                        }
+                    }
+
+                    if (chunk)
+                    {
+                        EBUS_EVENT(Debug::ReplicaDrillerBus, OnReceiveReplicaChunkBegin, chunk.get(), static_cast<AZ::u32>(iChunk), chunkContext.m_peer->GetId(), chunkContext.m_rm->GetLocalPeerId(), innerBuffer.Get(), chunkSize.GetSizeInBytesRoundUp());
+                        chunk->Unmarshal(chunkContext, static_cast<AZ::u32>(iChunk));
+                        EBUS_EVENT(Debug::ReplicaDrillerBus, OnReceiveReplicaChunkEnd, chunk.get(), static_cast<AZ::u32>(iChunk));
+                    }
+                    else
+                    {
+                        innerBuffer.Skip(innerBuffer.Left());
+                    }
+
+                    AZ_Warning("GridMate", innerBuffer.IsEmpty() && !innerBuffer.IsOverrun(), "Incorrect number of bytes read while unmarshaling chunk index %d, replica 0x%x. Data may be corrupted!", static_cast<int>(iChunk), GetRepId());
+                    innerBuffer.Skip(innerBuffer.Left());
+                }
+            }
+        }
+        return true;
+    }
+    //-----------------------------------------------------------------------------
+    ReplicaChunkPtr Replica::CreateReplicaChunkFromStream(ReplicaChunkClassId classId, UnmarshalContext& mc)
+    {
+        AZ_PROFILE_TIMER("GridMate");
+
+        ReplicaChunkPtr chunk = nullptr;
+        ReplicaChunkDescriptor* pDesc = ReplicaChunkDescriptorTable::Get().FindReplicaChunkDescriptor(classId);
+        if (pDesc)
+        {
+            ReplicaChunkDescriptorTable::Get().BeginConstructReplicaChunk(pDesc);
+            chunk = pDesc->CreateFromStream(mc);
+            ReplicaChunkDescriptorTable::Get().EndConstructReplicaChunk();
+
+            // Push back even if the chunk did not create so the proper indexes are maintained.
+            m_chunks.push_back(chunk);
+
+            if (chunk)
+            {
+                chunk->Init(pDesc);
+                chunk->AttachedToReplica(this);
+                AZ_Assert(chunk->GetReplica() == this, "Must be bound to the same replica");
+            }
+        }
+        return chunk;
+    }
+    //-----------------------------------------------------------------------------
+    bool Replica::IsUpdateFromReplicaEnabled() const
+    {
+        for (const auto& chunk : m_chunks)
+        {
+            if (chunk && !chunk->IsUpdateFromReplicaEnabled())
+            {
+                return false;
+            }
+        }
+
+        return true;
+    }
+    //-----------------------------------------------------------------------------
+    ReplicaChunkPtr Replica::GetChunkByIndex(size_t index)
+    {
+        return m_chunks[index];
+    }
+    //-----------------------------------------------------------------------------
+    bool Replica::IsBroadcast() const
+    {
+        for (const auto& chunk : m_chunks)
+        {
+            if (chunk && chunk->IsBroadcast())
+            {
+                return true;
+            }
+        }
+
+        return false;
+    }
+
+    //-----------------------------------------------------------------------------
+    // CtorContextBase
+    //-----------------------------------------------------------------------------
+    CtorContextBase::CtorDataSetBase::CtorDataSetBase()
+    {
+        CtorContextBase::s_pCur->m_members.push_back(this);
+    }
+    //-----------------------------------------------------------------------------
+    CtorContextBase* CtorContextBase::s_pCur = NULL;
+    //-----------------------------------------------------------------------------
+    CtorContextBase::CtorContextBase()
+    {
+        s_pCur = this;
+    }
+    //-----------------------------------------------------------------------------
+    void CtorContextBase::Marshal(WriteBuffer& wb)
+    {
+        for (MembersArrayType::iterator i = m_members.begin(); i != m_members.end(); ++i)
+        {
+            (*i)->Marshal(wb);
+        }
+    }
+    //-----------------------------------------------------------------------------
+    void CtorContextBase::Unmarshal(ReadBuffer& rb)
+    {
+        for (MembersArrayType::iterator i = m_members.begin(); i != m_members.end(); ++i)
+        {
+            (*i)->Unmarshal(rb);
+        }
+    }
+    //-----------------------------------------------------------------------------
+} // namespace GridMate

Code/Framework/GridMate/GridMate/Replica/Replica.h

@@ -0,0 +1,221 @@
+/*
+* All or portions of this file Copyright (c) Amazon.com, Inc. or its affiliates or
+* its licensors.
+*
+* For complete copyright and license terms please see the LICENSE at the root of this
+* distribution (the "License"). All use of this software is governed by the License,
+* or, if provided, by the license below or the license accompanying this file. Do not
+* remove or modify any license notices. This file is distributed on an "AS IS" BASIS,
+* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+*
+*/
+#ifndef GM_REPLICA_H
+#define GM_REPLICA_H
+
+#include <AzCore/std/containers/intrusive_list.h>
+#include <AzCore/std/typetraits/is_base_of.h>
+
+#include <GridMate/Containers/unordered_set.h>
+#include <GridMate/Containers/vector.h>
+
+#include <GridMate/Replica/ReplicaChunk.h>
+#include <GridMate/Replica/ReplicaStatusInterface.h>
+#include <GridMate/Replica/ReplicaTarget.h>
+
+#include <GridMate/Serialize/MarshalerTypes.h>
+
+namespace GridMate
+{
+    namespace ReplicaInternal
+    {
+        class MigrationSequence;
+    }
+
+    class ReplicaStatus;
+    class ReplicaTask;
+    class InterestManager;
+
+    //-------------------------------------------------------------------------
+    // Replica
+    //-------------------------------------------------------------------------
+    class Replica final
+        : public ReplicaStatusInterface
+    {
+        friend class ReplicaChunkBase;
+        friend class ReplicaManager;
+        friend class ReplicaPeer;
+        friend class ReplicaInternal::MigrationSequence;
+        friend ReplicaStatus;
+
+        friend class ReplicaMarshalTaskBase;
+        friend class ReplicaUpdateTaskBase;
+        friend class ReplicaMarshalUpstreamTask;
+        friend class ReplicaMarshalUpdateTask;
+        friend class RelayRpcsTask;
+        friend class ReplicaMarshalTask;
+        friend class ReplicaMarshalZombieToPeerTask;
+        friend class ReplicaMarshalZombieTask;
+        friend class SendLimitProcessPolicy;
+
+        friend class ReplicaMarshalNewTask;
+
+        friend class InterestManager;
+        friend class ReplicaTarget;
+
+        enum Flags
+        {
+            Rep_SyncStage         = 1 << 0,
+            Rep_ManagedAlloc      = 1 << 1,
+            Rep_CanMigrate        = 1 << 2,
+            Rep_New               = 1 << 3,
+            Rep_Master            = 1 << 4,
+            Rep_Active            = 1 << 6,
+            Rep_ChangedOwner      = 1 << 7,
+            Rep_SuspendDownstream = 1 << 8,
+
+            Rep_Traits = Rep_SyncStage | Rep_ManagedAlloc | Rep_CanMigrate
+        };
+
+    public:
+        typedef vector<ReplicaChunkPtr> ChunkListType;
+
+        GM_CLASS_ALLOCATOR(Replica);
+
+        static Replica* CreateReplica(const char* replicaName);
+
+        void Destroy();
+
+        void UpdateReplica(const ReplicaContext& rc); // Called when updating replica master from source
+        void UpdateFromReplica(const ReplicaContext& rc); // Called when updating game with replica info
+        bool AcceptChangeOwnership(PeerId requestor, const ReplicaContext& rc); // Return true to accept the transfer
+        void OnActivate(const ReplicaContext& rc);
+        void OnDeactivate(const ReplicaContext& rc);
+        void OnChangeOwnership(const ReplicaContext& rc);
+
+        bool AttachReplicaChunk(const ReplicaChunkPtr& chunk);
+        bool DetachReplicaChunk(const ReplicaChunkPtr& chunk);
+
+        ReplicaId GetRepId() const { return m_myId; }
+        PeerId GetPeerId() const;
+        const char* GetDebugName() const;
+        unsigned int GetCreateTime() const { return m_createTime; }
+        ReplicaContext GetMyContext() const;
+        ReplicaManager* GetReplicaManager() { return m_manager; }
+
+        void RegisterMarshalingTask(ReplicaTask* task) { m_marshalingTasks.insert(task); }
+        void UnregisterMarshalingTask(ReplicaTask* task) { m_marshalingTasks.erase(task); }
+        bool HasMarshalingTask() const { return !m_marshalingTasks.empty(); }
+
+        void RegisterUpdateTask(ReplicaTask* task) { m_updateTasks.insert(task); }
+        void UnregisterUpdateTask(ReplicaTask* task) { m_updateTasks.erase(task); }
+        bool HasUpdateTask() const { return !m_updateTasks.empty(); }
+
+        void RequestChangeOwnership(PeerId newOwner = InvalidReplicaPeerId); // If newOwner is not specified we assume it should be the local peer
+
+        bool IsMaster() const { return !IsActive() || !!(m_flags & Rep_Master); }
+        bool IsProxy() const { return !IsMaster(); }
+        bool IsNew() const { return !!(m_flags & Rep_New); }
+        bool IsNewOwner() const { return !!(m_flags & Rep_ChangedOwner); }
+        bool IsActive() const { return !!(m_flags & Rep_Active); }
+        void SetSyncStage(bool b = true);
+        bool IsSyncStage() const { return !!(m_flags & Rep_SyncStage); }
+        bool IsMigratable() const { return !!(m_flags & Rep_CanMigrate); }
+        bool IsDirty() const { return m_dirtyHook.m_prev || m_dirtyHook.m_next; }
+        bool IsBroadcast() const;
+        bool IsUpdateFromReplicaEnabled() const;
+
+        ReplicaPriority GetPriority() const { return m_priority; } // returns replica's priority aggregated across all its chunks
+
+        size_t GetNumChunks() const { return m_chunks.size(); }
+        ReplicaChunkPtr GetChunkByIndex(size_t index);
+
+        template<class R>
+        inline AZStd::intrusive_ptr<R> FindReplicaChunk();
+
+        AZ::u64 GetRevision() const { return m_revision; };
+
+        //---------------------------------------------------------------------
+        // DEBUG and Test Interface. Do not use in production code.
+        //---------------------------------------------------------------------
+        const ReplicaTargetList&    DebugGetTargets() const { return m_targets; }
+        PrepareDataResult           DebugPrepareData(EndianType endian, AZ::u32 marshalFlags) { return PrepareData(endian, marshalFlags); }
+        void                        DebugMarshal(MarshalContext& mc) { Marshal(mc); }
+        void                        DebugPreDestruct() { PreDestruct(); }
+        //---------------------------------------------------------------------
+        
+        explicit Replica(const char* replicaName);
+        ~Replica();
+
+    protected:
+        //---------------------------------------------------------------------
+        // refcount
+        //---------------------------------------------------------------------
+        template<class T>
+        friend struct AZStd::IntrusivePtrCountPolicy;
+        mutable unsigned int m_refCount;
+        AZ_FORCE_INLINE void add_ref() { ++m_refCount; }
+        void release();
+        //---------------------------------------------------------------------
+
+        //
+        // These functions are internal to the replication system
+        //
+        void InitReplica(ReplicaManager* manager); // Initialize internal replica components. Called by ReplicaManager right before calling Activate()
+        void Activate(const ReplicaContext& rc);
+        void Deactivate(const ReplicaContext& rc);
+
+        void PreDestruct();
+
+        ReplicaChunkPtr CreateReplicaChunkFromStream(ReplicaChunkClassId classId, UnmarshalContext& mc);
+
+        void MarkRPCsAsRelayed();
+
+        void SetMaster(bool isMaster) { m_flags = isMaster ? m_flags | Rep_Master : m_flags & ~Rep_Master; }
+        void SetNew() { m_flags |= Rep_New; }
+        void SetRepId(ReplicaId id);
+        void SetMigratable(bool migratable);
+        bool IsSuspendDownstream() const;
+
+        void InternalCreateInitialChunks(const char* replicaName);
+
+        PrepareDataResult PrepareData(EndianType endianType, AZ::u32 marshalFlags = 0);
+        void Marshal(MarshalContext& mc);
+        bool Unmarshal(UnmarshalContext& mc);
+
+        bool ProcessRPCs(const ReplicaContext& rc);
+        void ClearPendingRPCs();
+
+        void OnReplicaPriorityUpdated(ReplicaChunkBase* chunk);
+
+        //---------------------------------------------------------------------
+        // RPC handlers
+        //---------------------------------------------------------------------
+        bool RequestOwnershipFn(PeerId requestor, const RpcContext& rpcContext) override;
+        bool MigrationSuspendUpstreamFn(PeerId ownerId, AZ::u32 requestTime, const RpcContext& rpcContext) override;
+        bool MigrationRequestDownstreamAckFn(PeerId ownerId, AZ::u32 requestTime, const RpcContext& rpcContext) override;
+        //---------------------------------------------------------------------
+
+        ReplicaId           m_myId;
+        AZ::u32             m_flags;
+        unsigned int        m_createTime;
+        ReplicaManager*     m_manager;
+
+        ReplicaPeer*        m_upstreamHop;
+
+        ChunkListType       m_chunks;
+
+        typedef unordered_set<ReplicaTask*> PendingTasks;
+        PendingTasks        m_marshalingTasks;
+        PendingTasks        m_updateTasks;
+        AZStd::intrusive_list_node<Replica> m_dirtyHook;
+        ReplicaChunkPtr     m_replicaStatus;
+        ReplicaTargetList   m_targets;
+        ReplicaPriority     m_priority;
+        AZ::u64             m_revision;              ///< Change stamp. Increases every time a data set changes. Start at 1 to send initial value.
+    };
+    //-----------------------------------------------------------------------------
+} // namespace GridMate
+
+#include "ReplicaInline.inl"
+
+#endif // GM_REPLICA_H

Code/Framework/GridMate/GridMate/Replica/ReplicaChunk.cpp

@@ -0,0 +1,767 @@
+/*
+* All or portions of this file Copyright (c) Amazon.com, Inc. or its affiliates or
+* its licensors.
+*
+* For complete copyright and license terms please see the LICENSE at the root of this
+* distribution (the "License"). All use of this software is governed by the License,
+* or, if provided, by the license below or the license accompanying this file. Do not
+* remove or modify any license notices. This file is distributed on an "AS IS" BASIS,
+* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+*
+*/
+
+#include <AzCore/Debug/Profiler.h>
+
+#include <GridMate/Replica/ReplicaChunkDescriptor.h>
+#include <GridMate/Replica/DataSet.h>
+#include <GridMate/Replica/MigrationSequence.h>
+#include <GridMate/Replica/RemoteProcedureCall.h>
+#include <GridMate/Replica/Replica.h>
+#include <GridMate/Replica/ReplicaMgr.h>
+#include <GridMate/Replica/ReplicaUtils.h>
+#include <GridMate/Replica/ReplicaDrillerEvents.h>
+#include <GridMate/Serialize/CompressionMarshal.h>
+
+namespace GridMate
+{
+    //-----------------------------------------------------------------------------
+    ReplicaChunkBase::ReplicaChunkBase()
+        : m_refCount(0)
+        , m_replica(nullptr)
+        , m_descriptor(nullptr)
+        , m_flags(0)
+        , m_handler(nullptr)
+        , m_reliableDirtyBits()
+        , m_unreliableDirtyBits()
+        , m_nonDefaultValueBits()
+        , m_nDownstreamReliableRPCs(0)
+        , m_nDownstreamUnreliableRPCs(0)
+        , m_nUpstreamReliableRPCs(0)
+        , m_nUpstreamUnreliableRPCs(0)
+        , m_dirtiedDataSets(0xFFFFFFFF)
+        , m_priority(k_replicaPriorityNormal)
+        , m_revision(1)
+    {
+        ReplicaChunkInitContext* initContext = ReplicaChunkDescriptorTable::Get().GetCurrentReplicaChunkInitContext();
+        AZ_Assert(initContext, "Replica's descriptor is NOT pushed on the stack! Call Replica::Desriptor::Push() before construction!");
+        initContext->m_chunk = this;
+        EBUS_EVENT(Debug::ReplicaDrillerBus, OnCreateReplicaChunk, this);
+    }
+    //-----------------------------------------------------------------------------
+    ReplicaChunkBase::~ReplicaChunkBase()
+    {
+        AZ_Assert(m_refCount == 0, "Attempting to free replica with non-zero refCount(%d)!", m_refCount);
+        EBUS_EVENT(Debug::ReplicaDrillerBus, OnDestroyReplicaChunk, this);
+    }
+    //-----------------------------------------------------------------------------
+    void ReplicaChunkBase::Init(ReplicaChunkClassId chunkTypeId)
+    {
+        ReplicaChunkDescriptor* descriptor = ReplicaChunkDescriptorTable::Get().FindReplicaChunkDescriptor(chunkTypeId);
+        AZ_Assert(descriptor, "Init failed. Can't find replica chunk descriptor for chunk type 0x%x!", static_cast<AZ::u32>(chunkTypeId));
+        Init(descriptor);
+    }
+    //-----------------------------------------------------------------------------
+    void ReplicaChunkBase::Init(ReplicaChunkDescriptor* descriptor)
+    {
+        AZ_Assert(descriptor, "Init failed. descriptor can't be null!");
+        AZ_Assert(descriptor->IsInitialized(), "Init failed. Replica chunk descriptor for chunk type 0x%x has not been properly initialized!", static_cast<AZ::u32>(descriptor->GetChunkTypeId()));
+        m_descriptor = descriptor;
+        for (size_t iDataSet = 0; iDataSet < descriptor->GetDataSetCount(); ++iDataSet)
+        {
+            descriptor->GetDataSet(this, iDataSet)->m_replicaChunk = this;
+        }
+        for (size_t iRpc = 0; iRpc < descriptor->GetRpcCount(); ++iRpc)
+        {
+            descriptor->GetRpc(this, iRpc)->m_replicaChunk = this;
+        }
+    }
+    //-----------------------------------------------------------------------------
+    bool ReplicaChunkBase::IsClassType(ReplicaChunkClassId classId) const
+    {
+        return classId == GetDescriptor()->GetChunkTypeId();
+    }
+    //-----------------------------------------------------------------------------
+    ReplicaId ReplicaChunkBase::GetReplicaId() const
+    {
+        if (m_replica)
+        {
+            return m_replica->GetRepId();
+        }
+        return InvalidReplicaId;
+    }
+    //-----------------------------------------------------------------------------
+    PeerId ReplicaChunkBase::GetPeerId() const
+    {
+        PeerId peerId = InvalidReplicaPeerId;
+
+        if (m_replica != nullptr)
+        {
+            ReplicaContext context(m_replica->GetMyContext());
+
+            if (context.m_peer != nullptr)
+            {
+                peerId = context.m_peer->GetId();
+            }
+        }
+
+        return peerId;
+    }
+    //-----------------------------------------------------------------------------
+    ReplicaManager* ReplicaChunkBase::GetReplicaManager()
+    {
+        if (m_replica)
+        {
+            return m_replica->GetReplicaManager();
+        }
+        return nullptr;
+    }
+    //-----------------------------------------------------------------------------
+    bool ReplicaChunkBase::IsActive() const
+    {
+        if (m_replica)
+        {
+            return m_replica->IsActive();
+        }
+        return false;
+    }
+    //-----------------------------------------------------------------------------
+    bool ReplicaChunkBase::IsMaster() const
+    {
+        if (m_replica)
+        {
+            return m_replica->IsMaster();
+        }
+        return true;
+    }
+    //-----------------------------------------------------------------------------
+    bool ReplicaChunkBase::IsProxy() const
+    {
+        return !IsMaster();
+    }
+    //-----------------------------------------------------------------------------
+    bool ReplicaChunkBase::IsDirty(AZ::u32 marshalFlags) const
+    {
+        if (marshalFlags & ReplicaMarshalFlags::IncludeDatasets)
+        {
+            const auto& dirtyBits = (marshalFlags& ReplicaMarshalFlags::Reliable) ? m_reliableDirtyBits : m_unreliableDirtyBits;
+            if (dirtyBits.any())
+            {
+                return true;
+            }
+        }
+
+        // Always send RPCs, no need for a flag
+        return !m_rpcQueue.empty();
+    }
+    //-----------------------------------------------------------------------------
+    PrepareDataResult ReplicaChunkBase::PrepareData(EndianType endianType, AZ::u32 marshalFlags)
+    {
+        //AZ_PROFILE_TIMER("GridMate");
+
+        PrepareDataResult pdr(false, false, false, false);
+        bool forceDatasetsReliable = !!(marshalFlags & ReplicaMarshalFlags::ForceReliable);
+
+        m_nDownstreamReliableRPCs = m_nDownstreamUnreliableRPCs = m_nUpstreamReliableRPCs = m_nUpstreamUnreliableRPCs = 0;
+
+        // RPCs
+        for (auto i = m_rpcQueue.rbegin(); i != m_rpcQueue.rend(); ++i)
+        {
+            Internal::RpcRequest* rpc = *i;
+            if (!rpc->m_relayed)
+            {
+                bool isDownstream = rpc->m_authoritative;
+
+                // If there were reliable rpcs in queue -> keep all preceding rpcs reliable to guarantee the right order of execution
+                if ((pdr.m_isDownstreamReliableDirty && isDownstream) || (pdr.m_isUpstreamReliableDirty && !isDownstream))
+                {
+                    rpc->m_reliable = true;
+                }
+
+                pdr.m_isDownstreamReliableDirty |= isDownstream && rpc->m_reliable;
+                pdr.m_isDownstreamUnreliableDirty |= isDownstream && !rpc->m_reliable;
+                pdr.m_isUpstreamReliableDirty |= !isDownstream && rpc->m_reliable;
+                pdr.m_isUpstreamUnreliableDirty |= !isDownstream && !rpc->m_reliable;
+
+                m_nDownstreamReliableRPCs += isDownstream && rpc->m_reliable ? 1 : 0;
+                m_nDownstreamUnreliableRPCs += isDownstream && !rpc->m_reliable ? 1 : 0;
+                m_nUpstreamReliableRPCs += !isDownstream && rpc->m_reliable ? 1 : 0;
+                m_nUpstreamUnreliableRPCs += !isDownstream && !rpc->m_reliable ? 1 : 0;
+
+                // Force all datasets to be sent reliably if there are post-attached rpcs
+                // This guarantees correct state when post-attached rpcs arrive
+                forceDatasetsReliable |= isDownstream && rpc->m_rpc->IsPostAttached();
+            }
+        }
+
+        // DataSets
+        AZStd::bitset<GM_MAX_DATASETS_IN_CHUNK> dirtyDataSets;
+        ReplicaChunkDescriptor* descriptor = GetDescriptor();
+        for (size_t i = 0; i < descriptor->GetDataSetCount(); ++i)
+        {
+            DataSetBase* dataSet = descriptor->GetDataSet(this, i);
+            PrepareDataResult pdrDs = dataSet->PrepareData(endianType, marshalFlags);
+            dirtyDataSets.set(i, pdrDs.m_isDownstreamReliableDirty | pdrDs.m_isDownstreamUnreliableDirty);
+            forceDatasetsReliable |= pdrDs.m_isDownstreamReliableDirty;
+
+            if (!dataSet->IsDefaultValue())
+            {
+                /*
+                * Mark this dataset as having a non-default value.
+                * Note: default bits are never reset unlike dirty bits.
+                */
+                m_nonDefaultValueBits.set(i);
+            }
+        }
+
+        m_reliableDirtyBits.reset();
+        m_unreliableDirtyBits.reset();
+        if (dirtyDataSets.any())
+        {
+            if (forceDatasetsReliable)
+            {
+                pdr.m_isDownstreamReliableDirty = true;
+                m_reliableDirtyBits = dirtyDataSets;
+            }
+            else
+            {
+                pdr.m_isDownstreamUnreliableDirty = true;
+                m_unreliableDirtyBits = dirtyDataSets;
+            }
+        }
+
+        // If we know that the next data set send will be reliable,
+        // notify the datasets so they can reset their dirty state.
+        if (forceDatasetsReliable || m_replica->IsNew() || m_replica->IsNewOwner())
+        {
+            for (size_t i = 0; i < descriptor->GetDataSetCount(); ++i)
+            {
+                descriptor->GetDataSet(this, i)->ResetDirty();
+            }
+        }
+
+        return pdr;
+    }
+    //-----------------------------------------------------------------------------
+    void ReplicaChunkBase::SetPriority(ReplicaPriority priority)
+    {
+        m_priority = priority;
+
+        if (m_replica)
+        {
+            m_replica->OnReplicaPriorityUpdated(this);
+        }
+    }
+    //-----------------------------------------------------------------------------
+    bool ReplicaChunkBase::ShouldSendToPeer(ReplicaPeer* peer) const
+    {
+        //AZ_PROFILE_TIMER("GridMate");
+
+        // Only send chunks to the same zone as the peer
+        return !!(peer->GetZoneMask() & GetDescriptor()->GetZoneMask());
+    }
+    //-----------------------------------------------------------------------------
+    void ReplicaChunkBase::Marshal(MarshalContext& mc, AZ::u32 chunkIndex)
+    {
+        //AZ_PROFILE_TIMER("GridMate");
+
+        SafeGuardWrite(mc.m_outBuffer, [this, &mc, &chunkIndex]()
+        {
+            MarshalDataSets(mc, chunkIndex);
+            MarshalRpcs(mc, chunkIndex);
+        });
+    }
+    //-----------------------------------------------------------------------------
+    void ReplicaChunkBase::Unmarshal(UnmarshalContext& mc, AZ::u32 chunkIndex)
+    {
+        AZ_PROFILE_TIMER("GridMate");
+
+        SafeGuardRead(mc.m_iBuf, [this, &mc, &chunkIndex]()
+        {
+            UnmarshalDataSets(mc, chunkIndex);
+            UnmarshalRpcs(mc, chunkIndex);
+            m_flags |= RepChunk_Updated;
+        });
+    }
+
+    bool ReplicaChunkBase::ShouldBindToNetwork()
+    {
+        return GetReplica() && GetReplica()->IsActive();
+    }
+
+    //-----------------------------------------------------------------------------
+    AZ::u32 ReplicaChunkBase::CalculateDirtyDataSetMask(MarshalContext& mc)
+    {
+        AZ::u32 dataSetMask = 0;
+
+        if ((mc.m_marshalFlags & ReplicaMarshalFlags::ForceDirty))
+        {
+            // Set all the dataset bits manually because AZStd::bitset doesn't have a ranged set.
+            dataSetMask = static_cast<AZ::u32>((static_cast<AZ::u64>(1) << GetDescriptor()->GetDataSetCount()) - 1);
+        }
+        else if ((mc.m_marshalFlags & ReplicaMarshalFlags::OmitUnmodified))
+        {
+            // Send all bits that have ever been modified.
+            dataSetMask = m_nonDefaultValueBits.to_ulong();
+        }
+        else if ((mc.m_marshalFlags & ReplicaMarshalFlags::IncludeDatasets))
+        {
+            if (mc.m_marshalFlags & ReplicaMarshalFlags::Reliable)
+            {
+                dataSetMask = (*m_reliableDirtyBits.data());
+        }
+            else
+            {
+                dataSetMask = (*m_unreliableDirtyBits.data());
+                //Handle additional unAck'd for specific peer
+                if (ReplicaTarget::IsAckEnabled()
+                    && mc.m_peerLatestVersionAckd != 0)
+                {
+                    AZStd::bitset<GM_MAX_DATASETS_IN_CHUNK> dirtyDataSets;
+                    ReplicaChunkDescriptor* descriptor = GetDescriptor();
+                    for (size_t i = 0; i < descriptor->GetDataSetCount(); ++i)
+                    {
+                        const DataSetBase* dataSet = descriptor->GetDataSet(this, i);
+                        auto rev = dataSet->GetRevision();
+                        const bool isOld = rev > mc.m_peerLatestVersionAckd;
+                        if (isOld)
+                        {
+                            dirtyDataSets.set(i, isOld);
+                        }
+                    }
+                    dataSetMask |= *dirtyDataSets.data();   //Add additional un-ack'd data sets for this target
+                }
+            }
+        }
+
+        return dataSetMask;
+    }
+    //-----------------------------------------------------------------------------
+    void ReplicaChunkBase::MarshalDataSets(MarshalContext& mc, AZ::u32 chunkIndex)
+    {
+        //AZ_PROFILE_TIMER("GridMate");
+        AZ::u32 dirtyDataSetMask = CalculateDirtyDataSetMask(mc);
+        AZStd::bitset<GM_MAX_DATASETS_IN_CHUNK> changebits(dirtyDataSetMask);
+        ReplicaChunkDescriptor* descriptor = GetDescriptor();
+        bool wroteDataSet = false;
+        mc.m_outBuffer->Write(changebits.to_ulong(), VlqU32Marshaler());
+        if (dirtyDataSetMask == 0)
+        {
+            return;
+        }
+        for (size_t i = 0; i < descriptor->GetDataSetCount(); ++i)
+        {
+            if (changebits[i])
+            {
+                DataSetBase* dataset = descriptor->GetDataSet(this, i);
+                if (!dataset)
+                {
+                    AZ_Assert(false, "How can we have a dirty dataset that doesn't exist?")
+                    continue;
+                }
+
+                ReadBuffer data = dataset->GetMarshalData();
+                mc.m_outBuffer->WriteRaw(data.Get(), data.Size());
+                wroteDataSet = true;
+
+                EBUS_EVENT(Debug::ReplicaDrillerBus, OnSendDataSet,
+                    this,
+                    chunkIndex,
+                    dataset,
+                    mc.m_rm->GetLocalPeerId(),
+                    mc.m_peer->GetId(),
+                    data.Get(),
+                    data.Size().GetSizeInBytesRoundUp());
+            }
+        }
+        if(wroteDataSet)
+        {
+            //Add callback here
+            CallbackBuffer* callbackBuffer = mc.m_callbackBuffer;
+            if(mc.m_target && callbackBuffer && ReplicaTarget::IsAckEnabled())
+            {
+                callbackBuffer->push_back(mc.m_target->CreateCallback(m_replica->m_revision));
+            }
+        }
+    }
+    //-----------------------------------------------------------------------------
+    void ReplicaChunkBase::UnmarshalDataSets(UnmarshalContext& mc, AZ::u32 chunkIndex)
+    {
+        AZ_PROFILE_TIMER("GridMate");
+
+        AZStd::bitset<GM_MAX_DATASETS_IN_CHUNK> changebits;
+        if (!mc.m_iBuf->Read(*changebits.data(), VlqU32Marshaler()))
+        {
+            return;
+        }
+
+        if (changebits.any())
+        {
+            if (mc.m_peer != m_replica->m_upstreamHop)
+            {
+                AZ_TracePrintf("GridMate", "Received dataset updates for replica id %08x(%s) from unexpected peer.", GetReplicaId(), IsActive() && IsMaster() ? "master" : "proxy");
+                if (IsMaster())
+                {
+                    mc.m_iBuf->Skip(mc.m_iBuf->Left());
+                    return;
+                }
+            }
+        }
+
+        ReplicaChunkDescriptor* descriptor = GetDescriptor();
+        for (size_t i = 0; i < descriptor->GetDataSetCount(); ++i)
+        {
+            if (changebits[i])
+            {
+                DataSetBase* dataset = descriptor->GetDataSet(this, i);
+                if (!dataset)
+                {
+                    continue;
+                }
+
+                /*
+                 * Whenever we get a dataset from the network, we assume it was modified and thus
+                 * no longer has the default value.
+                 */
+                dataset->MarkAsNonDefaultValue();
+                m_nonDefaultValueBits.set(i);
+
+                const char* readPtr = mc.m_iBuf->GetCurrent();
+                dataset->Unmarshal(mc);
+
+                EBUS_EVENT(Debug::ReplicaDrillerBus, OnReceiveDataSet,
+                    this,
+                    chunkIndex,
+                    dataset,
+                    mc.m_peer->GetId(),
+                    mc.m_rm->GetLocalPeerId(),
+                    readPtr,
+                    mc.m_iBuf->GetCurrent() - readPtr);
+            }
+        }
+    }
+    //-----------------------------------------------------------------------------
+    void ReplicaChunkBase::MarshalRpcs(MarshalContext& mc, AZ::u32 chunkIndex)
+    {
+        //AZ_PROFILE_TIMER("GridMate");
+
+        bool isAuthoritative = (mc.m_marshalFlags & ReplicaMarshalFlags::Authoritative) == ReplicaMarshalFlags::Authoritative;
+        bool isReliable = (mc.m_marshalFlags & ReplicaMarshalFlags::Reliable) == ReplicaMarshalFlags::Reliable;
+
+        AZ::u32 downstreamReliableToSend = isAuthoritative && (isReliable || (mc.m_marshalFlags & ReplicaMarshalFlags::FullSync) == ReplicaMarshalFlags::FullSync) ? m_nDownstreamReliableRPCs : 0;
+        AZ::u32 downstreamUnreliableToSend = isAuthoritative && (!isReliable || (mc.m_marshalFlags & ReplicaMarshalFlags::FullSync) == ReplicaMarshalFlags::FullSync) ? m_nDownstreamUnreliableRPCs : 0;
+        AZ::u32 upstreamReliableToSend = !isAuthoritative && isReliable ? m_nUpstreamReliableRPCs : 0;
+        AZ::u32 upstreamUnreliableToSend = !isAuthoritative && !isReliable ? m_nUpstreamUnreliableRPCs : 0;
+
+        AZ::u32 rpcCount = downstreamReliableToSend + downstreamUnreliableToSend + upstreamReliableToSend + upstreamUnreliableToSend;
+        AZ_Assert(rpcCount < GM_MAX_RPC_SEND_PER_REPLICA, "Attempting to send too many RPCs");
+        mc.m_outBuffer->Write(rpcCount, VlqU32Marshaler());
+
+        AZ::u32 rpcsSent = 0;
+        for (Internal::RpcRequest* rpc : m_rpcQueue)
+        {
+            if (rpc->m_relayed || rpc->m_authoritative != isAuthoritative)
+            {
+                continue;
+            }
+
+            if (rpc->m_reliable != isReliable && (mc.m_marshalFlags & ReplicaMarshalFlags::ForceReliable) != ReplicaMarshalFlags::ForceReliable)
+            {
+                continue;
+            }
+
+            AZ::u8 rpcIndex = static_cast<AZ::u8>(GetDescriptor()->GetRpcIndex(this, rpc->m_rpc));
+
+            auto bufferSize = mc.m_outBuffer->Size();
+
+            SafeGuardWrite(mc.m_outBuffer, [rpc, rpcIndex, &mc]()
+            {
+                mc.m_outBuffer->Write(rpcIndex);
+                rpc->m_rpc->Marshal(*mc.m_outBuffer, rpc);
+            });
+
+            EBUS_EVENT(Debug::ReplicaDrillerBus, OnSendRpc,
+                this,
+                chunkIndex,
+                rpc,
+                mc.m_rm->GetLocalPeerId(),
+                mc.m_peer->GetId(),
+                mc.m_outBuffer->Get() + bufferSize,
+                mc.m_outBuffer->Size() - bufferSize);
+            rpc->m_relayed = !(mc.m_marshalFlags & ReplicaMarshalFlags::Authoritative); // marking upstream rpcs relayed, for downstream rpcs - replicamgr marks them relayed after marshaling is finished
+            rpcsSent++;
+        }
+        AZ_Assert(rpcsSent == rpcCount, "We did not write the expected number of rpcs! sent=%u, expected=%u.", rpcsSent, rpcCount);
+
+        if (!m_rpcQueue.empty() && m_replica->GetReplicaManager())
+        {
+            m_replica->GetReplicaManager()->EnqueueUpdateTask(m_replica);
+        }
+    }
+    //-----------------------------------------------------------------------------
+    void ReplicaChunkBase::UnmarshalRpcs(UnmarshalContext& mc, AZ::u32 chunkIndex)
+    {
+        AZ_PROFILE_TIMER("GridMate");
+
+        // Unmarshal RPCs
+        AZ::u32 rpcCount;
+        if (mc.m_iBuf->Read(rpcCount, VlqU32Marshaler()))
+        {
+            for (AZ::u32 rpcsRead = 0; rpcsRead < rpcCount; ++rpcsRead)
+            {
+                SafeGuardRead(mc.m_iBuf, [this, &mc, &chunkIndex]()
+                {
+                    unsigned char rpcIndex;
+                    if (!mc.m_iBuf->Read(rpcIndex))
+                    {
+                        return;
+                    }
+
+                    RpcBase* rpc = GetDescriptor()->GetRpc(this, rpcIndex);
+                    if (!rpc)
+                    {
+                        AZ_Assert(false, "Cannot find descriptor for rpcIndex %hhu!", rpcIndex);
+                        return;
+                    }
+
+                    const char* dataPtr = mc.m_iBuf->GetCurrent();
+                    Internal::RpcRequest* request = rpc->Unmarshal(*mc.m_iBuf);
+                    if (!request)
+                    {
+                        AZ_Assert(false, "Failed to unmarshal RPC <%s>!", GetDescriptor()->GetRpcName(this, rpc));
+                        return;
+                    }
+
+                    bool isRpcValid = true;
+                    if (request->m_authoritative)
+                    {
+                        if (m_replica->m_upstreamHop != mc.m_peer)
+                        {
+                            AZ_Assert(false, "Discarding authoritative RPC <%s> from %p because it did not come from the expected upstream hop(%p)!", GetDescriptor()->GetRpcName(this, rpc), mc.m_peer, m_replica->m_upstreamHop);
+                            isRpcValid = false;
+                        }
+                    }
+                    else
+                    {
+                        if (!rpc->IsAllowNonAuthoritativeRequests())
+                        {
+                            AZ_Assert(false, "Discarding non-authoritative RPC <%s> because s_allowNonAuthoritativeRequests trait is disabled!", GetDescriptor()->GetRpcName(this, rpc));
+                            isRpcValid = false;
+                        }
+                        if (!rpc->IsAllowNonAuthoritativeRequestsRelay() && !IsMaster())
+                        {
+                            AZ_Assert(false, "Discarding non-authoritative RPC <%s> because s_allowNonAuthoritativeRequestRelay trait is disabled!", GetDescriptor()->GetRpcName(this, rpc));
+                            isRpcValid = false;
+                        }
+                    }
+
+                    if (isRpcValid)
+                    {
+                        if (mc.m_rm->GetSecurityOptions().m_enableStrictSourceValidation)
+                        {
+                            if (!mc.m_peer->IsSyncHost() && !(request->m_authoritative && m_replica->m_upstreamHop == mc.m_peer))
+                            {
+                                request->m_sourcePeer = mc.m_peer->GetId();
+                            }
+                        }
+
+                        if (!request->m_sourcePeer)
+                        {
+                            request->m_sourcePeer = mc.m_peer->GetId();
+                        }
+
+                        size_t dataSize = mc.m_iBuf->GetCurrent() - dataPtr;
+                        EBUS_EVENT(Debug::ReplicaDrillerBus, OnReceiveRpc,
+                            this,
+                            chunkIndex,
+                            request,
+                            mc.m_peer->GetId(),
+                            mc.m_rm->GetLocalPeerId(),
+                            dataPtr,
+                            dataSize);
+                        m_rpcQueue.push_back(request);
+                    }
+                    else
+                    {
+                        delete request;
+                    }
+                });
+            }
+        }
+    }
+    //-----------------------------------------------------------------------------
+    void ReplicaChunkBase::InternalUpdateChunk(const ReplicaContext& rc)
+    {
+        UpdateChunk(rc);
+    }
+    //-----------------------------------------------------------------------------
+    void ReplicaChunkBase::InternalUpdateFromChunk(const ReplicaContext& rc)
+    {
+        if (!(m_flags & ReplicaChunkBase::RepChunk_Updated))
+        {
+            return;
+        }
+
+        // Call events for any upstream modified datasets
+        AZStd::bitset<GM_MAX_DATASETS_IN_CHUNK> eventbits = m_dirtiedDataSets;
+        m_dirtiedDataSets = 0;
+        m_flags &= ~RepChunk_Updated;
+        ReplicaChunkDescriptor* descriptor = GetDescriptor();
+        for (size_t i = 0; i < descriptor->GetDataSetCount(); ++i)
+        {
+            if (eventbits[i])
+            {
+                descriptor->GetDataSet(this, i)->DispatchChangedEvent(rc);
+            }
+        }
+
+        UpdateFromChunk(rc);
+    }
+    //-----------------------------------------------------------------------------
+    void ReplicaChunkBase::AddDataSetEvent(DataSetBase* dataset)
+    {
+        m_dirtiedDataSets |= (1 << GetDescriptor()->GetDataSetIndex(this,dataset));
+    }
+    //-----------------------------------------------------------------------------
+    void ReplicaChunkBase::release()
+    {
+        AZ_Assert(m_refCount > 0, "Reference count logic error, trying to remove reference when refcount is 0");
+        if (--m_refCount == 0)
+        {
+            GetDescriptor()->DeleteReplicaChunk(this);
+        }
+    }
+    //-----------------------------------------------------------------------------
+    bool ReplicaChunkBase::ProcessRPCs(const ReplicaContext& rc)
+    {
+        AZ_PROFILE_TIMER("GridMate");
+
+        // Process incoming RPCs
+        for (RPCQueue::iterator iRPC = m_rpcQueue.begin(); iRPC != m_rpcQueue.end(); )
+        {
+            Internal::RpcRequest* request = *iRPC;
+            bool isMaster = IsMaster(); // need to do this check after each RPC because ownership may change
+
+            if (!m_replica->IsActive()) // this can happen if replica was deactivated within a previous RPC call
+            {
+                request->m_relayed = true;
+            }
+            else if (!request->m_processed && (isMaster || request->m_authoritative))
+            {
+                request->m_realTime = rc.m_realTime;
+                request->m_localTime = rc.m_localTime;
+                bool ret = request->m_rpc->Invoke(request);
+                request->m_processed = true;
+                if (isMaster)
+                {
+                    if (ret)
+                    {
+                        // Trickle back down to proxies
+                        request->m_authoritative = true;
+                    }
+                    else
+                    {
+                        request->m_relayed = true;
+                    }
+                }
+            }
+
+            // This case can happen if the RPC we just invoked caused us to be removed
+            if (m_rpcQueue.empty())
+            {
+                return true;
+            }
+            if (request->m_relayed)
+            {
+                iRPC = m_rpcQueue.erase(iRPC);
+                delete request;
+            }
+            else
+            {
+                ++iRPC;
+            }
+        }
+
+        return m_rpcQueue.empty();
+    }
+    //-----------------------------------------------------------------------------
+    void ReplicaChunkBase::ClearPendingRPCs()
+    {
+        while (!m_rpcQueue.empty())
+        {
+            Internal::RpcRequest* request = *m_rpcQueue.begin();
+            delete request;
+            m_rpcQueue.pop_front();
+        }
+    }
+    //-----------------------------------------------------------------------------
+    void ReplicaChunkBase::MarkRPCsAsRelayed()
+    {
+        for (auto rpc : m_rpcQueue)
+        {
+            if (rpc->m_authoritative)
+            {
+                rpc->m_relayed = true;
+            }
+        }
+    }
+    //-----------------------------------------------------------------------------
+    void ReplicaChunkBase::QueueRPCRequest(GridMate::Internal::RpcRequest* rpc)
+    {
+        m_rpcQueue.push_back(rpc);
+
+        if (m_replica && m_replica->GetReplicaManager())
+        {
+            m_replica->GetReplicaManager()->OnRPCQueued(m_replica);
+        }
+    }
+    //-----------------------------------------------------------------------------
+    void ReplicaChunkBase::SignalDataSetChanged(const DataSetBase& dataset)
+    {
+        OnDataSetChanged(dataset);
+
+        EnqueueMarshalTask();
+    }
+    //-----------------------------------------------------------------------------
+    void ReplicaChunkBase::OnDataSetChanged(const DataSetBase& dataSet)
+    {
+        (void)dataSet;
+    }
+    //-----------------------------------------------------------------------------
+    void ReplicaChunkBase::EnqueueMarshalTask()
+    {
+        if (m_replica && m_replica->GetReplicaManager())
+        {
+            m_replica->GetReplicaManager()->OnReplicaChanged(m_replica);
+        }
+    }
+    //-----------------------------------------------------------------------------
+    void ReplicaChunkBase::AttachedToReplica(Replica* replica)
+    {
+        AZ_PROFILE_TIMER("GridMate");
+
+        AZ_Assert(!m_replica, "Should not be attached to a replica");
+
+        m_replica = replica;
+
+        EBUS_EVENT(Debug::ReplicaDrillerBus, OnAttachReplicaChunk, this);
+        {
+            GM_PROFILE_USER_CALLBACK("OnAttachedToReplica");
+            OnAttachedToReplica(replica);
+        }
+    }
+    //-----------------------------------------------------------------------------
+    void ReplicaChunkBase::DetachedFromReplica()
+    {
+        AZ_PROFILE_TIMER("GridMate");
+
+        AZ_Assert(m_replica, "Should be attached to a replica");
+        EBUS_EVENT(Debug::ReplicaDrillerBus, OnDetachReplicaChunk, this);
+        {
+            GM_PROFILE_USER_CALLBACK("OnDetachedFromReplica");
+            OnDetachedFromReplica(m_replica);
+        }
+
+        m_replica = nullptr;
+        ClearPendingRPCs();
+    }
+} // namespace GridMate

Code/Framework/GridMate/GridMate/Replica/ReplicaChunk.h

@@ -0,0 +1,254 @@
+/*
+* All or portions of this file Copyright (c) Amazon.com, Inc. or its affiliates or
+* its licensors.
+*
+* For complete copyright and license terms please see the LICENSE at the root of this
+* distribution (the "License"). All use of this software is governed by the License,
+* or, if provided, by the license below or the license accompanying this file. Do not
+* remove or modify any license notices. This file is distributed on an "AS IS" BASIS,
+* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+*
+*/
+#ifndef GM_REPLICA_CHUNK_H
+#define GM_REPLICA_CHUNK_H
+
+/** \file ReplicaChunk.h
+    ReplicaChunk is a logical unit of network data for replication across the network. This file contains the base functionality for a
+    ReplicaChunk. The user is expected to extend a ReplicaChunk object to create their own networkable classes.
+*/
+
+#include <GridMate/Containers/unordered_set.h>
+#include <GridMate/Containers/list.h>
+
+#include <GridMate/Replica/ReplicaCommon.h>
+#include <GridMate/Replica/ReplicaChunkInterface.h>
+
+#include <AzCore/std/containers/bitset.h>
+#include <AzCore/std/containers/ring_buffer.h>
+
+namespace UnitTest
+{
+    template <typename ComponentType>
+    class NetContextMarshalFixture;
+}
+
+namespace GridMate
+{
+    class ReplicaChunkDescriptor;
+
+    namespace Internal
+    {
+        struct RpcRequest;
+    }
+
+    // Replica Chunk Base
+    /** A single unit of network functionality
+        A ReplicaChunk is a user extendable network object. One or more ReplicaChunks can be
+        owned by a Replica, which is both a container and manager for them. A replica is owned
+        by a Master, and is propagated to other network nodes, who interact with is as a Proxy.
+        The data a ReplicaChunk contains should generally be related to the other data stored
+        within it. Since multiple chunks can be attached to a Replica, unrelated data can simply
+        be stored in other chunks on the same Replica.
+
+        A ReplicaChunk has two primary ways to interact with it: DataSets and Remote Procedure
+        Calls (RPCs).
+        DataSets store arbitrary data, which only the Master is able to modify. Any changes are
+        propagated to the Proxy ReplicaChunks on the other nodes.
+        RPCs are methods that can be executed on a remote node. They are first invoked on the
+        Master, who then decides if the invocation should be propagated to the Proxies.
+
+        ReplicaChunks can be created by inheriting from the class and registered by calling
+        ReplicaChunkDescriptorTable::RegisterChunkType() to create the factory required by
+        the network.
+        Every concrete replica chunk type needs to implement a static member function
+        const char* GetChunkName(). The string returned by this function will be used to generate
+        a ReplicaChunkClassId which will be used to identify this chunk type throughout the
+        system.
+
+        Replica chunks can be instantiated directly in a Replica, or standalone and attached to a Replica
+        afterwards. Once attached to a replica they are bound to the network.
+
+        To add a handler interface for RPC calls and DataSet changed events, call SetHandler with
+        an object that inherits from ReplicaChunkInterface.
+
+        Use ReplicaChunkBase as the parent class when the event handler logic is separate from the
+        chunk itself. This is useful for example when a client and server want to connect different
+        logic to the chunk.
+    */
+    class ReplicaChunkBase
+    {
+    public:
+        friend Replica;
+        friend RpcBase;
+        friend DataSetBase;
+        template<typename DataType, typename Marshaler, typename Throttle>
+        friend class DataSet;
+        template <typename ComponentType>
+        friend class UnitTest::NetContextMarshalFixture;
+
+        using RPCQueue = AZStd::ring_buffer<Internal::RpcRequest*, SysContAlloc>;
+        /**
+         * \brief Specify the maximum size of a RPC queues for each replica chunk.
+         * This queue can grow while RPCs are being delivered back to all clients.
+         */
+        static constexpr AZStd::size_t MaxRpcQueueSize = 512;
+
+        ReplicaChunkBase();
+        virtual ~ReplicaChunkBase();
+
+        //! Initializes the chunk. Must be called before the chunk can be used.
+        void Init(ReplicaChunkClassId chunkTypeId);
+        void Init(ReplicaChunkDescriptor* descriptor);
+
+        bool IsClassType(ReplicaChunkClassId classId) const;
+
+        ReplicaChunkDescriptor* GetDescriptor() const { return m_descriptor; }
+        ReplicaId GetReplicaId() const;
+        PeerId GetPeerId() const;
+        virtual ReplicaManager* GetReplicaManager();
+        bool IsActive() const;
+        bool IsMaster() const;
+        bool IsProxy() const;
+
+        virtual void OnAttachedToReplica(Replica* replica) { (void) replica; }
+        virtual void OnDetachedFromReplica(Replica* replica) { (void) replica; }
+
+        virtual bool IsReplicaMigratable() = 0; // Return true to allow migration. A single chunk rejecting migration will prevent the replica itself from migrating
+
+        virtual void UpdateChunk(const ReplicaContext& rc) { (void) rc; } // Called when updating replica with game info
+        virtual void UpdateFromChunk(const ReplicaContext& rc) { (void) rc; } // Called when updating game with replica info
+
+        virtual bool AcceptChangeOwnership(PeerId requestor, const ReplicaContext& rc) { (void) requestor; (void) rc; return true; } // Return true to accept the transfer
+        virtual void OnReplicaActivate(const ReplicaContext& rc) { (void) rc; }
+        virtual void OnReplicaDeactivate(const ReplicaContext& rc) { (void) rc; }
+        virtual void OnReplicaChangeOwnership(const ReplicaContext& rc) { (void) rc; }
+
+        virtual bool IsUpdateFromReplicaEnabled() { return true; } // Return false to suspend getting updates from replica, rpcs and dataset changes callbacks will be queued
+
+        Replica* GetReplica() { return m_replica; }
+
+        void SetHandler(ReplicaChunkInterface* handler) { m_handler = handler; }
+        ReplicaChunkInterface* GetHandler() { return m_handler; }
+
+        ReplicaPriority GetPriority() const { return m_priority; }
+        void SetPriority(ReplicaPriority priority);
+
+        virtual bool ShouldSendToPeer(ReplicaPeer* peer) const;
+
+        template<typename T>
+        bool IsType()
+        {
+            return IsClassType(ReplicaChunkClassId(T::GetChunkName()));
+        }
+
+        virtual bool IsBroadcast() { return false; }
+
+        AZ::u64 GetLastChangeStamp() const { return m_revision; };
+
+        virtual bool ShouldBindToNetwork();
+
+    private:
+        //---------------------------------------------------------------------
+        // DEBUG and Test Interface. Do not use in production code.
+        //---------------------------------------------------------------------
+        virtual AZ::u32 Debug_CalculateDirtyDataSetMask(MarshalContext& mc) { return CalculateDirtyDataSetMask(mc); }
+        virtual void Debug_OnDataSetChanged(const DataSetBase& dataSet) { OnDataSetChanged(dataSet); }
+        virtual void Debug_Marshal(MarshalContext& mc, AZ::u32 chunkIndex) { Marshal(mc, chunkIndex); }
+        virtual void Debug_Unmarshal(UnmarshalContext& mc, AZ::u32 chunkIndex) { Unmarshal(mc, chunkIndex); }
+        PrepareDataResult Debug_PrepareData(EndianType endianType, AZ::u32 marshalFlags) { return PrepareData(endianType, marshalFlags); }
+        void Debug_AttachedToReplica(Replica* replica) { AttachedToReplica(replica); }
+
+    protected:
+
+        virtual AZ::u32 CalculateDirtyDataSetMask(MarshalContext& mc);
+        virtual void OnDataSetChanged(const DataSetBase& dataSet);
+        virtual void Marshal(MarshalContext& mc, AZ::u32 chunkIndex);
+        virtual void Unmarshal(UnmarshalContext& mc, AZ::u32 chunkIndex);
+
+        //---------------------------------------------------------------------
+        // refcount
+        //---------------------------------------------------------------------
+        template<class T>
+        friend struct AZStd::IntrusivePtrCountPolicy;
+        mutable unsigned int m_refCount;
+        AZ_FORCE_INLINE void add_ref() { ++m_refCount; }
+        void release();
+
+        void AttachedToReplica(Replica* replica);
+        void DetachedFromReplica();
+
+        bool IsDirty(AZ::u32 marshalFlags) const;
+        PrepareDataResult PrepareData(EndianType endianType, AZ::u32 marshalFlags);
+
+        void MarshalDataSets(MarshalContext& mc, AZ::u32 chunkIndex);
+        void MarshalRpcs(MarshalContext& mc, AZ::u32 chunkIndex);
+
+        void UnmarshalDataSets(UnmarshalContext& mc, AZ::u32 chunkIndex);
+        void UnmarshalRpcs(UnmarshalContext& mc, AZ::u32 chunkIndex);
+
+        void InternalUpdateChunk(const ReplicaContext& rc); // Called when updating replica with game info
+        void InternalUpdateFromChunk(const ReplicaContext& rc); // Called when updating game with replica info
+
+        void AddDataSetEvent(DataSetBase* dataset); // Called to enqueue a user event handler for a modified DataSet on a proxy node
+
+        void SignalDataSetChanged(const DataSetBase& dataset); // Called when the DataSet changes on the master node
+
+        void EnqueueMarshalTask();
+
+        void QueueRPCRequest(GridMate::Internal::RpcRequest* rpc);
+        bool ProcessRPCs(const ReplicaContext& rc);
+        void MarkRPCsAsRelayed();
+        void ClearPendingRPCs();
+
+        enum Flags
+        {
+            RepChunk_Updated = 1 << 0
+        };
+
+        Replica *                   m_replica;
+        ReplicaChunkDescriptor *    m_descriptor;
+        AZ::u32                     m_flags;
+
+        RPCQueue m_rpcQueue{MaxRpcQueueSize};
+        ReplicaChunkInterface* m_handler;
+
+        AZStd::bitset<GM_MAX_DATASETS_IN_CHUNK> m_reliableDirtyBits;
+        AZStd::bitset<GM_MAX_DATASETS_IN_CHUNK> m_unreliableDirtyBits;
+
+        /*
+         * Each bit value of 0 marks a dataset as still having the default value from the initial creation of the replica.
+         * A bit value of 1 indicates that the associated dataset has been modified since its default constructor value.
+         *
+         * Internally, this is used to optimize marshaling of datasets to new proxies by omitting sending default constructor values of datasets.
+         */
+        AZStd::bitset<GM_MAX_DATASETS_IN_CHUNK> m_nonDefaultValueBits;
+
+        AZ::u32 m_nDownstreamReliableRPCs;
+        AZ::u32 m_nDownstreamUnreliableRPCs;
+        AZ::u32 m_nUpstreamReliableRPCs;
+        AZ::u32 m_nUpstreamUnreliableRPCs;
+
+        AZ::u32 m_dirtiedDataSets; // Downstream changed DataSet bits for triggering the event handler
+        ReplicaPriority m_priority;
+        AZ::u64             m_revision; //change stamp. increases every time a data set changes
+    };
+
+    // Replica Chunk - custom class for internal GridMate classes. You should use @ReplicaChunkBase for AZ::Component work!
+    /**
+        Use ReplicaChunk as a parent class when the chunk contains the logic for its network events.
+        This is useful for peer to peer environments and when the same code can be shared between
+        client and server.
+    **/
+    class ReplicaChunk
+        : public ReplicaChunkBase
+        , public ReplicaChunkInterface
+    {
+    public:
+        ReplicaChunk()
+        {
+            SetHandler(this);
+        }
+    };
+} // namespace GridMate
+
+#endif // GM_REPLICA_CHUNK_H

Code/Framework/GridMate/GridMate/Replica/ReplicaChunkDescriptor.cpp

@@ -0,0 +1,306 @@
+/*
+* All or portions of this file Copyright (c) Amazon.com, Inc. or its affiliates or
+* its licensors.
+*
+* For complete copyright and license terms please see the LICENSE at the root of this
+* distribution (the "License"). All use of this software is governed by the License,
+* or, if provided, by the license below or the license accompanying this file. Do not
+* remove or modify any license notices. This file is distributed on an "AS IS" BASIS,
+* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+*
+*/
+
+#include <GridMate/Replica/ReplicaChunkDescriptor.h>
+#include <GridMate/Replica/DataSet.h>
+#include <GridMate/Replica/ReplicaStatus.h>
+#include <GridMate/Replica/SystemReplicas.h>
+
+namespace GridMate
+{
+    //=========================================================================
+    // ReplicaChunkDescriptor
+    //=========================================================================
+    ReplicaChunkDescriptor::ReplicaChunkDescriptor(const char* pNameStr, AZStd::size_t classSize)
+        : m_chunkTypeId(pNameStr)
+        , m_chunkClassName(pNameStr)
+        , m_chunkClassSize(classSize)
+        , m_isInitialized(false)
+    {
+    }
+    //-----------------------------------------------------------------------------
+    void ReplicaChunkDescriptor::RegisterDataSet(const char* debugName, DataSetBase* ds)
+    {
+        ReplicaChunkBase* chunk = ReplicaChunkDescriptorTable::Get().GetCurrentReplicaChunkInitContext()->m_chunk;
+        AZ_Assert(chunk, "Replica chunk pointer was not pushed on the stack! Datasets can only be members of replica chunks!");
+
+        if (!m_isInitialized)
+        {
+            size_t dsOffset = reinterpret_cast<size_t>(ds);
+            size_t baseOffset = reinterpret_cast<size_t>(chunk);
+            AZ_Assert(baseOffset <= dsOffset && baseOffset + m_chunkClassSize > dsOffset, "Dataset offset is not within its parent's boundaries. Datasets must be part of replica chunks!");
+            const char* finalName = (debugName && strlen(debugName) ? debugName : "<Unknown DataSet>");
+            ptrdiff_t offset = dsOffset - baseOffset;
+            RegisterDataSet(finalName, offset);
+        }
+    }
+
+    //-----------------------------------------------------------------------------
+    void ReplicaChunkDescriptor::RegisterDataSet(const char* debugName, ptrdiff_t offset)
+    {
+        if (!m_isInitialized)
+        {
+            const char* finalName = (debugName && strlen(debugName) ? debugName : "<Unknown DataSet>");
+            AZ_Assert(m_vdt.size() < GM_MAX_DATASETS_IN_CHUNK, "Replica chunks can only support up to %d datasets.", GM_MAX_DATASETS_IN_CHUNK);
+            m_vdt.push_back();
+            m_vdt.back().m_offset = offset;
+            m_vdt.back().m_debugName = finalName;
+        }
+    }
+    //-----------------------------------------------------------------------------
+
+    DataSetBase* ReplicaChunkDescriptor::GetDataSet(const ReplicaChunkBase* base, size_t index) const
+    {
+        AZ_Assert(index < m_vdt.size(), "Invalid DataSet index!");
+        return reinterpret_cast<DataSetBase*>(reinterpret_cast<size_t>(base) + m_vdt[index].m_offset);
+    }
+    //-----------------------------------------------------------------------------
+    size_t ReplicaChunkDescriptor::GetDataSetIndex(const ReplicaChunkBase* base, const DataSetBase* dataset) const
+    {
+        ptrdiff_t offset = reinterpret_cast<size_t>(dataset) - reinterpret_cast<size_t>(base);
+        return GetDataSetIndex(offset);
+    }
+    //-----------------------------------------------------------------------------
+    size_t ReplicaChunkDescriptor::GetDataSetIndex(ptrdiff_t offset) const
+    {
+        for (size_t i = 0; i < m_vdt.size(); ++i)
+        {
+            if (m_vdt[i].m_offset == offset)
+            {
+                return i;
+            }
+        }
+        AZ_Assert(false, "Can't find DataSet index! Please check that DataSet pointer is valid!");
+        return static_cast<size_t>(-1);
+    }
+    //-----------------------------------------------------------------------------
+    const char* ReplicaChunkDescriptor::GetDataSetName(const ReplicaChunkBase* base, const DataSetBase* dataset) const
+    {
+        ptrdiff_t offset = reinterpret_cast<size_t>(dataset) - reinterpret_cast<size_t>(base);
+        for (size_t i = 0; i < m_vdt.size(); ++i)
+        {
+            if (m_vdt[i].m_offset == offset)
+            {
+                return m_vdt[i].m_debugName;
+            }
+        }
+        return "<Unknown DataSet>";
+    }
+    //-----------------------------------------------------------------------------
+    void ReplicaChunkDescriptor::RegisterRPC(const char* debugName, RpcBase* rpc)
+    {
+        ReplicaChunkBase* chunk = ReplicaChunkDescriptorTable::Get().GetCurrentReplicaChunkInitContext()->m_chunk;
+        AZ_Assert(chunk, "Replica chunk pointer was not pushed on the stack! RPCs can only be members of replica chunks!");
+
+        if (!m_isInitialized)
+        {
+            size_t rpcOffset = reinterpret_cast<size_t>(rpc);
+            size_t baseOffset = reinterpret_cast<size_t>(chunk);
+            AZ_Assert(baseOffset <= rpcOffset && baseOffset + m_chunkClassSize > rpcOffset, "RPC offset is not within its parent's boundaries. RPCs must be part of replica chunks!");
+            const char* finalName = (debugName && strlen(debugName) ? debugName : "<Unknown RPC>");
+            ptrdiff_t offset = rpcOffset - baseOffset;
+            RegisterRPC(finalName, offset);
+        }
+    }
+    //-----------------------------------------------------------------------------
+    void ReplicaChunkDescriptor::RegisterRPC(const char* debugName, ptrdiff_t offset)
+    {
+        if (!m_isInitialized)
+        {
+            const char* finalName = (debugName && strlen(debugName) ? debugName : "<Unknown RPC>");
+            AZ_Assert(m_vrt.size() < GM_MAX_RPCS_DECL_PER_CHUNK, "Replica chunks can only support up to %d RPCs.", GM_MAX_RPCS_DECL_PER_CHUNK);
+            m_vrt.push_back();
+            m_vrt.back().m_offset = offset;
+            m_vrt.back().m_debugName = finalName;
+        }
+    }
+    //-----------------------------------------------------------------------------
+    RpcBase* ReplicaChunkDescriptor::GetRpc(const ReplicaChunkBase* base, size_t index) const
+    {
+        if (index < m_vrt.size())
+        {
+            return reinterpret_cast<RpcBase*>(reinterpret_cast<size_t>(base)+m_vrt[index].m_offset);
+        }
+
+        AZ_Warning("GridMate", false, "Invalid RPC index!");
+        return nullptr;
+    }
+    //-----------------------------------------------------------------------------
+    size_t ReplicaChunkDescriptor::GetRpcIndex(const ReplicaChunkBase* base, const RpcBase* rpc) const
+    {
+        ptrdiff_t offset = reinterpret_cast<size_t>(rpc) - reinterpret_cast<size_t>(base);
+        return GetRpcIndex(offset);
+    }
+    //-----------------------------------------------------------------------------
+    size_t ReplicaChunkDescriptor::GetRpcIndex(ptrdiff_t offset) const
+    {
+        for (size_t i = 0; i < m_vrt.size(); ++i)
+        {
+            if (m_vrt[i].m_offset == offset)
+            {
+                return i;
+            }
+        }
+        AZ_Assert(false, "Can't find RPC index! Please check that rpc pointer is valid!");
+        return static_cast<size_t>(-1);
+    }
+    //-----------------------------------------------------------------------------
+    const char* ReplicaChunkDescriptor::GetRpcName(const ReplicaChunkBase* base, const RpcBase* rpc) const
+    {
+        ptrdiff_t offset = reinterpret_cast<size_t>(rpc) - reinterpret_cast<size_t>(base);
+        for (size_t i = 0; i < m_vrt.size(); ++i)
+        {
+            if (m_vrt[i].m_offset == offset)
+            {
+                return m_vrt[i].m_debugName;
+            }
+        }
+        return "<Unknown RPC>";
+    }
+    //-----------------------------------------------------------------------------
+
+    //=========================================================================
+    // ReplicaChunkDescriptorTable
+    //=========================================================================
+#   define GRIDMATE_DESCRIPTOR_TABLE_VARIABLE_NAME AZ_CRC("GridMateReplicaChunkDescriptorTable", 0xd1d00091)
+#   define GRIDMATE_CHUNK_INIT_CONTEXT_STACK_VARIABLE_NAME AZ_CRC("GridMateReplicaChunkInitContextStack", 0x67fbe724)
+
+    ReplicaChunkDescriptorTable ReplicaChunkDescriptorTable::s_theTable;
+    //-----------------------------------------------------------------------------
+    ReplicaChunkDescriptorTable& ReplicaChunkDescriptorTable::Get()
+    {
+        if (!s_theTable.m_globalDescriptorTable)
+        {
+            s_theTable.m_globalDescriptorTable = AZ::Environment::CreateVariable<DescriptorContainerType>(GRIDMATE_DESCRIPTOR_TABLE_VARIABLE_NAME);
+
+            // Register all the internal replica chunk types
+            if (!s_theTable.FindReplicaChunkDescriptor(ReplicaChunkClassId(ReplicaStatus::GetChunkName())))
+            {
+                ReplicaStatus::RegisterType();
+            }
+            if (!s_theTable.FindReplicaChunkDescriptor(ReplicaChunkClassId(ReplicaInternal::SessionInfo::GetChunkName())))
+            {
+                ReplicaInternal::SessionInfo::RegisterType();
+            }
+            if (!s_theTable.FindReplicaChunkDescriptor(ReplicaChunkClassId(ReplicaInternal::PeerReplica::GetChunkName())))
+            {
+                ReplicaInternal::PeerReplica::RegisterType();
+            }
+        }
+
+        if (!s_theTable.m_globalChunkInitContextStack)
+        {
+            s_theTable.m_globalChunkInitContextStack = AZ::Environment::CreateVariable<ReplicaChunkInitContextStack>(GRIDMATE_CHUNK_INIT_CONTEXT_STACK_VARIABLE_NAME);
+        }
+
+        return s_theTable;
+    }
+    //-----------------------------------------------------------------------------
+    ReplicaChunkDescriptorTable::~ReplicaChunkDescriptorTable()
+    {
+        // This cannot currently be shut down on some platforms, as this static is shutdown after
+        // all allocators (and in this case, specifically the OSAllocator) are gone
+        if (AZ::AllocatorInstance<AZ::OSAllocator>::IsReady())
+        {
+            while (!m_moduleDescriptorTable.empty())
+            {
+                UnregisterReplicaChunkDescriptor(m_moduleDescriptorTable.back().m_chunkTypeId);
+            }
+        }
+    }
+    //-----------------------------------------------------------------------------
+    ReplicaChunkDescriptor* ReplicaChunkDescriptorTable::FindReplicaChunkDescriptor(ReplicaChunkClassId chunkTypeId)
+    {
+        for (DescriptorInfo& info : * m_globalDescriptorTable)
+        {
+            if (chunkTypeId == info.m_chunkTypeId)
+            {
+                return info.m_descriptor;
+            }
+        }
+        return nullptr;
+    }
+    //-----------------------------------------------------------------------------
+    bool ReplicaChunkDescriptorTable::UnregisterReplicaChunkDescriptor(ReplicaChunkClassId chunkTypeId)
+    {
+        for (DescriptorContainerType::iterator itModuleTable = m_moduleDescriptorTable.begin(); itModuleTable != m_moduleDescriptorTable.end(); ++itModuleTable)
+        {
+            DescriptorInfo* descInfo = &*itModuleTable;
+            if (chunkTypeId == descInfo->m_chunkTypeId)
+            {
+                azdestroy(descInfo->m_descriptor, AZ::OSAllocator);
+                m_moduleDescriptorTable.erase(itModuleTable);
+                delete descInfo;
+                if (UnregisterReplicaChunkDescriptorFromGlobalTable(chunkTypeId))
+                {
+                    return true;
+                }
+                AZ_TracePrintf("GridMate", "Failed to find replica chunk descriptor in global table! Removing from local table.");
+                return false;
+            }
+        }
+        AZ_TracePrintf("GridMate", "Failed to find replica chunk descriptor in local table! Descriptor cannot be unregistered from this module!");
+        return false;
+    }
+    //-----------------------------------------------------------------------------
+    void ReplicaChunkDescriptorTable::AddReplicaChunkDescriptor(ReplicaChunkClassId chunkTypeId, ReplicaChunkDescriptor* descriptor)
+    {
+        DescriptorInfo* localDescInfo = aznew DescriptorInfo();
+        localDescInfo->m_chunkTypeId = chunkTypeId;
+        localDescInfo->m_descriptor = descriptor;
+        m_moduleDescriptorTable.push_back(*localDescInfo);
+
+        DescriptorInfo* globalDescInfo = aznew DescriptorInfo();
+        globalDescInfo->m_chunkTypeId = chunkTypeId;
+        globalDescInfo->m_descriptor = descriptor;
+        m_globalDescriptorTable->push_back(*globalDescInfo);
+    }
+    //-----------------------------------------------------------------------------
+    bool ReplicaChunkDescriptorTable::UnregisterReplicaChunkDescriptorFromGlobalTable(ReplicaChunkClassId chunkTypeId)
+    {
+        for (DescriptorContainerType::iterator itGlobalTable = m_globalDescriptorTable->begin(); itGlobalTable != m_globalDescriptorTable->end(); ++itGlobalTable)
+        {
+            DescriptorInfo* descInfo = &*itGlobalTable;
+            if (descInfo->m_chunkTypeId == chunkTypeId)
+            {
+                m_globalDescriptorTable->erase(itGlobalTable);
+                delete descInfo;
+                return true;
+            }
+        }
+        return false;
+    }
+    //-----------------------------------------------------------------------------
+    void ReplicaChunkDescriptorTable::BeginConstructReplicaChunk(ReplicaChunkDescriptor* descriptor)
+    {
+        m_globalChunkInitContextStack->push_back();
+        m_globalChunkInitContextStack->back().m_descriptor = descriptor;
+
+        // If the descriptor's tables have already been populated, don't re-populate them.
+        if (descriptor->GetDataSetCount() > 0 || descriptor->GetRpcCount() > 0)
+        {
+            descriptor->m_isInitialized = true;
+        }
+    }
+    //-----------------------------------------------------------------------------
+    void ReplicaChunkDescriptorTable::EndConstructReplicaChunk()
+    {
+        m_globalChunkInitContextStack->back().m_descriptor->m_isInitialized = true;
+        m_globalChunkInitContextStack->pop_back();
+    }
+    //-----------------------------------------------------------------------------
+    ReplicaChunkInitContext* ReplicaChunkDescriptorTable::GetCurrentReplicaChunkInitContext()
+    {
+        return m_globalChunkInitContextStack->size() > 0 ? &m_globalChunkInitContextStack->back() : nullptr;
+    }
+    //-----------------------------------------------------------------------------
+} // GridMate

Code/Framework/GridMate/GridMate/Replica/ReplicaChunkDescriptor.h

@@ -0,0 +1,235 @@
+/*
+* All or portions of this file Copyright (c) Amazon.com, Inc. or its affiliates or
+* its licensors.
+*
+* For complete copyright and license terms please see the LICENSE at the root of this
+* distribution (the "License"). All use of this software is governed by the License,
+* or, if provided, by the license below or the license accompanying this file. Do not
+* remove or modify any license notices. This file is distributed on an "AS IS" BASIS,
+* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+*
+*/
+#ifndef GM_REPLICA_CHUNK_DESCRIPTOR_H
+#define GM_REPLICA_CHUNK_DESCRIPTOR_H
+
+#include <GridMate/Replica/ReplicaCommon.h>
+#include <GridMate/Replica/ReplicaChunk.h>
+
+#include <AzCore/Memory/OSAllocator.h>
+#include <AzCore/std/containers/intrusive_list.h>
+
+namespace GridMate
+{
+    class ReplicaChunkBase;
+    class RpcBase;
+    class DataSetBase;
+
+    /**
+    * ReplicaChunkDescriptors provide the replica manager with
+    * structural information about replica chunk types so they can
+    * be created.
+    *
+    * Descriptors are created during replica chunk registration,
+    * but their tables are not populated until first time an instance
+    * of such chunk type is created.
+    *
+    * Replica chunk types are registered by calling
+    * ReplicaChunkDescriptorTable::RegisterChunkType().
+    */
+    class ReplicaChunkDescriptor
+    {
+        friend class ReplicaChunkDescriptorTable;
+
+    public:
+        ReplicaChunkDescriptor(const char* pNameStr, size_t classSize);
+        virtual ~ReplicaChunkDescriptor() { }
+
+        //! Called by the system when creating replica chunks from network data.
+        virtual ReplicaChunkBase* CreateFromStream(UnmarshalContext& mc) = 0;
+        //! Called by the system to skip ctor data from the stream.
+        virtual void DiscardCtorStream(UnmarshalContext& mc) = 0;
+        //! Hook to implement chunk object deletion.
+        virtual void DeleteReplicaChunk(ReplicaChunkBase* chunkInstance) = 0;
+        //! Called by the system when chunk ctor data needs to be sent.
+        virtual void MarshalCtorData(ReplicaChunkBase* chunkInstance, WriteBuffer& wb) = 0;
+
+        virtual ZoneMask GetZoneMask() const { return ZoneMask_All; }
+
+        bool IsInitialized() const { return m_isInitialized; }
+        const char* GetChunkName() const { return m_chunkClassName; }
+        ReplicaChunkClassId GetChunkTypeId() const { return m_chunkTypeId; }
+        size_t GetChunkSize() const { return m_chunkClassSize; }
+
+        void RegisterDataSet(const char* debugName, DataSetBase* ds);
+        void RegisterDataSet(const char* debugName, ptrdiff_t offset);
+        size_t GetDataSetCount() const { return m_vdt.size(); }
+        DataSetBase* GetDataSet(const ReplicaChunkBase* base, size_t index) const;
+        size_t GetDataSetIndex(const ReplicaChunkBase* base, const DataSetBase* dataset) const;
+        size_t GetDataSetIndex(ptrdiff_t offset) const;
+        const char* GetDataSetName(const ReplicaChunkBase* base, const DataSetBase* dataset) const;
+
+        void RegisterRPC(const char* debugName, RpcBase* rpc);
+        void RegisterRPC(const char* debugName, ptrdiff_t offset);
+        size_t GetRpcCount() const { return m_vrt.size(); }
+        RpcBase* GetRpc(const ReplicaChunkBase* base, size_t index) const;
+        size_t GetRpcIndex(const ReplicaChunkBase* base, const RpcBase* rpc) const;
+        size_t GetRpcIndex(ptrdiff_t offset) const;
+        const char* GetRpcName(const ReplicaChunkBase* base, const RpcBase* rpc) const;
+
+    protected:
+        struct ReplicaChunkMemberDescriptor
+        {
+            ptrdiff_t m_offset;
+            const char* m_debugName;
+        };
+
+        // Virtual DataSet Table
+        typedef AZStd::fixed_vector<ReplicaChunkMemberDescriptor, GM_MAX_DATASETS_IN_CHUNK> VDT;
+
+        // Virtual RPC Table
+        typedef AZStd::fixed_vector<ReplicaChunkMemberDescriptor, GM_MAX_RPCS_DECL_PER_CHUNK> VRT;
+
+        ReplicaChunkClassId m_chunkTypeId;
+        const char* m_chunkClassName;
+        size_t m_chunkClassSize;
+        VDT m_vdt;
+        VRT m_vrt;
+        bool m_isInitialized;
+    };
+
+    /**
+    * DefaultReplicaChunkDescriptor provides a common implementation for
+    * chunk descriptors.
+    * It can be used for chunk types that do not use ctor data and
+    * have no special construction/destruction requirements.
+    */
+    template<typename ReplicaChunkType, ZoneMask mask = ZoneMask_All>
+    class DefaultReplicaChunkDescriptor
+        : public ReplicaChunkDescriptor
+    {
+    public:
+        DefaultReplicaChunkDescriptor()
+            : ReplicaChunkDescriptor(ReplicaChunkType::GetChunkName(), sizeof(ReplicaChunkType))
+        { }
+
+        ReplicaChunkBase* CreateFromStream(UnmarshalContext&) override
+        {
+            // Pre/Post construct allow DataSets and RPCs to bind to the chunk.
+            ReplicaChunkBase* replicaChunk = aznew ReplicaChunkType;
+            return replicaChunk;
+        }
+
+        void DiscardCtorStream(UnmarshalContext&) override { }
+        void DeleteReplicaChunk(ReplicaChunkBase* chunkInstance) override { delete chunkInstance; }
+        void MarshalCtorData(ReplicaChunkBase*, WriteBuffer&) override { }
+        ZoneMask GetZoneMask() const override { return mask; }
+    };
+
+    /**
+    * ReplicaChunkInitStack is used during chunk creation to
+    * provide creation context and to build the VDT/VRT on first use.
+    */
+    struct ReplicaChunkInitContext
+    {
+        ReplicaChunkDescriptor* m_descriptor;
+        ReplicaChunkBase* m_chunk; // current replica chunk instance being constructed
+    };
+    typedef AZStd::fixed_vector<ReplicaChunkInitContext, 8> ReplicaChunkInitContextStack;
+
+    /**
+    * Stores descriptors for registered replica chunk types.
+    * This table is stored as an AZ::EnvironmentVariable for cross-dll compatibility,
+    * so it is subject to all the rules of AZ::Environment.
+    */
+    class ReplicaChunkDescriptorTable
+    {
+    public:
+        ~ReplicaChunkDescriptorTable();
+
+        static ReplicaChunkDescriptorTable& Get();
+
+        //! Register a replica chunk type. Replica chunk types must be registered before they can be instantiated.
+        //! Returns true if successfully registered, false otherwise.
+        template<typename ReplicaChunkType>
+        bool RegisterChunkType()
+        {
+            return RegisterChunkType<ReplicaChunkType, DefaultReplicaChunkDescriptor<ReplicaChunkType> >();
+        }
+
+        //! Register a replica chunk type. Replica chunk types must be registered before they can be instantiated.
+        //! Returns true if successfully registered, false otherwise.
+        template<typename ReplicaChunkType, typename ReplicaChunkDescriptorType>
+        bool RegisterChunkType()
+        {
+            ReplicaChunkClassId chunkTypeId(ReplicaChunkType::GetChunkName());
+            ReplicaChunkDescriptor* descriptor = FindReplicaChunkDescriptor(chunkTypeId);
+            if (descriptor)
+            {
+                // TODO: verify descriptor compatibility
+                AZ_TracePrintf("GridMate", "Replica type %s(0x%x) already registered. New registration ignored.\n", ReplicaChunkType::GetChunkName(), static_cast<unsigned int>(chunkTypeId));
+            }
+            else
+            {
+                // Descriptor memory is owned by the table.
+                // All entries will be freed automatically when the table is destroyed.
+                descriptor = azcreate(ReplicaChunkDescriptorType, (), AZ::OSAllocator);
+                AddReplicaChunkDescriptor(chunkTypeId, descriptor);
+            }
+
+            return true;
+        }
+
+        //! Returns the descriptor for a particular ReplicaChunk type
+        ReplicaChunkDescriptor* FindReplicaChunkDescriptor(ReplicaChunkClassId chunkTypeId);
+
+        //! Unregister a chunk descriptor. Returns false is descriptor was not found.
+        bool UnregisterReplicaChunkDescriptor(ReplicaChunkClassId chunkTypeId);
+
+        //! Called right before instantiating a replica chunk.
+        void BeginConstructReplicaChunk(ReplicaChunkDescriptor* descriptor);
+
+        //! Called right after instantiation of a replica chunk.
+        void EndConstructReplicaChunk();
+
+        //! Returns the current replica chunk init context.
+        ReplicaChunkInitContext* GetCurrentReplicaChunkInitContext();
+
+    protected:
+
+        /**
+        * The replica chunk descriptor table registers descriptors by adding them to two intrusive lists of DescriptorInfos,
+        * one owned by the module doing the registration used to track descriptor ownership, and a global one used for queries.
+        * The module list tracks the descriptors created by the module so they can be automatically unregistered when
+        * the module is unloaded. The global one is used to share the descriptors across modules.
+        * A static copy of ReplicaChunkDescriptorTable is created for each module and holds a reference to the global table to
+        * guarantee its existence, but individual entries are guaranteed to be unregistered before their contents become invalid.
+        * Descriptors and DescriptorInfos are allocated using AZ_OS_MALLOC to avoid dependencies on any allocators. This makes
+        * things easier for users by removing the requirement of always having to explicitly unregister descriptors.
+        */
+        struct DescriptorInfo
+            : public AZStd::intrusive_list_node<DescriptorInfo>
+        {
+            AZ_CLASS_ALLOCATOR(DescriptorInfo, AZ::OSAllocator, 0);
+            ReplicaChunkClassId m_chunkTypeId;
+            ReplicaChunkDescriptor* m_descriptor; // pointer to the replica descriptor
+        };
+        typedef AZStd::intrusive_list<DescriptorInfo, AZStd::list_base_hook<DescriptorInfo> > DescriptorContainerType;
+
+        //! Adds the descriptor to the tables. Does not check for duplicates!
+        void AddReplicaChunkDescriptor(ReplicaChunkClassId chunkTypeId, ReplicaChunkDescriptor* descriptor);
+
+        //! Unregisters the descriptor from the global table. Returns false if the descriptor is not found.
+        bool UnregisterReplicaChunkDescriptorFromGlobalTable(ReplicaChunkClassId chunkTypeId);
+
+        DescriptorContainerType m_moduleDescriptorTable;                                        // Tracks descriptors created by the module
+        AZ::EnvironmentVariable<DescriptorContainerType> m_globalDescriptorTable;               // Holds the global list of descriptors
+
+        AZ::EnvironmentVariable<ReplicaChunkInitContextStack> m_globalChunkInitContextStack;    // Tracks the current chunk type that is being constructed
+
+        static ReplicaChunkDescriptorTable s_theTable;  // Per-module static copy of the table
+    };
+} // namespace GridMate
+
+#endif // GM_REPLICA_CHUNK_DESCRIPTOR_H
+
+#pragma once

Code/Framework/GridMate/GridMate/Replica/ReplicaChunkInterface.h

@@ -0,0 +1,24 @@
+/*
+* All or portions of this file Copyright (c) Amazon.com, Inc. or its affiliates or
+* its licensors.
+*
+* For complete copyright and license terms please see the LICENSE at the root of this
+* distribution (the "License"). All use of this software is governed by the License,
+* or, if provided, by the license below or the license accompanying this file. Do not
+* remove or modify any license notices. This file is distributed on an "AS IS" BASIS,
+* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+*
+*/
+#ifndef GM_REPLICA_CHUNK_INTERFACE_H
+#define GM_REPLICA_CHUNK_INTERFACE_H
+
+namespace GridMate
+{
+    //-----------------------------------------------------------------------------
+    // Base class for handling chunk events
+    //-----------------------------------------------------------------------------
+    struct ReplicaChunkInterface
+    { };
+} // namespace GridMate
+
+#endif // GM_REPLICA_CHUNK_INTERFACE_H

Code/Framework/GridMate/GridMate/Replica/ReplicaCommon.h

@@ -0,0 +1,161 @@
+/*
+* All or portions of this file Copyright (c) Amazon.com, Inc. or its affiliates or
+* its licensors.
+*
+* For complete copyright and license terms please see the LICENSE at the root of this
+* distribution (the "License"). All use of this software is governed by the License,
+* or, if provided, by the license below or the license accompanying this file. Do not
+* remove or modify any license notices. This file is distributed on an "AS IS" BASIS,
+* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+*
+*/
+#ifndef GM_REPLICACOMMON_H
+#define GM_REPLICACOMMON_H
+
+#include <GridMate/Types.h>
+#include <GridMate/Replica/ReplicaDefs.h>
+#include <GridMate/Serialize/Buffer.h>
+
+#include <AzCore/std/smart_ptr/intrusive_ptr.h>
+#include <AzCore/std/smart_ptr/weak_ptr.h>
+#include <AzCore/std/smart_ptr/shared_ptr.h>
+#include <AzCore/std/smart_ptr/make_shared.h>
+#include <AzCore/Math/Crc.h>
+
+#include <AzCore/std/containers/unordered_map.h>
+
+#define GM_MAX_CHUNKS_PER_REPLICA   (64)
+#define GM_MAX_DATASETS_IN_CHUNK    (32)
+#define GM_MAX_RPCS_DECL_PER_CHUNK  (32)
+#define GM_MAX_RPC_SEND_PER_REPLICA (65535)
+#define GM_MAX_REPLICA_CLASS_TYPES  (256)
+#define GM_REPIDS_PER_BLOCK         (1<<25)        //~33M replicaIds/host with up to 128 hosts
+
+#define GM_REPLICA_MSG_CUTOFF 1100
+
+#if !defined(GM_REPLICA_HAS_DEBUG_NAME)
+    #if defined(AZ_RELEASE_BUILD)
+        #define GM_REPLICA_HAS_DEBUG_NAME 0
+    #else
+        #define GM_REPLICA_HAS_DEBUG_NAME 1
+    #endif
+#endif
+
+namespace GridMate
+{
+    class Replica;
+    class ReplicaChunkBase;
+    class ReplicaManager;
+    class ReplicaPeer;
+
+    class DataSetBase;
+    class RpcBase;
+    struct RpcContext;
+
+    typedef AZStd::intrusive_ptr<Replica> ReplicaPtr;
+    typedef AZStd::intrusive_ptr<ReplicaChunkBase> ReplicaChunkPtr;
+
+    /*
+    * constants
+    */
+    static const ReplicaId InvalidReplicaId = 0;
+    static const PeerId InvalidReplicaPeerId = 0;
+
+    class TargetCallbackBase
+    {
+    public:
+        virtual void operator()() = 0;
+    };
+
+    //-----------------------------------------------------------------------------
+    //-----------------------------------------------------------------------------
+    struct TimeContext
+    {
+        unsigned int m_realTime;
+        unsigned int m_localTime;
+    };
+    //-----------------------------------------------------------------------------
+
+    //-----------------------------------------------------------------------------
+    //-----------------------------------------------------------------------------
+    struct ReplicaContext
+        : public TimeContext
+    {
+        ReplicaManager* m_rm;
+        ReplicaPeer* m_peer; // peer the replica (or replica update) belongs to or came from
+
+        explicit ReplicaContext(ReplicaManager* rm, const TimeContext& tc, ReplicaPeer* peer = 0)
+            : TimeContext(tc)
+            , m_rm(rm)
+            , m_peer(peer) {}
+    };
+    //-----------------------------------------------------------------------------
+
+    //-----------------------------------------------------------------------------
+    //-----------------------------------------------------------------------------
+    struct PrepareDataResult
+    {
+        PrepareDataResult(bool isDownstreamReliableDirty, bool isDownstreamUnreliableDirty, bool isUpstreamReliableDirty, bool isUpstreamUnreliableDirty)
+            : m_isDownstreamReliableDirty(isDownstreamReliableDirty)
+            , m_isDownstreamUnreliableDirty(isDownstreamUnreliableDirty)
+            , m_isUpstreamReliableDirty(isUpstreamReliableDirty)
+            , m_isUpstreamUnreliableDirty(isUpstreamUnreliableDirty)
+        { }
+
+        bool m_isDownstreamReliableDirty;
+        bool m_isDownstreamUnreliableDirty;
+        bool m_isUpstreamReliableDirty;
+        bool m_isUpstreamUnreliableDirty;
+    };
+    //-----------------------------------------------------------------------------
+    using CallbackBuffer = AZStd::vector< AZStd::weak_ptr<TargetCallbackBase> >;
+    class ReplicaTarget;
+    //-----------------------------------------------------------------------------
+    struct MarshalContext
+        : public ReplicaContext
+    {
+        AZ::u32 m_marshalFlags;
+        WriteBuffer* m_outBuffer;
+        AZ::u64 m_peerLatestVersionAckd;
+        CallbackBuffer*     m_callbackBuffer;
+        ReplicaTarget*      m_target;
+        explicit MarshalContext(AZ::u32 marshalFlags, WriteBuffer* writeBuffer, CallbackBuffer* callbackBuffer, const ReplicaContext& rc, AZ::u64 lastVersionAckd = 0, ReplicaTarget* target = nullptr)
+            : ReplicaContext(rc)
+            , m_marshalFlags(marshalFlags)
+            , m_outBuffer(writeBuffer)
+            , m_peerLatestVersionAckd(lastVersionAckd)
+            , m_callbackBuffer(callbackBuffer)
+            , m_target(target)
+        { }
+    };
+
+    //-----------------------------------------------------------------------------
+    struct UnmarshalContext
+        : public ReplicaContext
+    {
+        ReadBuffer* m_iBuf;
+        AZ::u32 m_timestamp;
+        bool m_hasCtorData;
+
+        explicit UnmarshalContext(const ReplicaContext& rc)
+            : ReplicaContext(rc)
+            , m_iBuf(nullptr)
+            , m_timestamp(0)
+            , m_hasCtorData(false) { }
+    };
+    //-----------------------------------------------------------------------------
+    typedef AZ::u16 ReplicaPriority;
+
+    // Predifined replica priorities
+
+    // real time replicas have the highest priority and will not be cut off by any bandwidth limiter
+    static const ReplicaPriority k_replicaPriorityRealTime   = 0xFFFF;
+
+    static const ReplicaPriority k_replicaPriorityHighest    = 0xFFFE;
+    static const ReplicaPriority k_replicaPriorityHigh       = 0xC000;
+    static const ReplicaPriority k_replicaPriorityNormal     = 0x8000;
+    static const ReplicaPriority k_replicaPriorityLow        = 0x4000;
+    static const ReplicaPriority k_replicaPriorityLowest     = 0x0000;
+} // namespace GridMate
+
+#endif // GM_REPLICACOMMON_H

Code/Framework/GridMate/GridMate/Replica/ReplicaDefs.h

@@ -0,0 +1,87 @@
+/*
+* All or portions of this file Copyright (c) Amazon.com, Inc. or its affiliates or
+* its licensors.
+*
+* For complete copyright and license terms please see the LICENSE at the root of this
+* distribution (the "License"). All use of this software is governed by the License,
+* or, if provided, by the license below or the license accompanying this file. Do not
+* remove or modify any license notices. This file is distributed on an "AS IS" BASIS,
+* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+*
+*/
+#ifndef GM_REPLICADEFS_H
+#define GM_REPLICADEFS_H
+
+/// \file ReplicaChunk.h
+
+#include <AzCore/Math/Crc.h>
+
+namespace GridMate
+{
+    //-----------------------------------------------------------------------------
+    // ReplicaPeer Flags
+    //-----------------------------------------------------------------------------
+    struct PeerFlags
+    {
+        enum
+        {
+            Peer_New = 1 << 0,
+            Peer_SyncHost = 1 << 1,
+            Peer_ReadyForRemoval = 1 << 2,
+            Peer_Accepted = 1 << 4
+        };
+    };
+    //-----------------------------------------------------------------------------
+    typedef AZ::u32 ReplicaId;
+    typedef ReplicaId RepIdSeed;
+    typedef ReplicaId CmdId;
+    typedef AZ::Crc32 ReplicaChunkClassId;
+    typedef AZ::u32 PeerId; /// Crc32
+    //-----------------------------------------------------------------------------
+    enum ReservedIds : ReplicaId
+    {
+        Invalid_Cmd_Or_Id,  // Invalid
+        Cmd_Greetings,      // First message sent by newly connected peers
+        Cmd_NewProxy,       // Notify that a new proxy should be created
+        Cmd_DestroyProxy,   // Notify that a proxy should be deleted
+        Cmd_NewOwner,       // Notify that this replica has changed owner
+        Cmd_Heartbeat,      // DEBUG: heartbeat
+        Cmd_Count,          // Total number of Ids
+
+        RepId_SessionInfo,  // SessionInfo will always use this id;
+
+        Max_Reserved_Cmd_Or_Id,
+
+        // Replica Ids start here. The CmdId for 'UpdateReplica' is implied by a
+        // CmdId higher than Max_Reserved_Cmd_Or_Id (the Replica's Id). This is to save
+        // sending an unnecessary byte per update.
+    };
+    //-----------------------------------------------------------------------------
+    struct ReplicaMarshalFlags
+    {
+        enum
+        {
+            IncludeDatasets =   1 << 0,
+            ForceDirty =        1 << 1,
+            Authoritative =     1 << 2,
+            Reliable =          1 << 3,
+            IncludeCtorData =   1 << 4,
+            OmitUnmodified =    1 << 5,
+            ForceReliable =     1 << 6,
+
+            None = 0,
+            NewProxy = IncludeDatasets | OmitUnmodified | Authoritative | Reliable | ForceReliable,
+            FullSync = IncludeDatasets | ForceDirty | Authoritative | Reliable | ForceReliable,
+        };
+    };
+    //-----------------------------------------------------------------------------
+
+    /**
+        A user customisable set of flags that are used to logically separate
+        the different node types within the network topology.
+    **/
+    typedef AZ::u32 ZoneMask;
+    static const ZoneMask ZoneMask_All = (ZoneMask) - 1;
+}   // namespace Gridmate
+
+#endif // GM_REPLICADEFS_H

Code/Framework/GridMate/GridMate/Replica/ReplicaDrillerEvents.h

@@ -0,0 +1,120 @@
+/*
+* All or portions of this file Copyright (c) Amazon.com, Inc. or its affiliates or
+* its licensors.
+*
+* For complete copyright and license terms please see the LICENSE at the root of this
+* distribution (the "License"). All use of this software is governed by the License,
+* or, if provided, by the license below or the license accompanying this file. Do not
+* remove or modify any license notices. This file is distributed on an "AS IS" BASIS,
+* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+*
+*/
+#ifndef GM_REPLICA_DRILLER_EVENTS_H
+#define GM_REPLICA_DRILLER_EVENTS_H
+
+#include <AzCore/Driller/DrillerBus.h>
+
+/*!
+* The replica system emits debugging EBus events via the ReplicaDrillerEvents interface.
+* To listen for these events, derive from ReplicaDrillerBus::Handler and implement all
+* the functions declared in the ReplicaDrillerEvents interface.
+*/
+
+namespace GridMate
+{
+    class Replica;
+    class ReplicaChunk;
+    class ReplicaChunkBase;
+    class DataSetBase;
+    typedef AZ::u32 PeerId;
+
+    namespace Internal
+    {
+        struct RpcRequest;
+    }
+
+    namespace Debug
+    {
+        /*!
+        * These are the driller events that the replica system will emit.
+        * All functions in this interface should be implemented by the user.
+        */
+        class ReplicaDrillerEvents
+            : public AZ::Debug::DrillerEBusTraits
+        {
+        public:
+            //! Called when a replica is instantiated. It doesn't mean it will be added to the system.
+            virtual void OnCreateReplica(Replica* replica) { (void)replica; }
+            //! Called when a replica is actually destroyed.
+            virtual void OnDestroyReplica(Replica* replica) { (void)replica; }
+            //! Called when a replica is added to the system.
+            virtual void OnActivateReplica(Replica* replica) { (void)replica; }
+            //! Called when a replica is removed from the system.
+            virtual void OnDeactivateReplica(Replica* replica) { (void)replica; }
+            //! Called every time the replica data is sent to a peer.
+            virtual void OnSendReplicaBegin(Replica* replica) { (void)replica; }
+            //! Called every time the replica data is sent to a peer.
+            virtual void OnSendReplicaEnd(Replica* replica, const void* data, size_t len) { (void)replica; (void)data; (void)len; }
+            //! Called when data is received for a replica. Called with nullptr replica pointer when data for unknown replica received.
+            virtual void OnReceiveReplicaBegin(Replica* replica, const void* data, size_t len) { (void)replica; (void)data; (void)len; }
+            //! Called when data is received for a replica. Called with nullptr replica pointer when data for unknown replica received.
+            virtual void OnReceiveReplicaEnd(Replica* replica) { (void)replica; }
+            //! Called when an ownership transfer request is received.
+            virtual void OnRequestReplicaChangeOwnership(Replica* replica, PeerId requestor) { (void)replica; (void)requestor; }
+            //! Called when a replica changes ownership, not necessarily to or from the local node.
+            virtual void OnReplicaChangeOwnership(Replica* replica, bool wasMaster) { (void)replica; (void)wasMaster; }
+
+            //! Called when a chunk has been created. It doesn't mean it will be added to the system.
+            //! Object will be partially constructed at this point if you inherit from ReplicaChunk
+            virtual void OnCreateReplicaChunk(ReplicaChunkBase* chunk) { (void)chunk; }
+            //! Called when a chuck is actually destroyed.
+            virtual void OnDestroyReplicaChunk(ReplicaChunkBase* chunk) { (void)chunk; }
+            //! Called when a chunk is added to the system.
+            virtual void OnActivateReplicaChunk(ReplicaChunkBase* chunk) { (void)chunk; }
+            //! Called when a chunk is removed from the system.
+            virtual void OnDeactivateReplicaChunk(ReplicaChunkBase* chunk) { (void)chunk; }
+            //! Called when a chunk is attached to a replica.
+            virtual void OnAttachReplicaChunk(ReplicaChunkBase* chunk) { (void)chunk; }
+            //! Called when a chunk is detached from a replica.
+            virtual void OnDetachReplicaChunk(ReplicaChunkBase* chunk) { (void)chunk; }
+            //! Called every time the chunk data is sent to a peer.
+            virtual void OnSendReplicaChunkBegin(ReplicaChunkBase* chunk, AZ::u32 chunkIndex, PeerId from, PeerId to) { (void)chunk; (void)chunkIndex; (void)from; (void)to; }
+            //! Called every time the chunk data is sent to a peer.
+            virtual void OnSendReplicaChunkEnd(ReplicaChunkBase* chunk, AZ::u32 chunkIndex, const void* data, size_t len) { (void)chunk; (void)chunkIndex; (void)data; (void)len; }
+            //! Called when data is received for a chunk.
+            virtual void OnReceiveReplicaChunkBegin(ReplicaChunkBase* chunk, AZ::u32 chunkIndex, PeerId from, PeerId to, const void* data, size_t len) { (void)chunk; (void)chunkIndex; (void)from; (void)to; (void)data; (void)len; }
+            //! Called when data is received for a chunk.
+            virtual void OnReceiveReplicaChunkEnd(ReplicaChunkBase* chunk, AZ::u32 chunkIndex) { (void)chunk; (void)chunkIndex; }
+
+            //! Called every time a dataset is sent to a peer.
+            virtual void OnSendDataSet(ReplicaChunkBase* chunk, AZ::u32 chunkIndex,  DataSetBase* dataSet, PeerId from, PeerId to, const void* data, size_t len) { (void)chunk; (void)chunkIndex; (void)dataSet; (void)from; (void)to; (void)data; (void)len; }
+            //! Called when data is received for a dataset.
+            virtual void OnReceiveDataSet(ReplicaChunkBase* chunk, AZ::u32 chunkIndex, DataSetBase* dataSet, PeerId from, PeerId to, const void* data, size_t len) { (void)chunk; (void)chunkIndex; (void)dataSet; (void)from; (void)to; (void)data; (void)len; }
+
+            //! Called when an rpc request is received. RpcRequest pointer will be null if rpc is called on master replica.
+            virtual void OnRequestRpc(ReplicaChunkBase* chunk, Internal::RpcRequest* rpc) { (void)chunk; (void)rpc; }
+            //! Called when an rpc is invoked. RpcRequest pointer will be null if rpc is called on master replica.
+            virtual void OnInvokeRpc(ReplicaChunkBase* chunk, Internal::RpcRequest* rpc) { (void)chunk; (void)rpc; }
+            //! Called every time an rpc is sent to a peer.
+            virtual void OnSendRpc(ReplicaChunkBase* chunk, AZ::u32 chunkIndex, Internal::RpcRequest* rpc, PeerId from, PeerId to, const void* data, size_t len) { (void)chunk; (void)chunkIndex; (void)rpc; (void)from; (void)to; (void)data; (void)len; }
+            //! Called when an rpc is received.
+            virtual void OnReceiveRpc(ReplicaChunkBase* chunk, AZ::u32 chunkIndex, Internal::RpcRequest* rpc, PeerId from, PeerId to, const void* data, size_t len) { (void)chunk; (void)chunkIndex; (void)rpc; (void)from; (void)to; (void)data; (void)len; }
+
+            //! Called when a replica packet is sent.
+            virtual void OnSend(PeerId to, const void* data, size_t len, bool isReliable) { (void)to; (void)data; (void)len; (void)isReliable; }
+            //! Called when a replica packet is received.
+            virtual void OnReceive(PeerId from, const void* data, size_t len) { (void)from; (void)data; (void)len; }
+        };
+
+        /*!
+        * Replica driller events are sent are sent via this the ReplicaDrillerBus.
+        * To receive events, derive a handler from ReplicaDrillerBus::Handler and
+        * attach it to the bus.
+        */
+        typedef AZ::EBus<ReplicaDrillerEvents> ReplicaDrillerBus;
+    } // namespace Debug
+} // namespace GridMate
+
+#endif // GM_REPLICA_DRILLER_EVENTS_H
+
+#pragma once

Code/Framework/GridMate/GridMate/Replica/ReplicaFunctions.h

@@ -0,0 +1,73 @@
+/*
+* All or portions of this file Copyright (c) Amazon.com, Inc. or its affiliates or
+* its licensors.
+*
+* For complete copyright and license terms please see the LICENSE at the root of this
+* distribution (the "License"). All use of this software is governed by the License,
+* or, if provided, by the license below or the license accompanying this file. Do not
+* remove or modify any license notices. This file is distributed on an "AS IS" BASIS,
+* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+*
+*/
+#ifndef GM_REPLICA_FUNCTIONS_H
+#define GM_REPLICA_FUNCTIONS_H
+
+#include <AzCore/std/typetraits/is_base_of.h>
+
+#include <GridMate/Replica/Replica.h>
+#include <GridMate/Replica/ReplicaChunkDescriptor.h>
+
+namespace GridMate
+{
+    /**
+        Create a ReplicaChunk that isn't attached to a Replica. To attach it to a replica,
+        call replica->AttachReplicaChunk(chunk).
+     **/
+    template<class ChunkType, class ... Args>
+    ChunkType* CreateReplicaChunk(Args&& ... args)
+    {
+        static_assert(AZStd::is_base_of<ReplicaChunkBase, ChunkType>::value, "Class must inherit from ReplicaChunk");
+
+        ReplicaChunkDescriptor* descriptor = ReplicaChunkDescriptorTable::Get().FindReplicaChunkDescriptor(ReplicaChunkClassId(ChunkType::GetChunkName()));
+        AZ_Error("GridMate", descriptor, "Cannot find replica chunk descriptor for %s. Did you remember to register the chunk type?", ChunkType::GetChunkName());
+        if (descriptor != nullptr)
+        {
+            ReplicaChunkDescriptorTable::Get().BeginConstructReplicaChunk(descriptor);
+            ChunkType* chunk = aznew ChunkType(AZStd::forward<Args>(args) ...);
+            ReplicaChunkDescriptorTable::Get().EndConstructReplicaChunk();
+            chunk->Init(descriptor);
+
+            return chunk;
+        }
+        return nullptr;
+    }
+
+    /**
+        Create a ReplicaChunk that is automatically attached to the replica.
+     **/
+    template<class ChunkType, class ... Args>
+    ChunkType* CreateAndAttachReplicaChunk(const ReplicaPtr& replica, Args&& ... args)
+    {
+        return CreateAndAttachReplicaChunk<ChunkType>(replica.get(), AZStd::forward<Args>(args) ...);
+    }
+
+    /**
+        Create a ReplicaChunk that is automatically attached to the replica.
+     **/
+    template<class ChunkType, class ... Args>
+    ChunkType* CreateAndAttachReplicaChunk(Replica* replica, Args&& ... args)
+    {
+        // Chunks cannot be attached while active
+        if (replica->IsActive())
+        {
+            AZ_Warning("GridMate", false, "Cannot attach chunk %s while replica is active", ChunkType::GetChunkName());
+            return nullptr;
+        }
+
+        ChunkType* chunk = CreateReplicaChunk<ChunkType>(AZStd::forward<Args>(args) ...);
+        replica->AttachReplicaChunk(chunk);
+        return chunk;
+    }
+}
+
+#endif // GM_REPLICA_FUNCTIONS_H

Code/Framework/GridMate/GridMate/Replica/ReplicaFunctions.inl

@@ -0,0 +1,102 @@
+/*
+* All or portions of this file Copyright (c) Amazon.com, Inc. or its affiliates or
+* its licensors.
+*
+* For complete copyright and license terms please see the LICENSE at the root of this
+* distribution (the "License"). All use of this software is governed by the License,
+* or, if provided, by the license below or the license accompanying this file. Do not
+* remove or modify any license notices. This file is distributed on an "AS IS" BASIS,
+* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+*
+*/
+#if (GM_FUNCTION_NUM_ARGS == 0)
+    #define GM_FUNCTION_TEMPLATE_PARMS
+    #define GM_FUNCTION_ARGS
+    #define GM_FUNCTION_ARGS_CONCAT
+    #define GM_FUNCTION_FORWARD
+    #define GM_FUNCTION_FORWARD_CONCAT
+#elif (GM_FUNCTION_NUM_ARGS == 1)
+    #define GM_FUNCTION_TEMPLATE_PARMS  , typename T0
+    #define GM_FUNCTION_ARGS            T0 && t0
+    #define GM_FUNCTION_ARGS_CONCAT     , GM_FUNCTION_ARGS
+    #define GM_FUNCTION_FORWARD         AZStd::forward<T0>(t0)
+    #define GM_FUNCTION_FORWARD_CONCAT  , GM_FUNCTION_FORWARD
+#elif (GM_FUNCTION_NUM_ARGS == 2)
+    #define GM_FUNCTION_TEMPLATE_PARMS  , typename T0, typename T1
+    #define GM_FUNCTION_ARGS            T0 && t0, T1 && t1
+    #define GM_FUNCTION_ARGS_CONCAT     , GM_FUNCTION_ARGS
+    #define GM_FUNCTION_FORWARD         AZStd::forward<T0>(t0), AZStd::forward<T1>(t1)
+    #define GM_FUNCTION_FORWARD_CONCAT  , GM_FUNCTION_FORWARD
+#elif (GM_FUNCTION_NUM_ARGS == 3)
+    #define GM_FUNCTION_TEMPLATE_PARMS  , typename T0, typename T1, typename T2
+    #define GM_FUNCTION_ARGS            T0 && t0, T1 && t1, T2 && t2
+    #define GM_FUNCTION_ARGS_CONCAT     , GM_FUNCTION_ARGS
+    #define GM_FUNCTION_FORWARD         AZStd::forward<T0>(t0), AZStd::forward<T1>(t1), AZStd::forward<T2>(t2)
+    #define GM_FUNCTION_FORWARD_CONCAT  , GM_FUNCTION_FORWARD
+#elif (GM_FUNCTION_NUM_ARGS == 4)
+    #define GM_FUNCTION_TEMPLATE_PARMS  , typename T0, typename T1, typename T2, typename T3
+    #define GM_FUNCTION_ARGS            T0 && t0, T1 && t1, T2 && t2, T3 && t3
+    #define GM_FUNCTION_ARGS_CONCAT     , GM_FUNCTION_ARGS
+    #define GM_FUNCTION_FORWARD         AZStd::forward<T0>(t0), AZStd::forward<T1>(t1), AZStd::forward<T2>(t2), AZStd::forward<T3>(t3)
+    #define GM_FUNCTION_FORWARD_CONCAT  , GM_FUNCTION_FORWARD
+#elif (GM_FUNCTION_NUM_ARGS == 5)
+    #define GM_FUNCTION_TEMPLATE_PARMS  , typename T0, typename T1, typename T2, typename T3, typename T4
+    #define GM_FUNCTION_ARGS            T0 && t0, T1 && t1, T2 && t2, T3 && t3, T4 && t4
+    #define GM_FUNCTION_ARGS_CONCAT     , GM_FUNCTION_ARGS
+    #define GM_FUNCTION_FORWARD         AZStd::forward<T0>(t0), AZStd::forward<T1>(t1), AZStd::forward<T2>(t2), AZStd::forward<T3>(t3), AZStd::forward<T4>(t4)
+    #define GM_FUNCTION_FORWARD_CONCAT  , GM_FUNCTION_FORWARD
+#else
+    #error Unsupported argument count
+#endif
+
+/**
+    Create a ReplicaChunk that isn't attached to a Replica. To attach it to a replica,
+    call replica->AttachReplicaChunk(chunk).
+**/
+template<class ChunkType GM_FUNCTION_TEMPLATE_PARMS>
+ChunkType* CreateReplicaChunk(GM_FUNCTION_ARGS)
+{
+    static_assert(AZStd::is_base_of<ReplicaChunkBase, ChunkType>::value, "Class must inherit from ReplicaChunk");
+
+    ReplicaChunkDescriptor* descriptor = ReplicaChunkDescriptorTable::Get().FindReplicaChunkDescriptor(ReplicaChunkClassId(ChunkType::GetChunkName()));
+    AZ_Assert(descriptor, "Cannot find replica chunk descriptor for %s. Did you remember to register the chunk type?", ChunkType::GetChunkName());
+    ReplicaChunkDescriptorTable::Get().BeginConstructReplicaChunk(descriptor);
+    ChunkType* chunk = aznew ChunkType(GM_FUNCTION_FORWARD);
+    ReplicaChunkDescriptorTable::Get().EndConstructReplicaChunk();
+    chunk->Init(descriptor);
+
+    return chunk;
+}
+
+/**
+    Create a ReplicaChunk that is automatically attached to the replica.
+**/
+template<class ChunkType GM_FUNCTION_TEMPLATE_PARMS>
+ChunkType* CreateAndAttachReplicaChunk(const ReplicaPtr& replica GM_FUNCTION_ARGS_CONCAT)
+{
+    return CreateAndAttachReplicaChunk<ChunkType>(replica.get() GM_FUNCTION_FORWARD_CONCAT);
+}
+
+/**
+    Create a ReplicaChunk that is automatically attached to the replica.
+**/
+template<class ChunkType GM_FUNCTION_TEMPLATE_PARMS>
+ChunkType* CreateAndAttachReplicaChunk(Replica* replica GM_FUNCTION_ARGS_CONCAT)
+{
+    // Chunks cannot be attached while active
+    if (replica->IsActive())
+    {
+        AZ_Warning("GridMate", false, "Cannot attach chunk %s while replica is active", ChunkType::GetChunkName());
+        return nullptr;
+    }
+
+    ChunkType* chunk = CreateReplicaChunk<ChunkType>(GM_FUNCTION_FORWARD);
+    replica->AttachReplicaChunk(chunk);
+    return chunk;
+}
+
+#undef GM_FUNCTION_TEMPLATE_PARMS
+#undef GM_FUNCTION_ARGS
+#undef GM_FUNCTION_ARGS_CONCAT
+#undef GM_FUNCTION_FORWARD
+#undef GM_FUNCTION_FORWARD_CONCAT

Code/Framework/GridMate/GridMate/Replica/ReplicaInline.inl

@@ -0,0 +1,36 @@
+/*
+* All or portions of this file Copyright (c) Amazon.com, Inc. or its affiliates or
+* its licensors.
+*
+* For complete copyright and license terms please see the LICENSE at the root of this
+* distribution (the "License"). All use of this software is governed by the License,
+* or, if provided, by the license below or the license accompanying this file. Do not
+* remove or modify any license notices. This file is distributed on an "AS IS" BASIS,
+* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+*
+*/
+#ifndef GM_REPLICA_INLINE_H
+#define GM_REPLICA_INLINE_H
+
+namespace GridMate
+{
+    /**
+         Find a ReplicaChunk by type.
+     **/
+    template<class R>
+    inline AZStd::intrusive_ptr<R> Replica::FindReplicaChunk()
+    {
+        static_assert(AZStd::is_base_of<ReplicaChunkBase, R>::value, "Class must inherit from ReplicaChunkBase");
+
+        for (auto chunk : m_chunks)
+        {
+            if (chunk && chunk->IsType<R>())
+            {
+                return AZStd::static_pointer_cast<R>(chunk);
+            }
+        }
+        return nullptr;
+    }
+}
+
+#endif // GM_REPLICA_INLINE_H

Code/Framework/GridMate/GridMate/Replica/ReplicaMgr.h

@@ -0,0 +1,647 @@
+/*
+* All or portions of this file Copyright (c) Amazon.com, Inc. or its affiliates or
+* its licensors.
+*
+* For complete copyright and license terms please see the LICENSE at the root of this
+* distribution (the "License"). All use of this software is governed by the License,
+* or, if provided, by the license below or the license accompanying this file. Do not
+* remove or modify any license notices. This file is distributed on an "AS IS" BASIS,
+* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+*
+*/
+#ifndef GM_REPLICAMGR_H
+#define GM_REPLICAMGR_H
+
+#include <GridMate/Replica/Replica.h>
+#include <GridMate/Replica/SystemReplicas.h>
+#include <GridMate/Replica/ReplicaDefs.h>
+#include <GridMate/Types.h>
+#include <GridMate/Containers/unordered_map.h>
+#include <GridMate/Containers/list.h>
+#include <GridMate/MathUtils.h>
+#include <GridMate/Replica/Tasks/ReplicaTaskManager.h>
+#include <GridMate/Replica/Tasks/ReplicaPriorityPolicy.h>
+#include <GridMate/Replica/Tasks/ReplicaProcessPolicy.h>
+#include <AzCore/std/chrono/chrono.h>
+#include <AzCore/std/containers/intrusive_set.h>
+#include <AzCore/std/containers/map.h>
+
+namespace UnitTest
+{
+    template <typename ComponentType>
+    class NetContextMarshalFixture;
+}
+
+namespace GridMate
+{
+    class Carrier;
+    namespace ReplicaInternal
+    {
+        class SessionInfo;
+        class SessionInfoDesc;
+        class MigrationSequence;
+        class PeerReplica;
+        class MigrationSequence;
+    }
+
+    struct ReplicaObject
+        : public AZStd::intrusive_multiset_node<ReplicaObject>
+    {
+        AZ_FORCE_INLINE bool operator<(const ReplicaObject& right) const { return m_replica->GetCreateTime() < right.m_replica->GetCreateTime(); }
+        ReplicaPtr m_replica;
+    };
+
+    typedef unordered_map<ReplicaId, ReplicaObject> ReplicaMap;
+    typedef AZStd::intrusive_multiset<ReplicaObject, AZStd::intrusive_multiset_base_hook<ReplicaObject> > ReplicaTimeSet; // Set sorted on time
+    typedef list<ReplicaPeer*> ReplicaPeerList;
+
+    //-----------------------------------------------------------------------------
+    // ReplicaPeer
+    //-----------------------------------------------------------------------------
+    enum RemotePeerMode : AZ::u8
+    {
+        Mode_Undefined,
+        Mode_Peer,      // All authoritative objects (owned + clients) will be replicated
+        Mode_Client,    // All objects (authoritative + non-authoritative) will be replicated
+    };
+
+    class PeerAckCallbacks final : public CarrierACKCallback
+    {
+        CallbackBuffer m_callbackTargets;
+    public:
+        GM_CLASS_ALLOCATOR(PeerAckCallbacks);
+        /**
+         *  Initializes by capturing the callback buffer
+         */
+        explicit PeerAckCallbacks(CallbackBuffer &callbacks)
+            : m_callbackTargets(AZStd::move(callbacks))
+        {
+        }
+        void Run() override
+        {
+            for( auto& cb : m_callbackTargets)
+            {
+                auto ptr = cb.lock();
+                if (ptr)
+                {
+                    (*ptr)();
+                }
+            }
+        }
+    };
+
+    class ReplicaPeer
+    {
+        friend class ReplicaInternal::SessionInfo;
+        friend class ReplicaManager;
+        friend class Replica;
+        friend class ReplicaInternal::MigrationSequence;
+        friend class ReplicaUpdateTaskBase;
+        friend class ReplicaTarget;
+        friend class SendLimitProcessPolicy;
+        friend class ReplicaMarshalTaskBase;
+        friend class ReplicaMarshalTask;
+        template <typename ComponentType>
+        friend class UnitTest::NetContextMarshalFixture;
+
+        AZ::u32 m_flags;
+        PeerId m_peerId;
+        ConnectionID m_connId;
+        RemotePeerMode m_mode;
+        ReplicaMap m_objectsMap;
+        ReplicaTimeSet m_objectsTimeSort;
+        PeerTargetList m_targets;
+        WriteBufferDynamic m_reliableOutBuffer;
+        WriteBufferDynamic m_unreliableOutBuffer;
+        CallbackBuffer m_reliableCallbacks;
+        CallbackBuffer m_unreliableCallbacks;
+        WriteBuffer::Marker<AZ::u32> m_reliableTimestamp;
+        WriteBuffer::Marker<AZ::u32> m_unreliableTimestamp;
+        WriteBuffer::Marker<AZ::Crc32> m_reliableMsgCrc;
+        WriteBuffer::Marker<AZ::Crc32> m_unreliableMsgCrc;
+        ZoneMask m_zoneMask;
+        ReplicaManager* m_rm;
+
+        // orphan resolution
+        list<PeerId> m_pendingReports;
+
+        int m_lastReceiveTicks; // Debug
+
+        // Bandwidth throttling
+        RollingSum<unsigned int, 10> m_dataSentLastSecond; // rolling send rate for last second
+        float m_avgSendRateBurst; // send rate averaged for >=1 seconds used for burst control
+        int m_sentBytes; // number of bytes of replica data current sent
+        int m_sendBytesAllowed; // number of bytes allowed to be sent current frame
+        ////
+
+        void SetNew(bool b)
+        {
+            if (b)
+            {
+                m_flags |= PeerFlags::Peer_New;
+            }
+            else
+            {
+                m_flags &= ~PeerFlags::Peer_New;
+            }
+        }
+        void MakeSyncHost(bool b) { m_flags = b ? m_flags | PeerFlags::Peer_SyncHost : m_flags & ~PeerFlags::Peer_SyncHost; }
+        void Add(Replica* pObj);
+        void Remove(Replica* pObj);
+
+    public:
+        GM_CLASS_ALLOCATOR(ReplicaPeer);
+
+        ReplicaPeer(ReplicaManager* manager, ConnectionID connId = InvalidConnectionID, RemotePeerMode mode = Mode_Undefined);
+
+        void Accept();
+        PeerId GetId() const { return m_peerId; }
+        ConnectionID GetConnectionId() const { return m_connId; }
+        RemotePeerMode GetMode() const { return m_mode; }
+        bool IsNew() const { return !!(m_flags & PeerFlags::Peer_New); }
+        bool IsSyncHost() const { return !!(m_flags & PeerFlags::Peer_SyncHost); }
+        bool IsOrphan() const { return GetConnectionId() == InvalidConnectionID; }
+
+        void SetEndianType(EndianType endianType);
+
+        bool CanAcceptData(ReplicaPeer* from) const;
+        ZoneMask GetZoneMask() const { return m_zoneMask; }
+
+        WriteBuffer& GetReliableOutBuffer() { return m_reliableOutBuffer; }
+        WriteBuffer& GetUnreliableOutBuffer() { return m_unreliableOutBuffer; }
+        void SendBuffer(Carrier* carrier, unsigned char commChannel, const AZ::u32 replicaManagerTimer);
+        void ResetBuffer();
+        CallbackBuffer& GetReliableCallbackBuffer() { return m_reliableCallbacks; }
+        CallbackBuffer& GetUnreliableCallbackBuffer() { return m_unreliableCallbacks; }
+
+        ReplicaPtr GetReplica(ReplicaId repId);
+
+    private:
+        //---------------------------------------------------------------------
+        // DEBUG and Test Interface. Do not use in production code.
+        //---------------------------------------------------------------------
+        void Debug_Add(Replica* pObj) { Add(pObj); }
+        void Debug_Remove(Replica* pObj) { Remove(pObj); }
+    };
+    //-----------------------------------------------------------------------------
+
+    //-----------------------------------------------------------------------------
+    // ReplicaMgrDesc
+    //-----------------------------------------------------------------------------
+    struct ReplicaMgrDesc
+    {
+        // Single-master roles that replica managers can have
+        enum Roles
+        {
+            Role_SyncHost = 1 << 0,
+        };
+
+        // default value for m_targetFixedTimeStepsPerSecond, used to indicate fixed time step is disabled
+        static const AZ::s16 k_fixedTimeStepDisabled = -1;
+
+        // id for the local peer
+        AZ::Crc32 m_myPeerId;
+
+        // pointer to underlying carrier
+        Carrier* m_carrier;
+
+        // carrier comm channel to use
+        unsigned char m_commChannel;
+
+        // roles for this replica manager
+        AZ::u32 m_roles;
+
+        // target milliseconds between sends
+        unsigned int m_targetSendTimeMS;
+
+        // incoming bandwidth limit per peer in bytes per second (0 - unlimited)
+        unsigned int m_targetSendLimitBytesPerSec;
+
+        // burst in bandwidth will be allowed for the given amount of time maximum. burst will only be allowed if bandwidth is not capped at the time of burst
+        float m_targetSendLimitBurst;
+
+        // -1 (default) means use real time (time from Carrier) when adding timestamp to send buffer read in Unmarshal and propagated to datasets and replicas
+        // as m_lastUpdateTime, otherwise specify a value that indicates the target server frame rate and the server will send a fixed time step in packets.
+        // This should match your intended target frame rate.
+        // This feature would really only be useful if you are running a server, since clients should be time stamping with their local time. The idea would be
+        // that the application should read a config file or cvar to know when to set this value.
+        AZ::s16 m_targetFixedTimeStepsPerSecond;
+
+        ReplicaMgrDesc(const AZ::Crc32& myPeerId = AZ::Crc32()
+            , Carrier* carrier = NULL
+            , unsigned char commChannel = 0
+            , AZ::u32 roles = 0
+            , unsigned int targetSendTimeMS = 0
+            , unsigned int targetSendLimitBytesPerSec = 0)
+            : m_myPeerId(myPeerId)
+            , m_carrier(carrier)
+            , m_commChannel(commChannel)
+            , m_roles(roles)
+            , m_targetSendTimeMS(targetSendTimeMS)
+            , m_targetSendLimitBytesPerSec(targetSendLimitBytesPerSec)
+            , m_targetSendLimitBurst(10.f)
+            , m_targetFixedTimeStepsPerSecond(k_fixedTimeStepDisabled)
+        {
+        }
+    };
+    //-----------------------------------------------------------------------------
+
+    //-----------------------------------------------------------------------------
+    // ReplicationSecurityOptions
+    //-----------------------------------------------------------------------------
+    struct ReplicationSecurityOptions
+    {
+        /**
+        * If turned on, only requests from verifiable authority are allowed.
+        * For RPCs with source peer id forwarding, only the host is allowed to specify the source peer id.
+        * Breaks object migration, including host migration.
+        */
+        bool m_enableStrictSourceValidation = false;
+    };
+
+    //-----------------------------------------------------------------------------
+    // FixedTimeStep
+    //-----------------------------------------------------------------------------
+    class FixedTimeStep
+    {
+    public:
+        static const AZ::u16 k_millisecondsPerSecond = 1000;
+
+        FixedTimeStep()
+            : m_updateCount(0)
+            , m_updateCountTargetPerSecond(0)
+            , m_currentTime(0)
+            , m_seconds(0)
+        {
+        }
+
+        void UpdateFixedTimeStep()
+        {
+            m_updateCount++;
+
+            // every second update the seconds count
+            if (m_updateCount % m_updateCountTargetPerSecond == 0)
+            {
+                m_seconds += 1;
+            }
+
+            // generate a ratio of the progress through the current second, this solves rounding issues created by trying to accumulate repeating decimal values (16.66666 for example)
+            const AZ::u64 oneSecondRatio = (k_millisecondsPerSecond * (m_updateCount % m_updateCountTargetPerSecond)) / m_updateCountTargetPerSecond;
+
+            // update the time to be the second count plus the ratio of our progress through the current second
+            m_currentTime = (m_seconds * k_millisecondsPerSecond) + oneSecondRatio;
+        }
+
+        void SetTargetUpdateRate(AZ::u32 updateCountTargetPerSecond)
+        {
+            AZ_Warning("GridMate", (updateCountTargetPerSecond == 0), "Calling SetTargetUpdateRate() while the system is updating will lead to inconsistencies in timing, this value should be set ONCE!\n");
+
+            if (updateCountTargetPerSecond > k_millisecondsPerSecond)
+            {
+                AZ_Warning("GridMate", false, "SetTargetUpdateRate() is clamping rate from requested [%u] to max value of [%u]!\n", updateCountTargetPerSecond, k_millisecondsPerSecond);
+                updateCountTargetPerSecond = k_millisecondsPerSecond;
+            }
+            m_updateCountTargetPerSecond = updateCountTargetPerSecond;
+
+            // this could allow for changing on the fly but it would need to ensure that if it were in the middle of a second, that the new rate would result in landing on the
+        }
+
+        AZ::u64 GetCurrentTime() const
+        {
+            return m_currentTime;
+        }
+
+    private:
+        AZ::u64 m_updateCount;
+        AZ::u32 m_updateCountTargetPerSecond;
+
+        AZ::u64 m_currentTime;     // the local time which we will time stamp outgoing changes with in calls to SendBuffer(), updated once a frame and guaranteed to be consistent between sends that occur on the same frame.
+        AZ::u64 m_seconds;         // an accumulation of seconds used when calculating m_fixedTimeStepCurrentTime
+    };
+
+    //-----------------------------------------------------------------------------
+    // ReplicaManager
+    //-----------------------------------------------------------------------------
+    class ReplicaManager : public CarrierEventBus::Handler
+    {
+        friend class Replica;
+        friend class ReplicaInternal::SessionInfo;
+        friend class ReplicaInternal::SessionInfoDesc;
+        friend class ReplicaInternal::MigrationSequence;
+        friend class ReplicaInternal::PeerReplica;
+        friend class ReplicaMarshalTask;
+        friend class ReplicaUpdateTaskBase;
+        friend class ReplicaDestroyPeerTask;
+        friend class SendLimitProcessPolicy;
+        friend class InterestManager;
+
+        typedef unordered_map<int, void*> UserContextMapType;
+        typedef unordered_map<ReplicaId, ReplicaPtr> ReplicaMap;
+
+        //---------------------------------------------------------------------
+        // RepIdMgrClient
+        // Responsible for dispensing replica ids on the client side
+        //---------------------------------------------------------------------
+        class RepIdMgrClient
+        {
+            typedef unordered_map<RepIdSeed, ReplicaId> RepIdContainerType;
+
+            RepIdContainerType m_idBlocks;
+            size_t m_nAvailableIds;
+
+        public:
+            RepIdMgrClient()
+                : m_nAvailableIds(0) {}
+
+            void AddBlock(RepIdSeed seed);
+            void RemoveBlock(RepIdSeed seed);
+            ReplicaId Alloc();
+            void Dealloc(ReplicaId id);
+            size_t Available() const { return m_nAvailableIds; }
+        };
+
+        enum
+        {
+            // Status flags
+            Rm_Initialized = 1 << 0,
+            Rm_Processing  = 1 << 2,
+            Rm_Terminating = 1 << 3,
+        };
+
+        AZ::u32 m_flags;
+        ReplicaMgrDesc m_cfg;
+        UserContextMapType m_userContexts;
+        AZStd::chrono::system_clock::time_point m_lastCheckTime; // last time we tried to send
+        AZStd::chrono::system_clock::time_point m_nextSendTime; // next expected send slot
+
+        FixedTimeStep m_fixedTimeStep;
+
+        ReplicaPeer m_self; // the local peer
+        AZStd::recursive_mutex  m_mutexRemotePeers; // mutex for remote peers
+        ReplicaPeerList         m_remotePeers;      // remote peers
+        unordered_map<PeerId, ReplicaInternal::PeerReplica::Ptr> m_peerReplicas;
+
+        vector<char> m_receiveBuffer;
+        ReplicaInternal::SessionInfo::Ptr m_sessionInfo;
+
+        ReplicaMap m_replicas;
+        RepIdMgrClient m_localIdBlocks; // used by every peer to track their own id assignments
+
+        typedef unordered_map<ReplicaId, ReplicaInternal::MigrationSequence*> MigrationsContainer;
+        MigrationsContainer m_activeMigrations;
+
+        typedef unordered_map<ReplicaId, unsigned int> TombstoneRecords;
+        TombstoneRecords m_tombstones;
+
+        typedef AZStd::intrusive_list<Replica, AZStd::list_member_hook<Replica, & Replica::m_dirtyHook> > DirtyReplicas;
+        DirtyReplicas m_dirtyReplicas;
+        AZ::PoolAllocator m_tasksAllocator;
+        ReplicaTaskManager<SendLimitProcessPolicy, SendPriorityPolicy> m_marshalingTasks;
+        ReplicaTaskManager<NullProcessPolicy, NullPriorityPolicy> m_updateTasks;
+        ReplicaTaskManager<NullProcessPolicy, NullPriorityPolicy> m_peerUpdateTasks;
+
+        TimeContext m_currentFrameTime;
+        AZ::u32 m_latchedCarrierTime;   // timer that is constant across a frame
+
+        ReplicationSecurityOptions m_securityOptions;
+        bool m_autoBroadcast; ///< should replicas be automatically broadcast to every session member?
+
+        // forbidding replica manager copying
+        ReplicaManager(const ReplicaManager&) = delete;
+        ReplicaManager& operator=(const ReplicaManager&) = delete;
+
+        bool AcceptPeer(ReplicaPeer* peer);
+        void DiscardOrphans(PeerId orphanId);
+        ReplicaPeer* FindPeer(PeerId peerId);
+
+        void OnPeerReplicaActivated(ReplicaInternal::PeerReplica::Ptr peerReplica);
+        void OnPeerReplicaDeactivated(ReplicaInternal::PeerReplica::Ptr peerReplica);
+
+        RepIdSeed ReserveIdBlock(PeerId requestor);
+        size_t ReleaseIdBlock(PeerId requestor);
+
+        void _Unmarshal(ReadBuffer& rb, ReplicaPeer* from);
+        void RegisterReplica(const ReplicaPtr& pReplica, bool isMaster, ReplicaContext& rc);
+        void UnregisterReplica(const ReplicaPtr& replica, const ReplicaContext& rc);
+        void RemoveReplicaFromDownstream(const ReplicaPtr& replica, const ReplicaContext& rc);
+        void MigrateReplica(ReplicaPtr replica, PeerId newOwnerId);
+        void AnnounceReplicaMigrated(ReplicaId replicaId, PeerId newOwnerId);
+        void OnReplicaMigrated(ReplicaPtr replica, bool isOwner, const ReplicaContext& rc);
+        void ChangeReplicaOwnership(ReplicaPtr replica, const ReplicaContext& rc, bool isMaster);
+        void AckUpstreamSuspended(ReplicaId replicaId, PeerId sendTo, AZ::u32 requestTime);
+        void OnAckUpstreamSuspended(ReplicaId replicaId, PeerId from, AZ::u32 requestTime);
+        void AckDownstream(ReplicaId replicaId, PeerId sendTo, AZ::u32 requestTime);
+        void OnAckDownstream(ReplicaId replicaId, PeerId from, AZ::u32 requestTime);
+        void SendGreetings(ReplicaPeer* peer);
+
+        virtual bool Destroy(Replica* requestor);
+        virtual void GetReplicaContext(const Replica* requestor, ReplicaContext& rc);
+
+        bool ShouldBroadcastReplica(Replica* replica) const;
+    protected:
+
+        /***
+        * RateConnectionPair wrapper for priority queue sorting and searching by connection rate
+        */
+        struct RateConnectionPair
+        {
+            AZ::u32            m_rate;
+            ConnectionID       m_connection;
+
+            RateConnectionPair(AZ::u32 value1, ConnectionID value2) : m_rate(value1), m_connection(value2) { }
+
+            ///Searches for the connection
+            bool operator==(const ConnectionID right) const
+            {
+                return (m_connection == right);
+            }
+
+            ///Compares the stored rates of two pairs
+            bool operator<(const RateConnectionPair& right) const
+            {
+                return (m_rate < right.m_rate);
+            }
+        };
+        static bool                                 k_enableBackPressure;
+        AZStd::priority_queue<RateConnectionPair>   m_connByCongestionState;      ///< Connections priority queue sorted by congestion window
+        /***
+        * Updates connection's rate in priority and updates send limit
+        *
+        *
+        * \param rate connections's new rate
+        * \param conn connection being updated
+        */
+        void UpdateConnectionRate(AZ::u32 rate, ConnectionID id);
+
+        //////////////////////////////////////////////////////////////////////////
+        // CarrierEventBus
+        void OnConnectionEstablished(Carrier* carrier, ConnectionID id) override
+        {
+            if (m_cfg.m_carrier != carrier)
+            {
+                return; //Not our carrier
+            }
+            AZ_Assert(carrier, "NULL carrier!");
+
+            m_connByCongestionState.emplace(RateConnectionPair(AZ::u32(1500), id)); //default to 1500Bps (ex 1 Ethernet frame/second minimum)
+        }
+        void OnDisconnect(Carrier* carrier, ConnectionID id, CarrierDisconnectReason reason) override
+        {
+            (void)reason;
+            if (m_cfg.m_carrier != carrier)
+            {
+                return; //Not our carrier
+            }
+            AZ_Assert(carrier, "NULL carrier!");
+
+            auto connIt = AZStd::find(m_connByCongestionState.get_container().begin(), m_connByCongestionState.get_container().end(), id);
+            if (connIt != m_connByCongestionState.get_container().end())
+            {
+                //Since we are using a weakly sorted heap, we need to re-generate when the top is removed
+                bool remake = (connIt == m_connByCongestionState.get_container().begin());
+
+                m_connByCongestionState.get_container().erase(connIt);
+
+                if (remake)
+                {
+                    AZStd::make_heap(m_connByCongestionState.get_container().begin(), m_connByCongestionState.get_container().end());
+                }
+            }
+        }
+        void OnRateChange(Carrier* carrier, ConnectionID id, AZ::u32 sendLimitBytesPerSec) override
+        {
+            if (!k_enableBackPressure)
+            {
+                return;
+            }
+            if (m_cfg.m_carrier != carrier)
+            {
+                return; //Not our carrier
+            }
+
+            AZ_Assert(carrier, "NULL carrier!");
+
+            UpdateConnectionRate(sendLimitBytesPerSec, id);
+        };
+        //End CarrierEventBus
+        //////////////////////////////////////////////////////////////////////////
+
+    public:
+        GM_CLASS_ALLOCATOR(ReplicaManager);
+
+        ReplicaManager();
+        virtual ~ReplicaManager() {}
+
+        /*
+         * Init/Shutdown
+         */
+        void Init(const ReplicaMgrDesc& desc);
+        void Shutdown();
+
+        /// Access to the owning gridmate instance.
+        IGridMate* GetGridMate() const;
+
+        /*
+         * Query functions
+         */
+        bool IsInitialized() const { return !!(m_flags & Rm_Initialized); }
+        bool IsReady() const { return m_sessionInfo && m_sessionInfo->GetReplica() && m_sessionInfo->GetReplica()->IsActive(); }
+        bool IsSyncHost() const { return m_self.IsSyncHost(); }
+        bool HasValidHost() const { return m_sessionInfo->GetReplica() && m_sessionInfo->GetReplica()->IsActive() && m_sessionInfo->m_pHostPeer && !m_sessionInfo->m_pHostPeer->IsOrphan(); }
+        PeerId GetLocalPeerId() const { return m_cfg.m_myPeerId; }
+        TimeContext GetTime() const;
+
+        AZ::u32 GetTimeForNetworkTimestamp() const;
+        void UpdateFixedTimeStep();
+        bool IsUsingFixedTimeStep() const { return m_cfg.m_targetFixedTimeStepsPerSecond != ReplicaMgrDesc::k_fixedTimeStepDisabled; }
+
+        void SetSendTimeInterval(unsigned int sendTimeMs); // Set time interval between sends (in milliseconds), 0 will bound sends to GridMate tick rate
+        unsigned int GetSendTimeInterval() const; // Returns time interval between sends (in milliseconds)
+
+        void SetSendLimit(unsigned int sendLimitBytesPerSec); // Sets outgoing bandwidth limit per peer per second
+        unsigned int GetSendLimit() const; // Returns outgoing bandwidth limit per peer per second
+
+        void SetSendLimitBurstRange(float rangeSec); // Sets burst range for bandwidth limiter, burst in bandwidth will be allowed for the given amount of time in seconds
+        float GetSendLimitBurstRange() const; // Returns burst range for bandwidth limiter
+
+        void SetAutoBroadcast(bool isEnabled);
+
+        void SetLocalLagAmt(unsigned int ms);
+
+        /*
+         * Custom user-contexts
+         * These will be passed to the replicas during frame ticks.
+         */
+        void RegisterUserContext(int key, void* data);
+        void UnregisterUserContext(int key);
+        void* GetUserContext(int key);
+
+        /*
+         * Operations
+         */
+        void UpdateFromReplicas(); // Updates local states from replica information
+        void UpdateReplicas(); // Updates replicas with local information
+        void Marshal(); // Send updates
+        void Unmarshal(); // Receive updates
+        void Promote(); // Promote this manager to host
+
+        /*
+         * Replica Peers
+         */
+        void AddPeer(ConnectionID connId, RemotePeerMode peerMode);
+        void RemovePeer(ConnectionID connId);
+
+        /*
+         * Replicas
+         */
+        virtual ReplicaPtr FindReplica(ReplicaId replicaId);
+        ReplicaId AddMaster(const ReplicaPtr& pMaster);
+
+        /*
+        * Tasks
+        */
+        void EnqueueUpdateTask(ReplicaPtr replica);
+        void UpdateReplicaTargets(ReplicaPtr replica);
+        void OnPeerAccepted(ReplicaPeer* peer);
+        void OnPeerReadyToRemove(ReplicaPeer* peer);
+        void OnReplicaChanged(ReplicaPtr replica);
+        void OnRPCQueued(ReplicaPtr replica);
+        void OnReplicaUnmarshaled(ReplicaPtr replica);
+        void RemoveFromDirtyList(Replica& replica);
+        void CancelTasks(ReplicaPtr replica);
+        void OnDestroyProxy(ReplicaId repId);
+        void OnPendingReportsReceived(PeerId peerId);
+        void OnMigratePeer(ReplicaPeer* peer);
+
+        void OnReplicaPriorityUpdated(Replica* replica);
+
+        void SetSecurityOptions(const ReplicationSecurityOptions& options);
+        ReplicationSecurityOptions GetSecurityOptions() const;
+    };
+    //-----------------------------------------------------------------------------
+
+    //-----------------------------------------------------------------------------
+    // ReplicaMgrCallbackBus
+    // Systems interested in receiving notification events from the replica manager
+    // should listen on this bus
+    //-----------------------------------------------------------------------------
+    class ReplicaMgrCallbacks
+        : public GridMateEBusTraits
+    {
+    public:
+        virtual ~ReplicaMgrCallbacks() {}
+
+        // sent when host migration has completed
+        virtual void OnNewHost(bool /*isHost*/, ReplicaManager* /*pMgr*/) {}
+        // Sent when a replica is unregistered from the system
+        virtual void OnDeactivateReplica(ReplicaId /*replicaId*/, ReplicaManager* /*pMgr*/) {}
+        // Sent when a new peer is discovered
+        virtual void OnNewPeer(PeerId /*peerId*/, ReplicaManager* /*pMgr*/) {}
+        // Sent when a peer is removed
+        virtual void OnPeerRemoved(PeerId /*peerId*/, ReplicaManager* /*pMgr*/) {}
+    };
+    //-----------------------------------------------------------------------------
+    typedef AZ::EBus<ReplicaMgrCallbacks> ReplicaMgrCallbackBus;
+} // namespace Gridmate
+
+#endif // GM_REPLICAMGR_H

Code/Framework/GridMate/GridMate/Replica/ReplicaStatus.cpp

@@ -0,0 +1,86 @@
+/*
+* All or portions of this file Copyright (c) Amazon.com, Inc. or its affiliates or
+* its licensors.
+*
+* For complete copyright and license terms please see the LICENSE at the root of this
+* distribution (the "License"). All use of this software is governed by the License,
+* or, if provided, by the license below or the license accompanying this file. Do not
+* remove or modify any license notices. This file is distributed on an "AS IS" BASIS,
+* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+*
+*/
+
+#include <GridMate/Replica/Replica.h>
+#include <GridMate/Replica/ReplicaStatus.h>
+#include <GridMate/Replica/ReplicaChunkDescriptor.h>
+
+namespace GridMate
+{
+    ReplicaStatus::ReplicaStatus()
+        : RequestOwnership("RequestOwnership")
+        , MigrationSuspendUpstream("MigrationSuspendUpstream")
+        , MigrationRequestDownstreamAck("MigrationRequestDownstreamAck")
+        , m_options("Options")
+        , m_ownerSeq("OwnerSeq")
+    {
+        SetPriority(0);
+    }
+
+    void ReplicaStatus::RegisterType()
+    {
+        ReplicaChunkDescriptorTable::Get().RegisterChunkType<ReplicaStatus>();
+    }
+
+    void ReplicaStatus::OnAttachedToReplica(Replica* replica)
+    {
+        SetHandler(replica);
+    }
+
+    void ReplicaStatus::OnDetachedFromReplica(Replica* replica)
+    {
+        (void)replica;
+        SetHandler(nullptr);
+    }
+
+    bool ReplicaStatus::IsReplicaMigratable()
+    {
+        // Return true to not interfere with the other chunks migration election
+        return true;
+    }
+
+    const char* ReplicaStatus::GetDebugName() const
+    {
+        return m_options.Get().m_replicaName.c_str();
+    }
+
+    void ReplicaStatus::SetDebugName(const char* debugName)
+    {
+        m_options.Modify([debugName](ReplicaOptions& opts)
+        {
+            if (debugName)
+            {
+                opts.SetDebugName(debugName);
+            }
+            else
+            {
+                opts.UnsetDebugName();
+            }
+            return true;
+        });
+    }
+
+    void ReplicaStatus::SetUpstreamSuspended(bool isSuspended)
+    {
+        m_options.Modify([isSuspended](ReplicaOptions& opts)
+        {
+            bool wasSuspended = opts.IsUpstreamSuspended();
+            opts.SetUpstreamSuspended(isSuspended);
+            return wasSuspended != isSuspended;
+        });
+    }
+
+    bool ReplicaStatus::IsUpstreamSuspended() const
+    {
+        return m_options.Get().IsUpstreamSuspended();
+    }
+} // namespace GridMate

Code/Framework/GridMate/GridMate/Replica/ReplicaStatus.h

@@ -0,0 +1,117 @@
+/*
+* All or portions of this file Copyright (c) Amazon.com, Inc. or its affiliates or
+* its licensors.
+*
+* For complete copyright and license terms please see the LICENSE at the root of this
+* distribution (the "License"). All use of this software is governed by the License,
+* or, if provided, by the license below or the license accompanying this file. Do not
+* remove or modify any license notices. This file is distributed on an "AS IS" BASIS,
+* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+*
+*/
+#ifndef GM_REPLICA_STATUS_H
+#define GM_REPLICA_STATUS_H
+
+#include <GridMate/Replica/DataSet.h>
+#include <GridMate/Replica/RemoteProcedureCall.h>
+#include <GridMate/Replica/ReplicaChunk.h>
+#include <GridMate/Replica/ReplicaStatusInterface.h>
+#include <GridMate/String/string.h>
+#include <GridMate/Serialize/ContainerMarshal.h>
+
+namespace GridMate
+{
+    //-------------------------------------------------------------------------
+    // ReplicaStatus - Replica management chunk
+    //-------------------------------------------------------------------------
+    class ReplicaStatus
+        : public ReplicaChunkBase
+    {
+    public:
+        typedef AZStd::intrusive_ptr<ReplicaStatus> Ptr;
+
+        GM_CLASS_ALLOCATOR(ReplicaStatus);
+
+        ReplicaStatus();
+
+        static const char* GetChunkName() { return "GridMateReplicaStatus"; }
+        static void RegisterType();
+
+        void OnAttachedToReplica(Replica* replica) override;
+        void OnDetachedFromReplica(Replica* replica) override;
+        bool IsReplicaMigratable() override;
+
+        const char* GetDebugName() const;
+        void SetDebugName(const char* debugName);
+
+        void SetUpstreamSuspended(bool isSuspended);
+        bool IsUpstreamSuspended() const;
+
+        //! Called on the originator node to request replica migration.
+        Rpc<RpcArg<PeerId> >::BindInterface<ReplicaStatusInterface, & ReplicaStatusInterface::RequestOwnershipFn> RequestOwnership;
+        
+        //! Called by the master to suspend upstream requests during replica migration.
+        Rpc<RpcArg<PeerId>, RpcArg<AZ::u32> >::BindInterface<ReplicaStatusInterface, & ReplicaStatusInterface::MigrationSuspendUpstreamFn, RpcAuthoritativeTraits> MigrationSuspendUpstream;
+        
+        //! Called by the master to signal downstream flush during replica migration.
+        Rpc<RpcArg<PeerId>, RpcArg<AZ::u32> >::BindInterface<ReplicaStatusInterface, & ReplicaStatusInterface::MigrationRequestDownstreamAckFn, RpcAuthoritativeTraits> MigrationRequestDownstreamAck;
+
+        struct ReplicaOptions
+        {
+            ReplicaOptions()
+                : m_flags(0)
+            {}
+
+            AZ_FORCE_INLINE bool IsUpstreamSuspended() const { return !!(m_flags & ReplicaUpstreamSuspended); }
+            AZ_FORCE_INLINE void SetUpstreamSuspended(bool isSuspended) { if (isSuspended) m_flags |= ReplicaUpstreamSuspended; else m_flags &= ~ReplicaUpstreamSuspended; }
+
+            AZ_FORCE_INLINE bool HasDebugName() const { return !!(m_flags & ReplicaHasDebugName); }
+            AZ_FORCE_INLINE void SetDebugName(const char* debugName) { m_flags |= ReplicaHasDebugName; m_replicaName = debugName; }
+            AZ_FORCE_INLINE void UnsetDebugName() { m_flags &= ~ReplicaHasDebugName; m_replicaName.clear(); }
+
+            AZ_FORCE_INLINE bool operator==(const ReplicaOptions& rhs) const
+            {
+                if (m_flags != rhs.m_flags)
+                    return false;
+
+                return !HasDebugName() || m_replicaName == rhs.m_replicaName;
+            }
+
+            enum ReplicaStatusFlags
+            {
+                ReplicaUpstreamSuspended   = 1 << 0,
+                ReplicaHasDebugName        = 1 << 1
+            };
+
+            struct Marshaler
+            {
+                void Marshal(WriteBuffer& wb, const ReplicaOptions& value)
+                {
+                    wb.Write(value.m_flags);
+                    if (value.HasDebugName())
+                    {
+                        wb.Write(value.m_replicaName);
+                    }
+                }
+
+                void Unmarshal(ReplicaOptions& value, GridMate::ReadBuffer& rb)
+                {
+                    rb.Read(value.m_flags);
+                    value.m_replicaName.clear();
+                    if (value.HasDebugName())
+                    {
+                        rb.Read(value.m_replicaName);
+                    }
+                }
+            };
+
+            AZ::u8 m_flags;
+            string m_replicaName;
+        };
+
+        DataSet<ReplicaOptions, ReplicaOptions::Marshaler> m_options; // Flags and debug info
+        DataSet<AZ::u32> m_ownerSeq; // used to determine who is the most recent owner when we learn about proxies as it is being migrated.
+    };
+} // namespace GridMate
+
+#endif // GM_REPLICA_STATUS_H

Code/Framework/GridMate/GridMate/Replica/ReplicaStatusInterface.h

@@ -0,0 +1,31 @@
+/*
+* All or portions of this file Copyright (c) Amazon.com, Inc. or its affiliates or
+* its licensors.
+*
+* For complete copyright and license terms please see the LICENSE at the root of this
+* distribution (the "License"). All use of this software is governed by the License,
+* or, if provided, by the license below or the license accompanying this file. Do not
+* remove or modify any license notices. This file is distributed on an "AS IS" BASIS,
+* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+*
+*/
+#ifndef GM_REPLICA_STATUS_INTERFACE_H
+#define GM_REPLICA_STATUS_INTERFACE_H
+
+#include <GridMate/Replica/ReplicaCommon.h>
+#include <GridMate/Replica/ReplicaChunkInterface.h>
+
+namespace GridMate
+{
+    //-------------------------------------------------------------------------
+    // Replica RPC interface
+    //-------------------------------------------------------------------------
+    struct ReplicaStatusInterface
+        : public ReplicaChunkInterface
+    {
+        virtual bool RequestOwnershipFn(PeerId requestor, const RpcContext& rpcContext) = 0;
+        virtual bool MigrationSuspendUpstreamFn(PeerId ownerId, AZ::u32 requestTime, const RpcContext& rpcContext) = 0;
+        virtual bool MigrationRequestDownstreamAckFn(PeerId ownerId, AZ::u32 requestTime, const RpcContext& rpcContext) = 0;
+    };
+}   // namespace GridMate
+#endif  // GM_REPLICA_STATUS_INTERFACE_H

Code/Framework/GridMate/GridMate/Replica/ReplicaTarget.cpp

@@ -0,0 +1,103 @@
+/*
+* All or portions of this file Copyright (c) Amazon.com, Inc. or its affiliates or
+* its licensors.
+*
+* For complete copyright and license terms please see the LICENSE at the root of this
+* distribution (the "License"). All use of this software is governed by the License,
+* or, if provided, by the license below or the license accompanying this file. Do not
+* remove or modify any license notices. This file is distributed on an "AS IS" BASIS,
+* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+*
+*/
+#include <GridMate/Replica/ReplicaTarget.h>
+
+#include <GridMate/Replica/ReplicaMgr.h>
+
+namespace
+{
+    // Intrusive list helpers
+    template<class T>
+    void InitNode(AZStd::intrusive_list_node<T>& node)
+    {
+        node.m_next = node.m_prev = nullptr;
+    }
+
+    template<class T, AZStd::intrusive_list_node<T> T::* PtrToMember>
+    void UnlinkNode(AZStd::intrusive_list_node<T>& node)
+    {
+        if (node.m_prev)
+        {
+            (node.m_prev->*PtrToMember).m_next = node.m_next;
+        }
+
+        if (node.m_next)
+        {
+            (node.m_next->*PtrToMember).m_prev = node.m_prev;
+        }
+
+        node.m_next = node.m_prev = nullptr;
+    }
+}
+
+namespace GridMate
+{
+    bool ReplicaTarget::k_enableAck = false;
+
+    ReplicaTarget::ReplicaTarget()
+        : m_peer(nullptr)
+        , m_flags(0)
+        , m_slotMask(0)
+        , m_replicaRevision(0)
+    {
+        InitNode<ReplicaTarget>(m_replicaHook);
+        InitNode<ReplicaTarget>(m_peerHook);
+    }
+
+    ReplicaTarget::~ReplicaTarget()
+    {
+        UnlinkNode<ReplicaTarget, & ReplicaTarget::m_replicaHook>(m_replicaHook);
+        UnlinkNode<ReplicaTarget, & ReplicaTarget::m_peerHook>(m_peerHook);
+    }
+
+    ReplicaTarget* ReplicaTarget::AddReplicaTarget(ReplicaPeer* peer, Replica* replica)
+    {
+        ReplicaTarget* newTarget = aznew ReplicaTarget();
+        newTarget->m_peer = peer;
+        newTarget->SetNew(peer->IsNew() || replica->IsNew());
+        replica->m_targets.push_back(*newTarget);
+        peer->m_targets.push_back(*newTarget);
+        return newTarget;
+    }
+
+    void ReplicaTarget::SetNew(bool isNew)
+    {
+        if (isNew)
+        {
+            m_flags |= TargetNew;
+        }
+        else
+        {
+            m_flags &= ~TargetNew;
+        }
+    }
+
+    bool ReplicaTarget::IsNew() const
+    {
+        return !!(m_flags & TargetNew);
+    }
+
+    bool ReplicaTarget::IsRemoved() const
+    {
+        return !!(m_flags & TargetRemoved);
+    }
+
+    ReplicaPeer* ReplicaTarget::GetPeer() const
+    {
+        return m_peer;
+    }
+
+    void ReplicaTarget::Destroy()
+    {
+        delete this;
+    }
+}

Code/Framework/GridMate/GridMate/Replica/ReplicaTarget.h

@@ -0,0 +1,165 @@
+/*
+* All or portions of this file Copyright (c) Amazon.com, Inc. or its affiliates or
+* its licensors.
+*
+* For complete copyright and license terms please see the LICENSE at the root of this
+* distribution (the "License"). All use of this software is governed by the License,
+* or, if provided, by the license below or the license accompanying this file. Do not
+* remove or modify any license notices. This file is distributed on an "AS IS" BASIS,
+* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+*
+*/
+#ifndef GM_REPLICA_REPLICATARGET_H
+#define GM_REPLICA_REPLICATARGET_H
+
+#include <GridMate/Replica/ReplicaCommon.h>
+
+#include <AzCore/std/containers/intrusive_list.h>
+
+namespace GridMate
+{
+    class ReplicaPeer;
+    class TargetCallback final : public TargetCallbackBase
+    {
+        friend ReplicaTarget;
+    public:
+        GM_CLASS_ALLOCATOR(TargetCallback);
+        TargetCallback(AZ::u64 revision, AZ::u64 *replicaStamp)
+            : m_revision(revision)
+            , m_currentRevision(replicaStamp) {}
+
+        AZ_FORCE_INLINE void operator()() override
+        {
+            AZ_Warning("GridMate", m_revision >= *m_currentRevision, "Cannot decrease Replica revision. Possible network re-ordering: %u<%u.", m_revision, m_currentRevision);
+
+            if (m_revision > *m_currentRevision)
+            {
+                *m_currentRevision = m_revision;
+            }
+        }
+    private:
+        AZ::u64     m_revision;
+        AZ::u64     *m_currentRevision;
+    };
+    /**
+    *  ReplicaTarget: keeps replica's marshaling target (peer) and related meta data,
+    *  Replica itself keeps an intrusive list of targets it needs to be forwarded to
+    *  Peers keep all their associated replica targets as well
+    *  Once target is removed from replica it is automatically removed from the corresponding peer and vice versa
+    *  Once replica is destroyed - all its targets are automatically removed from peers, same goes for peers
+    */
+    class ReplicaTarget
+    {
+        friend class InterestManager;
+
+    public:
+        static ReplicaTarget* AddReplicaTarget(ReplicaPeer* peer, Replica* replica);
+
+        void SetNew(bool isNew);
+        bool IsNew() const;
+        bool IsRemoved() const;
+
+        // Returns ReplicaPeer associated with given replica
+        ReplicaPeer* GetPeer() const;
+
+        // Destroys current target. Target will be removed both from peer and replica
+        void Destroy();
+
+        // Create Callback
+        AZStd::weak_ptr<TargetCallbackBase> CreateCallback(AZ::u64 revision)
+        {
+            AZ_Assert(IsAckEnabled(), "ACK disabled.")  //Shouldn't happen
+            AZ_Assert(m_replicaRevision <= revision, "Cannot decrease replica revision");
+
+            if(!m_callback || m_callback->m_revision != revision)
+            {
+                m_callback = AZStd::make_shared<TargetCallback>(revision, &m_replicaRevision);
+            }
+            //else, the version hasn't changed so re-use the callback
+
+            return m_callback;
+        }
+
+        // Checks replica stamp
+        AZ_FORCE_INLINE AZ::u64 GetRevision() const
+        {
+            return m_replicaRevision;
+        }
+
+        // Checks replica stamp
+        AZ_FORCE_INLINE bool HasOldRevision(AZ::u64 newRevision) const
+        {
+            return m_replicaRevision < newRevision;
+        }
+
+        AZ_FORCE_INLINE static bool IsAckEnabled()
+        {
+            return k_enableAck;
+        }
+
+        // Intrusive hooks to keep this target node both in replica and peer
+        AZStd::intrusive_list_node<ReplicaTarget> m_replicaHook;
+        AZStd::intrusive_list_node<ReplicaTarget> m_peerHook;
+
+    private:
+        GM_CLASS_ALLOCATOR(ReplicaTarget);
+
+        ReplicaTarget();
+        ~ReplicaTarget();
+        ReplicaTarget(const ReplicaTarget&) = delete;
+        ReplicaTarget& operator=(const ReplicaTarget&) = delete;
+
+        enum TargetStatus
+        {
+            TargetNone       =      0,
+            TargetNew        = 1 << 0, // it's a newly added target
+            TargetRemoved    = 1 << 1, // target was removed
+        };
+
+        ReplicaPeer*                        m_peer;             ///< Holds peer ptr for marshaling until marshaling is fully moved under peers,
+                                                                ///  it is safe to keep raw ptr as this node will be auto-destroyed when its peer goes away
+        AZ::u32                             m_flags;
+
+        AZ::u32                             m_slotMask;
+
+        static bool                         k_enableAck;
+        AZStd::shared_ptr<TargetCallback>   m_callback;
+        AZ::u64                             m_replicaRevision; ///< Last ACK'd replica stamp; 0 means NULL
+    };
+
+
+    /**
+    *  Intrusive list for replica targets, destroys targets when cleared
+    *  Cannot use AZStd::intrusive_list directly because it does not support auto-unlinking of nodes
+    */
+    template<class T, class Hook>
+    class ReplicaTargetAutoDestroyList
+        : private AZStd::intrusive_list<T, Hook>
+    {
+        typedef typename AZStd::intrusive_list<T, Hook> BaseListType;
+
+    public:
+
+        using BaseListType::begin;
+        using BaseListType::end;
+        using BaseListType::push_back;
+
+        AZ_FORCE_INLINE ~ReplicaTargetAutoDestroyList()
+        {
+            clear();
+        }
+
+        AZ_FORCE_INLINE void clear()
+        {
+            while (BaseListType::begin() != BaseListType::end())
+            {
+                BaseListType::begin()->Destroy();
+            }
+        }
+    };
+
+    typedef ReplicaTargetAutoDestroyList<ReplicaTarget, AZStd::list_member_hook<ReplicaTarget, & ReplicaTarget::m_replicaHook> > ReplicaTargetList;
+    typedef ReplicaTargetAutoDestroyList<ReplicaTarget, AZStd::list_member_hook<ReplicaTarget, & ReplicaTarget::m_peerHook> > PeerTargetList;
+}
+
+#endif // GM_REPLICA_REPLICATARGET_H

Code/Framework/GridMate/GridMate/Replica/ReplicaUtils.h

@@ -0,0 +1,93 @@
+/*
+* All or portions of this file Copyright (c) Amazon.com, Inc. or its affiliates or
+* its licensors.
+*
+* For complete copyright and license terms please see the LICENSE at the root of this
+* distribution (the "License"). All use of this software is governed by the License,
+* or, if provided, by the license below or the license accompanying this file. Do not
+* remove or modify any license notices. This file is distributed on an "AS IS" BASIS,
+* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+*
+*/
+#ifndef GM_REPLICAUTILS_H
+#define GM_REPLICAUTILS_H
+
+#include <AzCore/Math/Crc.h>
+
+#include <GridMate/Serialize/Buffer.h>
+
+#define GM_CRC_REPLICA_DATA 0
+
+namespace GridMate
+{
+#if (GM_CRC_REPLICA_DATA)
+    template<typename T>
+    void SafeGuardRead(ReadBuffer* buffer, T function)
+    {
+        AZ::u32 size;
+        AZ::u32 crc;
+
+        if (!buffer->Read(size))
+        {
+            AZ_Assert(readSize == size, "Read the wrong amount");
+            return;
+        }
+        if (!buffer->Read(crc))
+        {
+            AZ_Assert(readSize == size, "Read the wrong amount");
+            return;
+        }
+
+        const char* currentPos = buffer->GetCurrent();
+
+        AZ::Crc32 msgCrc(static_cast<const void*>(currentPos), size);
+        AZ_Assert(static_cast<AZ::u32>(msgCrc) == crc, "CRC is wrong");
+
+        function();
+
+        AZ::u32 readSize = static_cast<AZ::u32>(buffer->GetCurrent() - currentPos);
+        (void) readSize;
+        AZ_Assert(readSize == size, "Read the wrong amount");
+    }
+
+    template<typename T>
+    void SafeGuardWrite(WriteBuffer* buffer, T function)
+    {
+        auto sizeMarker = buffer->InsertMarker<AZ::u32>();
+        auto crcMarker = buffer->InsertMarker<AZ::u32>();
+
+        size_t oldSize = buffer->Size();
+
+        function();
+
+        size_t newSize = buffer->Size();
+
+        AZ::Crc32 msgCrc(static_cast<const void*>(buffer->Get() + oldSize), newSize - oldSize);
+
+        sizeMarker.SetData(static_cast<AZ::u32>(newSize - oldSize));
+        crcMarker.SetData(msgCrc);
+    }
+#else
+    template<typename T>
+    void SafeGuardRead(ReadBuffer*, T function)
+    {
+        function();
+    }
+
+    template<typename T>
+    void SafeGuardWrite(WriteBuffer*, T function)
+    {
+        function();
+    }
+#endif
+}
+
+#define GM_ENABLE_PROFILE_USER_CALLBACKS 1
+
+#if (GM_ENABLE_PROFILE_USER_CALLBACKS)
+#define GM_PROFILE_USER_CALLBACK(callback) AZ_PROFILE_TIMER("GridMate User Code", callback);
+#else
+#define GM_PROFILE_USER_CALLBACK(callback)
+#endif
+
+#endif // GM_REPLICAUTILS_H

Code/Framework/GridMate/GridMate/Replica/SystemReplicas.cpp

@@ -0,0 +1,261 @@
+/*
+* All or portions of this file Copyright (c) Amazon.com, Inc. or its affiliates or
+* its licensors.
+*
+* For complete copyright and license terms please see the LICENSE at the root of this
+* distribution (the "License"). All use of this software is governed by the License,
+* or, if provided, by the license below or the license accompanying this file. Do not
+* remove or modify any license notices. This file is distributed on an "AS IS" BASIS,
+* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+*
+*/
+
+#include <GridMate/Replica/MigrationSequence.h>
+#include <GridMate/Replica/ReplicaChunkDescriptor.h>
+#include <GridMate/Replica/ReplicaFunctions.h>
+#include <GridMate/Replica/ReplicaMgr.h>
+#include <GridMate/Replica/SystemReplicas.h>
+
+namespace GridMate
+{
+    namespace ReplicaInternal
+    {
+        /*
+        * Custom descriptor to override allocation because
+        * session info is an integral part of replica manager
+        */
+        class SessionInfoDesc
+            : public ReplicaChunkDescriptor
+        {
+        public:
+            SessionInfoDesc()
+                : ReplicaChunkDescriptor(SessionInfo::GetChunkName(), sizeof(SessionInfo))
+            {
+            }
+
+            ReplicaChunkBase* CreateFromStream(UnmarshalContext& mc) override
+            {
+                AZ_Assert(!mc.m_rm->m_sessionInfo->GetReplica() ||
+                    !mc.m_rm->m_sessionInfo->GetReplica()->IsActive(), "We should not have more than one sessionInfo replica!!!");
+                return mc.m_rm->m_sessionInfo.get();
+            }
+
+            void DiscardCtorStream(UnmarshalContext&) override {}
+            void DeleteReplicaChunk(ReplicaChunkBase* chunkInstance) override { delete chunkInstance; }
+            void MarshalCtorData(ReplicaChunkBase*, WriteBuffer&) override {}
+        };
+
+        //-----------------------------------------------------------------------------
+        // SessionInfo
+        //-----------------------------------------------------------------------------
+        void SessionInfo::RegisterType()
+        {
+            ReplicaChunkDescriptorTable::Get().RegisterChunkType<SessionInfo, SessionInfoDesc>();
+        }
+        //-----------------------------------------------------------------------------
+        SessionInfo::SessionInfo(ReplicaManager* pMgr)
+            : AnnounceNewHostRpc("AnnounceNewHostRpc")
+            , DiscardOrphansRpc("DiscardOrphansRpc")
+            , RequestPeerMigration("RequestPeerMigration")
+            , ReportPeerState("ReportPeerState")
+            , m_acceptedPeers("AcceptedPeers")
+            , m_localLagAmt("LocalLag")
+            , m_nextAvailableIdBlock("NextAvailableIdBlock")
+            , m_pMgr(pMgr)
+            , m_pHostPeer(nullptr)
+            , m_formerHost(0)
+        {
+            m_localLagAmt.Set(0);
+            m_acceptedPeers.SetMaxIdleTime(0.f);
+            SetPriority(k_replicaPriorityRealTime);
+        }
+        //-----------------------------------------------------------------------------
+        bool SessionInfo::IsReplicaMigratable()
+        {
+            return true;
+        }
+        //-----------------------------------------------------------------------------
+        void SessionInfo::OnReplicaActivate(const ReplicaContext& rc)
+        {
+            rc.m_rm->m_sessionInfo = this;
+            m_pHostPeer = rc.m_peer;
+            m_pHostPeer->MakeSyncHost(true);
+
+            // on activation of this replica, create our PeerInfo replica
+            Replica* peerReplica = Replica::CreateReplica("PeerInfo");
+            CreateAndAttachReplicaChunk<PeerReplica>(peerReplica);
+            rc.m_rm->AddMaster(peerReplica);
+        }
+        //-----------------------------------------------------------------------------
+        void SessionInfo::OnReplicaDeactivate(const ReplicaContext& rc)
+        {
+            (void)rc;
+        }
+        //-----------------------------------------------------------------------------
+        void SessionInfo::OnReplicaChangeOwnership(const ReplicaContext& rc)
+        {
+            if (m_pHostPeer)
+            {
+                m_pHostPeer->MakeSyncHost(false);
+                m_formerHost = m_pHostPeer->GetId();
+            }
+            m_pHostPeer = rc.m_peer;
+            m_pHostPeer->MakeSyncHost(true);
+            //AZ_TracePrintf("GridMate", "SystemReplica migrated from peerId=0x%x to peerId=0x%x.\n", m_formerHost, m_pHostPeer->GetId());
+        }
+        //-----------------------------------------------------------------------------
+        bool SessionInfo::AcceptChangeOwnership(PeerId requestor, const ReplicaContext& rc)
+        {
+            // nobody can request ownership transfer of the system replica
+            // ReplicaMgr does this manually during host migration.
+            (void)requestor;
+            (void)rc;
+            return false;
+        }
+        //-----------------------------------------------------------------------------
+        bool SessionInfo::AnnounceNewHost(const RpcContext& rc)
+        {
+            (void)rc;
+            EBUS_EVENT_ID(m_pMgr->GetGridMate(), ReplicaMgrCallbackBus, OnNewHost, m_pMgr->IsSyncHost(), m_pMgr);
+            return true;
+        }
+        //-----------------------------------------------------------------------------
+        bool SessionInfo::DiscardOrphans(PeerId orphanId, const RpcContext& rc)
+        {
+            (void)rc;
+            m_pMgr->DiscardOrphans(orphanId);
+            return true;
+        }
+        //-----------------------------------------------------------------------------
+        bool SessionInfo::OnPeerMigrationRequest(PeerId peerId, const RpcContext& rc)
+        {
+            (void)rc;
+            if (m_pMgr->IsSyncHost())
+            {
+                AZ_Assert(IsMaster(), "The host should always own sessionInfo!!!");
+                AZ_Assert(m_pendingPeerReports.find(peerId) == m_pendingPeerReports.end(), "We are already waiting for reports for peer 0x%8x!", peerId);
+
+                vector<PeerId> peers;
+                for (AZStd::size_t i = 0; i < m_acceptedPeers.Get().size(); ++i)
+                {
+                    // we need to wait for replies from all currently accepted peers except for ourselves.
+                    if (m_acceptedPeers.Get()[i] != m_pMgr->GetLocalPeerId())
+                    {
+                        peers.push_back(m_acceptedPeers.Get()[i]);
+                    }
+                }
+
+                if (!peers.empty())
+                {
+                    auto ret = m_pendingPeerReports.insert_key(peerId);
+                    ret.first->second = AZStd::move(peers);
+                }
+            }
+            else
+            {
+                AZ_Assert(IsProxy(), "Only the host should own sessionInfo!!!");
+                ReportPeerState(peerId, m_pMgr->GetLocalPeerId());
+            }
+            return true;
+        }
+        //-----------------------------------------------------------------------------
+        bool SessionInfo::OnReportPeerState(PeerId orphan, PeerId from, const RpcContext& rc)
+        {
+            (void)rc;
+            auto it = m_pendingPeerReports.find(orphan);
+            if (it != m_pendingPeerReports.end())
+            {
+                for (auto it2 = it->second.begin(); it2 != it->second.end(); ++it2)
+                {
+                    if (*it2 == from)
+                    {
+                        it->second.erase(it2);
+
+                        if (it->second.empty())
+                        {
+                            m_pendingPeerReports.erase(it);
+                            GetReplicaManager()->OnPendingReportsReceived(orphan);
+                        }
+                        break;
+                    }
+                }
+            }
+            return false;
+        }
+        //-----------------------------------------------------------------------------
+        bool SessionInfo::IsInAcceptList(PeerId peerId) const
+        {
+            return AZStd::find(m_acceptedPeers.Get().begin(), m_acceptedPeers.Get().end(), peerId) != m_acceptedPeers.Get().end();
+        }
+        //-----------------------------------------------------------------------------
+        bool SessionInfo::HasPendingReports(PeerId orphan) const
+        {
+            auto it = m_pendingPeerReports.find(orphan);
+            return it != m_pendingPeerReports.end() && !it->second.empty();
+        }
+        //-----------------------------------------------------------------------------
+
+        //-----------------------------------------------------------------------------
+        // PeerReplica
+        //-----------------------------------------------------------------------------
+        void PeerReplica::RegisterType()
+        {
+            ReplicaChunkDescriptorTable::Get().RegisterChunkType<PeerReplica>();
+        }
+        //-----------------------------------------------------------------------------
+        void PeerReplica::OnReplicaActivate(const ReplicaContext& rc)
+        {
+            if (IsMaster())
+            {
+                m_peerId.Set(rc.m_rm->GetLocalPeerId());
+            }
+            rc.m_rm->OnPeerReplicaActivated(this);
+        }
+        //-----------------------------------------------------------------------------
+        void PeerReplica::OnReplicaDeactivate(const ReplicaContext& rc)
+        {
+            (void)rc;
+            rc.m_rm->OnPeerReplicaDeactivated(this);
+        }
+        //-----------------------------------------------------------------------------
+        bool PeerReplica::OnAckUpstreamSuspendedFn(ReplicaId replicaId, PeerId peerId, AZ::u32 requestTime, const RpcContext& rpcContext)
+        {
+            (void)rpcContext;
+            //AZ_TracePrintf("GridMate", "Received upstream suspend ack response requested at %u for 0x%x from 0x%x.\n", requestTime, replicaId, peerId);
+            GetReplicaManager()->OnAckUpstreamSuspended(replicaId, peerId, requestTime);
+            return false;
+        }
+        //-----------------------------------------------------------------------------
+        bool PeerReplica::OnAckDownstreamFn(ReplicaId replicaId, PeerId peerId, AZ::u32 requestTime, const RpcContext& rpcContext)
+        {
+            (void)rpcContext;
+            //AZ_TracePrintf("GridMate", "Received downstream ack response requested at %u for 0x%x from 0x%x.\n", requestTime, replicaId, peerId);
+            GetReplicaManager()->OnAckDownstream(replicaId, peerId, requestTime);
+            return false;
+        }
+        //-----------------------------------------------------------------------------
+        bool PeerReplica::OnReplicaMigratedFn(ReplicaId replicaId, PeerId newOwnerId, const RpcContext& rpcContext)
+        {
+            (void)rpcContext;
+            (void)replicaId;
+            ReplicaManager* manager = GetReplicaManager();
+
+            if (IsProxy())
+            {
+                ReplicaContext rc = GetReplica()->GetMyContext();
+                ReplicaPtr replica = manager->FindReplica(replicaId);
+                if (replica)
+                {
+                    manager->MigrateReplica(replica, newOwnerId);
+                    if (newOwnerId == manager->GetLocalPeerId())
+                    {
+                        // replica is migrating to our local peer
+                        rc.m_peer = &manager->m_self;
+                        manager->OnReplicaMigrated(replica, true, rc);
+                    }
+                }
+            }
+            return true;
+        }
+    }   // namespace ReplicaInternal
+}   // namespace GridMate