diff --git a/Code/Framework/AzNetworking/AzNetworking/Serialization/StringifySerializer.cpp b/Code/Framework/AzNetworking/AzNetworking/Serialization/StringifySerializer.cpp index b35b01d45c..4f31dd0f33 100644 --- a/Code/Framework/AzNetworking/AzNetworking/Serialization/StringifySerializer.cpp +++ b/Code/Framework/AzNetworking/AzNetworking/Serialization/StringifySerializer.cpp @@ -10,22 +10,9 @@ namespace AzNetworking { - StringifySerializer::StringifySerializer(char delimeter, bool outputFieldNames, const AZStd::string& seperator) - : m_delimeter(delimeter) - , m_outputFieldNames(outputFieldNames) - , m_separator(seperator) + const StringifySerializer::ValueMap& StringifySerializer::GetValueMap() const { - ; - } - - const AZStd::string& StringifySerializer::GetString() const - { - return m_string; - } - - const StringifySerializer::StringMap& StringifySerializer::GetValueMap() const - { - return m_map; + return m_valueMap; } SerializerMode StringifySerializer::GetSerializerMode() const @@ -137,22 +124,9 @@ namespace AzNetworking template bool StringifySerializer::ProcessData(const char* name, const T& value) { - // Only add delimeters after we have processed at least one element - if (!m_string.empty()) - { - m_string += m_delimeter; - } - - if (m_outputFieldNames) - { - m_string += m_prefix; - m_string += name; - m_string += m_separator; - } - - AZ::CVarFixedString string = AZ::ConsoleTypeHelpers::ValueToString(value); - m_string += string.c_str(); - m_map[m_prefix + name] = string.c_str(); + const AZStd::string keyString = m_prefix + name; + AZ::CVarFixedString valueString = AZ::ConsoleTypeHelpers::ValueToString(value); + m_valueMap[keyString] = valueString.c_str(); return true; } } diff --git a/Code/Framework/AzNetworking/AzNetworking/Serialization/StringifySerializer.h b/Code/Framework/AzNetworking/AzNetworking/Serialization/StringifySerializer.h index d93f08822b..3ea0bfa412 100644 --- a/Code/Framework/AzNetworking/AzNetworking/Serialization/StringifySerializer.h +++ b/Code/Framework/AzNetworking/AzNetworking/Serialization/StringifySerializer.h @@ -20,17 +20,12 @@ namespace AzNetworking { public: - using StringMap = AZStd::map; + using ValueMap = AZStd::map; - StringifySerializer(char delimeter = ' ', bool outputFieldNames = true, const AZStd::string& seperator = "="); + StringifySerializer() = default; - // GetString - // After serializing objects, get the serialized values as a single string - const AZStd::string& GetString() const; - - // GetValueMap - // After serializing objects, get the serialized values as key value pairs - const StringMap& GetValueMap() const; + //! After serializing objects, get the serialized values as a map of key/value pairs. + const ValueMap& GetValueMap() const; // ISerializer interfaces SerializerMode GetSerializerMode() const override; @@ -62,15 +57,8 @@ namespace AzNetworking template bool ProcessData(const char* name, const T& value); - private: - - char m_delimeter; - bool m_outputFieldNames = true; - - StringMap m_map; - AZStd::string m_string; + ValueMap m_valueMap; AZStd::string m_prefix; - AZStd::string m_separator; AZStd::deque m_prefixSizeStack; }; } diff --git a/Gems/Multiplayer/Code/Include/Multiplayer/Components/LocalPredictionPlayerInputComponent.h b/Gems/Multiplayer/Code/Include/Multiplayer/Components/LocalPredictionPlayerInputComponent.h index fbb7131f76..f26f043b85 100644 --- a/Gems/Multiplayer/Code/Include/Multiplayer/Components/LocalPredictionPlayerInputComponent.h +++ b/Gems/Multiplayer/Code/Include/Multiplayer/Components/LocalPredictionPlayerInputComponent.h @@ -10,11 +10,10 @@ #include #include +#include namespace Multiplayer { - using CorrectionEvent = AZ::Event<>; - class LocalPredictionPlayerInputComponent : public LocalPredictionPlayerInputComponentBase { @@ -41,8 +40,7 @@ namespace Multiplayer ( AzNetworking::IConnection* invokingConnection, const Multiplayer::NetworkInputArray& inputArray, - const AZ::HashValue32& stateHash, - const AzNetworking::PacketEncodingBuffer& clientState + const AZ::HashValue32& stateHash ) override; void HandleSendMigrateClientInput @@ -58,11 +56,6 @@ namespace Multiplayer const AzNetworking::PacketEncodingBuffer& correction ) override; - //! Return true if we're currently replaying inputs after a correction. - //! If this value returns true, effects, audio, and other cosmetic triggers should be suppressed - //! @return true if we're within correction scope and replaying inputs - bool IsReplayingInput() const; - //! Return true if we're currently migrating from one host to another. //! @return boolean true if we're currently migrating from one host to another bool IsMigrating() const; @@ -70,8 +63,6 @@ namespace Multiplayer ClientInputId GetLastInputId() const; HostFrameId GetInputFrameId(const NetworkInput& input) const; - void CorrectionEventAddHandle(CorrectionEvent::Handler& handler); - private: void OnMigrateStart(ClientInputId migratedInputId); @@ -79,6 +70,9 @@ namespace Multiplayer void UpdateAutonomous(AZ::TimeMs deltaTimeMs); void UpdateBankedTime(AZ::TimeMs deltaTimeMs); + using StateHistoryItem = AZStd::unique_ptr; + AZStd::map m_predictiveStateHistory; + // Implicitly sorted player input history, back() is the input that corresponds to the latest client input Id NetworkInputHistory m_inputHistory; @@ -88,7 +82,6 @@ namespace Multiplayer AZ::ScheduledEvent m_autonomousUpdateEvent; // Drives autonomous input collection AZ::ScheduledEvent m_updateBankedTimeEvent; // Drives authority bank time updates - CorrectionEvent m_correctionEvent; EntityMigrationStartEvent::Handler m_migrateStartHandler; EntityMigrationEndEvent::Handler m_migrateEndHandler; @@ -104,7 +97,6 @@ namespace Multiplayer ClientInputId m_lastMigratedInputId = ClientInputId{ 0 }; // Used to resend inputs that were queued during a migration event HostFrameId m_serverMigrateFrameId = InvalidHostFrameId; - bool m_replayingInput = false; // True if we're replaying inputs under a correction event (use this to suppress effects or audio) bool m_allowMigrateClientInput = false; // True if this component was migrated, we will allow the client to send us migrated inputs (one time only) }; } diff --git a/Gems/Multiplayer/Code/Include/Multiplayer/Components/NetBindComponent.h b/Gems/Multiplayer/Code/Include/Multiplayer/Components/NetBindComponent.h index e0574e23a3..65bac09726 100644 --- a/Gems/Multiplayer/Code/Include/Multiplayer/Components/NetBindComponent.h +++ b/Gems/Multiplayer/Code/Include/Multiplayer/Components/NetBindComponent.h @@ -36,6 +36,7 @@ namespace Multiplayer using EntityMigrationEndEvent = AZ::Event<>; using EntityServerMigrationEvent = AZ::Event; using EntityPreRenderEvent = AZ::Event; + using EntityCorrectionEvent = AZ::Event<>; //! @class NetBindComponent //! @brief Component that provides net-binding to a networked entity. @@ -87,9 +88,19 @@ namespace Multiplayer AzNetworking::ConnectionId GetOwningConnectionId() const; void SetAllowAutonomy(bool value); MultiplayerComponentInputVector AllocateComponentInputs(); + + //! Return true if we're currently processing inputs. + //! @return true if we're within ProcessInput scope and writing to predictive state bool IsProcessingInput() const; + + //! Return true if we're currently replaying inputs after a correction. + //! If this value returns true, effects, audio, and other cosmetic triggers should be suppressed + //! @return true if we're within correction scope and replaying inputs + bool IsReprocessingInput() const; + void CreateInput(NetworkInput& networkInput, float deltaTime); void ProcessInput(NetworkInput& networkInput, float deltaTime); + void ReprocessInput(NetworkInput& networkInput, float deltaTime); bool HandleRpcMessage(AzNetworking::IConnection* invokingConnection, NetEntityRole remoteRole, NetworkEntityRpcMessage& message); bool HandlePropertyChangeMessage(AzNetworking::ISerializer& serializer, bool notifyChanges = true); @@ -108,6 +119,7 @@ namespace Multiplayer void NotifyMigrationEnd(); void NotifyServerMigration(HostId hostId, AzNetworking::ConnectionId connectionId); void NotifyPreRender(float deltaTime, float blendFactor); + void NotifyCorrection(); void AddEntityStopEventHandler(EntityStopEvent::Handler& eventHandler); void AddEntityDirtiedEventHandler(EntityDirtiedEvent::Handler& eventHandler); @@ -116,6 +128,7 @@ namespace Multiplayer void AddEntityMigrationEndEventHandler(EntityMigrationEndEvent::Handler& eventHandler); void AddEntityServerMigrationEventHandler(EntityServerMigrationEvent::Handler& eventHandler); void AddEntityPreRenderEventHandler(EntityPreRenderEvent::Handler& eventHandler); + void AddEntityCorrectionEventHandler(EntityCorrectionEvent::Handler& handler); bool SerializeEntityCorrection(AzNetworking::ISerializer& serializer); @@ -165,6 +178,7 @@ namespace Multiplayer EntityMigrationEndEvent m_entityMigrationEndEvent; EntityServerMigrationEvent m_entityServerMigrationEvent; EntityPreRenderEvent m_entityPreRenderEvent; + EntityCorrectionEvent m_entityCorrectionEvent; AZ::Event<> m_onRemove; RpcSendEvent::Handler m_handleLocalServerRpcMessageEventHandle; AZ::Event<>::Handler m_handleMarkedDirty; @@ -177,10 +191,11 @@ namespace Multiplayer AzNetworking::ConnectionId m_owningConnectionId = AzNetworking::InvalidConnectionId; - bool m_isProcessingInput = false; - bool m_isMigrationDataValid = false; - bool m_needsToBeStopped = false; - bool m_allowAutonomy = false; // Set to true for the hosts controlled entity + bool m_isProcessingInput = false; // Set to true when we are processing input + bool m_isReprocessingInput = false; // Set to true when we are reprocessing input (during a correction) + bool m_isMigrationDataValid = false; + bool m_needsToBeStopped = false; + bool m_allowAutonomy = false; // Set to true for the hosts controlled entity friend class NetworkEntityManager; friend class EntityReplicationManager; diff --git a/Gems/Multiplayer/Code/Include/Multiplayer/Components/NetworkTransformComponent.h b/Gems/Multiplayer/Code/Include/Multiplayer/Components/NetworkTransformComponent.h index 16aa8157aa..914aeaadd3 100644 --- a/Gems/Multiplayer/Code/Include/Multiplayer/Components/NetworkTransformComponent.h +++ b/Gems/Multiplayer/Code/Include/Multiplayer/Components/NetworkTransformComponent.h @@ -30,6 +30,7 @@ namespace Multiplayer private: void OnPreRender(float deltaTime, float blendFactor); + void OnCorrection(); void OnRotationChangedEvent(const AZ::Quaternion& rotation); void OnTranslationChangedEvent(const AZ::Vector3& translation); @@ -47,6 +48,7 @@ namespace Multiplayer AZ::Event::Handler m_resetCountEventHandler; EntityPreRenderEvent::Handler m_entityPreRenderEventHandler; + EntityCorrectionEvent::Handler m_entityCorrectionEventHandler; Multiplayer::HostFrameId m_targetHostFrameId = HostFrameId(0); }; diff --git a/Gems/Multiplayer/Code/Include/Multiplayer/NetworkTime/INetworkTime.h b/Gems/Multiplayer/Code/Include/Multiplayer/NetworkTime/INetworkTime.h index 43bcf5404e..ae47aa8373 100644 --- a/Gems/Multiplayer/Code/Include/Multiplayer/NetworkTime/INetworkTime.h +++ b/Gems/Multiplayer/Code/Include/Multiplayer/NetworkTime/INetworkTime.h @@ -58,6 +58,11 @@ namespace Multiplayer //! @return the HostFrameId taking into account the provided rewinding connectionId virtual HostFrameId GetHostFrameIdForRewindingConnection(AzNetworking::ConnectionId rewindConnectionId) const = 0; + //! Forcibly sets the current network time to the provided frameId and game time in milliseconds. + //! @param frameId the new HostFrameId to use + //! @param timeMs the new HostTimeMs to use + virtual void ForceSetTime(HostFrameId frameId, AZ::TimeMs timeMs) = 0; + //! Alters the current HostFrameId and binds that alteration to the provided ConnectionId. //! @param frameId the new HostFrameId to use //! @param timeMs the new HostTimeMs to use diff --git a/Gems/Multiplayer/Code/Include/Multiplayer/NetworkTime/RewindableObject.inl b/Gems/Multiplayer/Code/Include/Multiplayer/NetworkTime/RewindableObject.inl index 69283c496d..b0c9bc0c46 100644 --- a/Gems/Multiplayer/Code/Include/Multiplayer/NetworkTime/RewindableObject.inl +++ b/Gems/Multiplayer/Code/Include/Multiplayer/NetworkTime/RewindableObject.inl @@ -78,7 +78,7 @@ namespace Multiplayer const HostFrameId frameTime = GetCurrentTimeForProperty(); if (frameTime < m_headTime) { - AZ_Assert(false, "Trying to mutate a rewindable in the past"); + AZ_Assert(false, "Trying to mutate a rewindable value in the past"); } else if (m_headTime < frameTime) { diff --git a/Gems/Multiplayer/Code/Source/AutoGen/AutoComponent_Source.jinja b/Gems/Multiplayer/Code/Source/AutoGen/AutoComponent_Source.jinja index d2be2f682a..56ab828ffe 100644 --- a/Gems/Multiplayer/Code/Source/AutoGen/AutoComponent_Source.jinja +++ b/Gems/Multiplayer/Code/Source/AutoGen/AutoComponent_Source.jinja @@ -643,16 +643,15 @@ bool {{ ClassName }}::Serialize{{ AutoComponentMacros.GetNetPropertiesSetName(Re const uint32_t lastBit = static_cast({{ AutoComponentMacros.GetNetPropertiesQualifiedPropertyDirtyEnum(Component.attrib['Name'], ReplicateFrom, ReplicateTo, Property, 'End') }}); {% endif %} -{% if Property.attrib['IsRewindable']|booleanTrue %} AzNetworking::FixedSizeBitsetView deltaRecord(replicationRecord.m_{{ LowerFirst(AutoComponentMacros.GetNetPropertiesSetName(ReplicateFrom, ReplicateTo)) }}, firstBit, lastBit - firstBit + 1); - m_{{ LowerFirst(Property.attrib['Name']) }}.Serialize(serializer, deltaRecord); -{% else %} + if (deltaRecord.AnySet()) + { {% if Property.attrib['Container'] == 'Vector' %} - serializer.Serialize>(m_{{ LowerFirst(Property.attrib['Name']) }}, "{{ LowerFirst(Property.attrib['Name']) }}"); + serializer.Serialize>(m_{{ LowerFirst(Property.attrib['Name']) }}, "{{ LowerFirst(Property.attrib['Name']) }}"); {% elif Property.attrib['Container'] == 'Array' %} - serializer.Serialize>(m_{{ LowerFirst(Property.attrib['Name']) }}, "{{ LowerFirst(Property.attrib['Name']) }}"); + serializer.Serialize>(m_{{ LowerFirst(Property.attrib['Name']) }}, "{{ LowerFirst(Property.attrib['Name']) }}"); {% endif %} -{% endif %} + } } {% else %} Multiplayer::SerializeNetworkPropertyHelper diff --git a/Gems/Multiplayer/Code/Source/AutoGen/LocalPredictionPlayerInputComponent.AutoComponent.xml b/Gems/Multiplayer/Code/Source/AutoGen/LocalPredictionPlayerInputComponent.AutoComponent.xml index 0c799a7b3f..1a7496a77d 100644 --- a/Gems/Multiplayer/Code/Source/AutoGen/LocalPredictionPlayerInputComponent.AutoComponent.xml +++ b/Gems/Multiplayer/Code/Source/AutoGen/LocalPredictionPlayerInputComponent.AutoComponent.xml @@ -22,7 +22,6 @@ - diff --git a/Gems/Multiplayer/Code/Source/Components/LocalPredictionPlayerInputComponent.cpp b/Gems/Multiplayer/Code/Source/Components/LocalPredictionPlayerInputComponent.cpp index 242db95ecb..72f5aa8e1c 100644 --- a/Gems/Multiplayer/Code/Source/Components/LocalPredictionPlayerInputComponent.cpp +++ b/Gems/Multiplayer/Code/Source/Components/LocalPredictionPlayerInputComponent.cpp @@ -21,28 +21,54 @@ namespace Multiplayer AZ_CVAR(AZ::TimeMs, cl_MaxRewindHistoryMs, AZ::TimeMs{ 2000 }, nullptr, AZ::ConsoleFunctorFlags::Null, "Maximum number of milliseconds to keep for server correction rewind and replay"); #ifndef AZ_RELEASE_BUILD AZ_CVAR(float, cl_DebugHackTimeMultiplier, 1.0f, nullptr, AZ::ConsoleFunctorFlags::Null, "Scalar value used to simulate clock hacking cheats for validating bank time system and anticheat"); - AZ_CVAR(bool, cl_EnableDesyncDebugging, false, nullptr, AZ::ConsoleFunctorFlags::Null, "If enabled, debug logs will contain verbose information on detected state desyncs"); + AZ_CVAR(bool, cl_EnableDesyncDebugging, true, nullptr, AZ::ConsoleFunctorFlags::Null, "If enabled, debug logs will contain verbose information on detected state desyncs"); + AZ_CVAR(uint32_t, cl_PredictiveStateHistorySize, 120, nullptr, AZ::ConsoleFunctorFlags::Null, "Controls how many inputs of predictive state should be retained for debugging desyncs"); #endif + AZ_CVAR(bool, sv_ForceCorrections, false, nullptr, AZ::ConsoleFunctorFlags::Null, "If enabled, the server will force a correction for every input received for debugging"); AZ_CVAR(bool, sv_EnableCorrections, true, nullptr, AZ::ConsoleFunctorFlags::Null, "Enables server corrections on autonomous proxy desyncs"); AZ_CVAR(double, sv_MaxBankTimeWindowSec, 0.2, nullptr, AZ::ConsoleFunctorFlags::Null, "Maximum bank time we allow before we start rejecting autonomous proxy move inputs due to anticheat kicking in"); AZ_CVAR(double, sv_BankTimeDecay, 0.025, nullptr, AZ::ConsoleFunctorFlags::Null, "Amount to decay bank time by, in case of more permanent shifts in client latency"); AZ_CVAR(AZ::TimeMs, sv_MinCorrectionTimeMs, AZ::TimeMs{ 100 }, nullptr, AZ::ConsoleFunctorFlags::Null, "Minimum time to wait between sending out corrections in order to avoid flooding corrections on high-latency connections"); AZ_CVAR(AZ::TimeMs, sv_InputUpdateTimeMs, AZ::TimeMs{ 5 }, nullptr, AZ::ConsoleFunctorFlags::Null, "Minimum time between component updates"); - // Debug helper functions - AZStd::string GetInputString(NetworkInput& input) + void PrintCorrectionDifferences(const AzNetworking::StringifySerializer& client, const AzNetworking::StringifySerializer& server) { - AzNetworking::StringifySerializer serializer(',', false); - input.Serialize(serializer); - return serializer.GetString(); + const auto& clientMap = client.GetValueMap(); + const auto& serverMap = server.GetValueMap(); + + AzNetworking::StringifySerializer::ValueMap differences = clientMap; + for (auto iter = server.GetValueMap().begin(); iter != server.GetValueMap().end(); ++iter) + { + auto serverValueIter = clientMap.find(iter->first); + if (iter->second == differences[iter->first]) + { + differences.erase(iter->first); + } + } + + if (differences.empty()) + { + AZLOG_ERROR("The hash mismatched, but no differences were found.") + } + + for (auto iter = differences.begin(); iter != differences.end(); ++iter) + { + auto clientValueIter = clientMap.find(iter->first); + auto serverValueIter = serverMap.find(iter->first); + if (clientValueIter == clientMap.end() || serverValueIter == serverMap.end()) + { + AZLOG_ERROR(" %s (Not found in server and/or client value map!)", iter->first.c_str()); + continue; + } + + AZLOG_ERROR(" %s Server=%s Client=%s", iter->first.c_str(), serverValueIter->second.c_str(), clientValueIter->second.c_str()); + } } - AZStd::string GetCorrectionDataString(NetBindComponent* netBindComponent) + inline double ConvertTimeMsToSeconds(AZ::TimeMs value) { - AzNetworking::StringifySerializer serializer(',', false); - netBindComponent->SerializeEntityCorrection(serializer); - return serializer.GetString(); + return static_cast(static_cast(value)) / 1000.0; } void LocalPredictionPlayerInputComponent::LocalPredictionPlayerInputComponent::Reflect(AZ::ReflectContext* context) @@ -106,8 +132,7 @@ namespace Multiplayer ( AzNetworking::IConnection* invokingConnection, const Multiplayer::NetworkInputArray& inputArray, - const AZ::HashValue32& stateHash, - [[maybe_unused]] const AzNetworking::PacketEncodingBuffer& clientState + const AZ::HashValue32& stateHash ) { if (invokingConnection == nullptr) @@ -131,7 +156,7 @@ namespace Multiplayer } const AZ::TimeMs currentTimeMs = AZ::GetElapsedTimeMs(); - const double clientInputRateSec = static_cast(static_cast(cl_InputRateMs)) / 1000.0; + const double clientInputRateSec = ConvertTimeMsToSeconds(cl_InputRateMs); m_lastInputReceivedTimeMs = currentTimeMs; // Keep track of last inputs received, also allows us to update frame ids @@ -156,8 +181,8 @@ namespace Multiplayer if (m_clientBankedTime < sv_MaxBankTimeWindowSec) { // Client blends from previous frame to target so here we subtract blend factor to get to that state - const float blendFactor = AZStd::min(AZStd::max(0.f, input.GetHostBlendFactor()), 1.f); - const AZ::TimeMs blendMs = AZ::TimeMs(static_cast(static_cast(cl_InputRateMs)) * (1.f - blendFactor)); + const float blendFactor = AZStd::min(AZStd::max(0.f, input.GetHostBlendFactor()), 1.0f); + const AZ::TimeMs blendMs = AZ::TimeMs(static_cast(static_cast(cl_InputRateMs)) * (1.0f - blendFactor)); m_clientBankedTime = AZStd::min(m_clientBankedTime + clientInputRateSec, (double)sv_MaxBankTimeWindowSec); // clamp to boundary { ScopedAlterTime scopedTime(input.GetHostFrameId(), input.GetHostTimeMs() - blendMs, input.GetHostBlendFactor(), invokingConnection->GetConnectionId()); @@ -179,7 +204,7 @@ namespace Multiplayer } } - if (sv_EnableCorrections && (currentTimeMs - m_lastCorrectionSentTimeMs > sv_MinCorrectionTimeMs)) + if (sv_ForceCorrections || (sv_EnableCorrections && (currentTimeMs - m_lastCorrectionSentTimeMs > sv_MinCorrectionTimeMs))) { m_lastCorrectionSentTimeMs = currentTimeMs; @@ -213,69 +238,6 @@ namespace Multiplayer // Send correction SendClientInputCorrection(GetLastInputId(), correction); - -#ifndef AZ_RELEASE_BUILD - AZStd::string clientStateString; - AZStd::string serverStateString; - if (cl_EnableDesyncDebugging) - { - // In debug, show which states caused the correction - // Write in client state - AzNetworking::NetworkOutputSerializer clientStateSerializer(clientState.GetBuffer(), static_cast(clientState.GetSize())); - GetNetBindComponent()->SerializeEntityCorrection(clientStateSerializer); - - // Read out state values - AzNetworking::StringifySerializer clientValues; - GetNetBindComponent()->SerializeEntityCorrection(clientValues); - - // Restore server state - AzNetworking::NetworkOutputSerializer serverStateSerializer(correction.GetBuffer(), static_cast(correction.GetSize())); - GetNetBindComponent()->SerializeEntityCorrection(serverStateSerializer); - - // Read out state values - AzNetworking::StringifySerializer serverValues; - GetNetBindComponent()->SerializeEntityCorrection(serverValues); - - AZStd::map> mapComparison; - - // put the server value in the first part of the pair - for (const auto& pair : serverValues.GetValueMap()) - { - mapComparison[pair.first].first = pair.second; - } - - // put the client value in the second part of the pair - for (const auto& pair : clientValues.GetValueMap()) - { - mapComparison[pair.first].second = pair.second; - } - - bool firstIt = true; - for (const auto& mapPair : mapComparison) - { - if (mapPair.second.first != mapPair.second.second) - { - if (!firstIt) - { - clientStateString += ","; - serverStateString += ","; - } - firstIt = false; - - AZStd::string clientValue = mapPair.second.second.empty() ? "" : mapPair.second.second; - AZStd::string serverValue = mapPair.second.first.empty() ? "" : mapPair.second.first; - clientStateString += mapPair.first + "=" + clientValue; - serverStateString += mapPair.first + "=" + serverValue; - } - } - } - else - { - clientStateString = "available in debug only"; - serverStateString = "available in debug only"; - } - AZLOG_ERROR("** Autonomous proxy desync detected! ** clientState=[%s], serverState=[%s]", clientStateString.c_str(), serverStateString.c_str()); -#endif } } } @@ -302,7 +264,7 @@ namespace Multiplayer return; } - const float clientInputRateSec = static_cast(static_cast(cl_InputRateMs)) / 1000.0; + const double clientInputRateSec = ConvertTimeMsToSeconds(cl_InputRateMs); // Copy array so we can modify input ids NetworkInputMigrationVector inputArrayCopy = inputArray; @@ -317,14 +279,7 @@ namespace Multiplayer ScopedAlterTime scopedTime(input.GetHostFrameId(), input.GetHostTimeMs(), input.GetHostBlendFactor(), invokingConnection->GetConnectionId()); GetNetBindComponent()->ProcessInput(input, clientInputRateSec); - AZLOG - ( - NET_Prediction, - "Migrated InputId=%d - i=[%s] o=[%s]", - aznumeric_cast(input.GetClientInputId()), - GetInputString(input).c_str(), - GetCorrectionDataString(GetNetBindComponent()).c_str() - ); + AZLOG(NET_Prediction, "Migrated InputId=%d", aznumeric_cast(input.GetClientInputId())); // Don't bother checking for corrections here, the next regular input will trigger any corrections if necessary // Also don't bother with any cheat detection here, because the input array is limited in size and at most and can only be sent once @@ -334,7 +289,7 @@ namespace Multiplayer void LocalPredictionPlayerInputComponentController::HandleSendClientInputCorrection ( - AzNetworking::IConnection* invokingConnection, + [[maybe_unused]] AzNetworking::IConnection* invokingConnection, const Multiplayer::ClientInputId& inputId, const AzNetworking::PacketEncodingBuffer& correction ) @@ -357,15 +312,26 @@ namespace Multiplayer // Apply the correction AzNetworking::TrackChangedSerializer serializer(correction.GetBuffer(), static_cast(correction.GetSize())); GetNetBindComponent()->SerializeEntityCorrection(serializer); - m_correctionEvent.Signal(); + GetNetBindComponent()->NotifyCorrection(); - AZLOG - ( - NET_Prediction, - "Corrected InputId=%d - o=[%s]", - aznumeric_cast(m_lastCorrectionInputId), - GetCorrectionDataString(GetNetBindComponent()).c_str() - ); +#ifndef AZ_RELEASE_BUILD + if (cl_EnableDesyncDebugging) + { + AZLOG_INFO("** Autonomous Desync - Corrected clientInputId=%d ", aznumeric_cast(inputId)); + auto iter = m_predictiveStateHistory.find(inputId); + if (iter != m_predictiveStateHistory.end()) + { + // Read out state values + AzNetworking::StringifySerializer serverValues; + GetNetBindComponent()->SerializeEntityCorrection(serverValues); + PrintCorrectionDifferences(*iter->second, serverValues); + } + else + { + AZLOG_INFO("Received correction that is too old to diff, increase cl_PredictiveStateHistorySize"); + } + } +#endif const uint32_t inputHistorySize = static_cast(m_inputHistory.Size()); const uint32_t historicalDelta = aznumeric_cast(m_clientInputId - inputId); // Do not replay the move we just corrected, that was already processed by the server @@ -373,46 +339,18 @@ namespace Multiplayer // If this correction is for a move outside our input history window, just start replaying from the oldest move we have available const uint32_t startReplayIndex = (inputHistorySize > historicalDelta) ? (inputHistorySize - historicalDelta) : 0; - // Flag that we are replaying inputs - struct ScopedReplayingInput - { - ScopedReplayingInput(LocalPredictionPlayerInputComponentController* instance) - : m_instance(instance) - { - m_instance->m_replayingInput = true; - } - ~ScopedReplayingInput() - { - m_instance->m_replayingInput = false; - } - LocalPredictionPlayerInputComponentController* m_instance; - }; - ScopedReplayingInput markReplayingInput(this); - - const float clientInputRateSec = static_cast(static_cast(cl_InputRateMs)) / 1000.0; + const double clientInputRateSec = ConvertTimeMsToSeconds(cl_InputRateMs); for (uint32_t replayIndex = startReplayIndex; replayIndex < inputHistorySize; ++replayIndex) { // Reprocess the input for this frame NetworkInput& input = m_inputHistory[replayIndex]; ScopedAlterTime scopedTime(input.GetHostFrameId(), input.GetHostTimeMs(), input.GetHostBlendFactor(), invokingConnection->GetConnectionId()); - GetNetBindComponent()->ProcessInput(input, clientInputRateSec); + GetNetBindComponent()->ReprocessInput(input, clientInputRateSec); - AZLOG - ( - NET_Prediction, - "Replayed InputId=%d - i=[%s] o=[%s]", - aznumeric_cast(input.GetClientInputId()), - GetInputString(input).c_str(), - GetCorrectionDataString(GetNetBindComponent()).c_str() - ); + AZLOG(NET_Prediction, "Replayed InputId=%d", aznumeric_cast(input.GetClientInputId())); } } - bool LocalPredictionPlayerInputComponentController::IsReplayingInput() const - { - return m_replayingInput; - } - bool LocalPredictionPlayerInputComponentController::IsMigrating() const { return m_lastMigratedInputId != ClientInputId{ 0 }; @@ -432,11 +370,6 @@ namespace Multiplayer return (input.GetHostFrameId() == InvalidHostFrameId) ? m_serverMigrateFrameId : input.GetHostFrameId(); } - void LocalPredictionPlayerInputComponentController::CorrectionEventAddHandle(CorrectionEvent::Handler& handler) - { - handler.Connect(m_correctionEvent); - } - void LocalPredictionPlayerInputComponentController::OnMigrateStart(ClientInputId migratedInputId) { m_lastMigratedInputId = migratedInputId; @@ -477,9 +410,9 @@ namespace Multiplayer void LocalPredictionPlayerInputComponentController::UpdateAutonomous(AZ::TimeMs deltaTimeMs) { - const double deltaTime = static_cast(deltaTimeMs) / 1000.0; - const double inputRate = static_cast(static_cast(cl_InputRateMs)) / 1000.0; - const double maxRewindHistory = static_cast(static_cast(cl_MaxRewindHistoryMs)) / 1000.0; + const double deltaTime = ConvertTimeMsToSeconds(deltaTimeMs); + const double clientInputRateSec = ConvertTimeMsToSeconds(cl_InputRateMs); + const double maxRewindHistory = ConvertTimeMsToSeconds(cl_MaxRewindHistoryMs); #ifndef AZ_RELEASE_BUILD m_moveAccumulator += deltaTime * cl_DebugHackTimeMultiplier; @@ -487,13 +420,13 @@ namespace Multiplayer m_moveAccumulator += deltaTime; #endif - const uint32_t maxClientInputs = inputRate > 0.0 ? static_cast(maxRewindHistory / inputRate) : 0; + const uint32_t maxClientInputs = clientInputRateSec > 0.0 ? static_cast(maxRewindHistory / clientInputRateSec) : 0; IMultiplayer* multiplayer = GetMultiplayer(); INetworkTime* networkTime = GetNetworkTime(); - while (m_moveAccumulator >= inputRate) + while (m_moveAccumulator >= clientInputRateSec) { - m_moveAccumulator -= inputRate; + m_moveAccumulator -= clientInputRateSec; ++m_clientInputId; NetworkInputArray inputArray(GetEntityHandle()); @@ -505,35 +438,17 @@ namespace Multiplayer input.SetHostBlendFactor(multiplayer->GetCurrentBlendFactor()); // Allow components to form the input for this frame - GetNetBindComponent()->CreateInput(input, inputRate); + GetNetBindComponent()->CreateInput(input, clientInputRateSec); // Process the input for this frame - GetNetBindComponent()->ProcessInput(input, inputRate); + GetNetBindComponent()->ProcessInput(input, clientInputRateSec); - AZLOG - ( - NET_Prediction, - "Processed InutId=%d - i=[%s] o=[%s]", - aznumeric_cast(m_clientInputId), - GetInputString(input).c_str(), - GetCorrectionDataString(GetNetBindComponent()).c_str() - ); + AZLOG(NET_Prediction, "Processed InputId=%d", aznumeric_cast(m_clientInputId)); // Generate a hash based on the current client predicted states AzNetworking::HashSerializer hashSerializer; GetNetBindComponent()->SerializeEntityCorrection(hashSerializer); - // In debug, send the entire client output state to the server to make it easier to debug desync issues - AzNetworking::PacketEncodingBuffer processInputResult; -#ifndef AZ_RELEASE_BUILD - if (cl_EnableDesyncDebugging) - { - AzNetworking::NetworkInputSerializer processInputResultSerializer(processInputResult.GetBuffer(), static_cast(processInputResult.GetCapacity())); - GetNetBindComponent()->SerializeEntityCorrection(processInputResultSerializer); - processInputResult.Resize(processInputResultSerializer.GetSize()); - } -#endif - // Save this input and discard move history outside our client rewind window m_inputHistory.PushBack(input); while (m_inputHistory.Size() > maxClientInputs) @@ -552,10 +467,23 @@ namespace Multiplayer inputArray[i] = m_inputHistory[historyIndex]; } +#ifndef AZ_RELEASE_BUILD + if (cl_EnableDesyncDebugging) + { + StateHistoryItem inputHistory = AZStd::make_unique(); + while (m_predictiveStateHistory.size() > cl_PredictiveStateHistorySize) + { + m_predictiveStateHistory.erase(m_predictiveStateHistory.begin()); + } + GetNetBindComponent()->SerializeEntityCorrection(*inputHistory); + m_predictiveStateHistory.emplace(m_clientInputId, AZStd::move(inputHistory)); + } +#endif + // Send the input to server (only when we are not migrating) if (!IsMigrating()) { - SendClientInput(inputArray, hashSerializer.GetHash(), processInputResult); + SendClientInput(inputArray, hashSerializer.GetHash()); } } } @@ -563,7 +491,7 @@ namespace Multiplayer void LocalPredictionPlayerInputComponentController::UpdateBankedTime(AZ::TimeMs deltaTimeMs) { const double deltaTime = static_cast(deltaTimeMs) / 1000.0; - const double inputRate = static_cast(static_cast(cl_InputRateMs)) / 1000.0; + const double clientInputRateSec = static_cast(static_cast(cl_InputRateMs)) / 1000.0; const double maxRewindHistory = static_cast(static_cast(cl_MaxRewindHistoryMs)) / 1000.0; // Update banked time accumulator @@ -577,18 +505,11 @@ namespace Multiplayer NetworkInput& input = m_lastInputReceived[0]; { - ScopedAlterTime scopedTime(input.GetHostFrameId(), input.GetHostTimeMs(), DefaultBlendFactor, AzNetworking::InvalidConnectionId); - GetNetBindComponent()->ProcessInput(input, inputRate); + ScopedAlterTime scopedTime(input.GetHostFrameId(), input.GetHostTimeMs(), DefaultBlendFactor, GetNetBindComponent()->GetOwningConnectionId()); + GetNetBindComponent()->ProcessInput(input, clientInputRateSec); } - AZLOG - ( - NET_Prediction, - "Forced InputId=%d - i=[%s] o=[%s]", - aznumeric_cast(input.GetClientInputId()), - GetInputString(input).c_str(), - GetCorrectionDataString(GetNetBindComponent()).c_str() - ); + AZLOG(NET_Prediction, "Forced InputId=%d", aznumeric_cast(input.GetClientInputId())); } // Decay our bank time window, in case the remote endpoint has suffered a more persistent shift in latency, this should cause the client to eventually recover diff --git a/Gems/Multiplayer/Code/Source/Components/NetBindComponent.cpp b/Gems/Multiplayer/Code/Source/Components/NetBindComponent.cpp index cfaeaedeff..d8e8a765ce 100644 --- a/Gems/Multiplayer/Code/Source/Components/NetBindComponent.cpp +++ b/Gems/Multiplayer/Code/Source/Components/NetBindComponent.cpp @@ -267,10 +267,15 @@ namespace Multiplayer return m_isProcessingInput; } + bool NetBindComponent::IsReprocessingInput() const + { + return m_isReprocessingInput; + } + void NetBindComponent::CreateInput(NetworkInput& networkInput, float deltaTime) { - // Only autonomous or authority runs this logic - AZ_Assert(m_netEntityRole == NetEntityRole::Autonomous || m_netEntityRole == NetEntityRole::Authority, "Incorrect network role for input creation"); + // Only autonomous runs this logic + AZ_Assert(IsNetEntityRoleAutonomous(), "Incorrect network role for input creation"); for (MultiplayerComponent* multiplayerComponent : m_multiplayerInputComponentVector) { multiplayerComponent->GetController()->CreateInput(networkInput, deltaTime); @@ -279,12 +284,21 @@ namespace Multiplayer void NetBindComponent::ProcessInput(NetworkInput& networkInput, float deltaTime) { + m_isProcessingInput = true; // Only autonomous and authority runs this logic AZ_Assert((NetworkRoleHasController(m_netEntityRole)), "Incorrect network role for input processing"); for (MultiplayerComponent* multiplayerComponent : m_multiplayerInputComponentVector) { multiplayerComponent->GetController()->ProcessInput(networkInput, deltaTime); } + m_isProcessingInput = false; + } + + void NetBindComponent::ReprocessInput(NetworkInput& networkInput, float deltaTime) + { + m_isReprocessingInput = true; + ProcessInput(networkInput, deltaTime); + m_isReprocessingInput = false; } bool NetBindComponent::HandleRpcMessage(AzNetworking::IConnection* invokingConnection, NetEntityRole remoteRole, NetworkEntityRpcMessage& message) @@ -394,6 +408,11 @@ namespace Multiplayer m_entityPreRenderEvent.Signal(deltaTime, blendFactor); } + void NetBindComponent::NotifyCorrection() + { + m_entityCorrectionEvent.Signal(); + } + void NetBindComponent::AddEntityStopEventHandler(EntityStopEvent::Handler& eventHandler) { eventHandler.Connect(m_entityStopEvent); @@ -429,6 +448,11 @@ namespace Multiplayer eventHandler.Connect(m_entityPreRenderEvent); } + void NetBindComponent::AddEntityCorrectionEventHandler(EntityCorrectionEvent::Handler& eventHandler) + { + eventHandler.Connect(m_entityCorrectionEvent); + } + bool NetBindComponent::SerializeEntityCorrection(AzNetworking::ISerializer& serializer) { m_predictableRecord.ResetConsumedBits(); @@ -656,6 +680,7 @@ namespace Multiplayer MultiplayerComponent* multiplayerComponent = azrtti_cast(component); if (multiplayerComponent != nullptr) { + multiplayerComponent->SetOwningConnectionId(m_owningConnectionId); m_multiplayerInputComponentVector.push_back(multiplayerComponent); } } diff --git a/Gems/Multiplayer/Code/Source/Components/NetworkTransformComponent.cpp b/Gems/Multiplayer/Code/Source/Components/NetworkTransformComponent.cpp index df4d7618fa..fa08794c4d 100644 --- a/Gems/Multiplayer/Code/Source/Components/NetworkTransformComponent.cpp +++ b/Gems/Multiplayer/Code/Source/Components/NetworkTransformComponent.cpp @@ -31,6 +31,7 @@ namespace Multiplayer , m_scaleEventHandler([this](float scale) { OnScaleChangedEvent(scale); }) , m_resetCountEventHandler([this](const uint8_t&) { OnResetCountChangedEvent(); }) , m_entityPreRenderEventHandler([this](float deltaTime, float blendFactor) { OnPreRender(deltaTime, blendFactor); }) + , m_entityCorrectionEventHandler([this]() { OnCorrection(); }) { ; } @@ -47,6 +48,7 @@ namespace Multiplayer ScaleAddEvent(m_scaleEventHandler); ResetCountAddEvent(m_resetCountEventHandler); GetNetBindComponent()->AddEntityPreRenderEventHandler(m_entityPreRenderEventHandler); + GetNetBindComponent()->AddEntityCorrectionEventHandler(m_entityCorrectionEventHandler); // When coming into relevance, reset all blending factors so we don't interpolate to our start position OnResetCountChangedEvent(); @@ -119,6 +121,18 @@ namespace Multiplayer } } + void NetworkTransformComponent::OnCorrection() + { + // Snap to latest + OnResetCountChangedEvent(); + + // Hard set the entities transform + if (!GetTransformComponent()->GetWorldTM().IsClose(m_targetTransform)) + { + GetTransformComponent()->SetWorldTM(m_targetTransform); + } + } + NetworkTransformComponentController::NetworkTransformComponentController(NetworkTransformComponent& parent) : NetworkTransformComponentControllerBase(parent) diff --git a/Gems/Multiplayer/Code/Source/MultiplayerSystemComponent.cpp b/Gems/Multiplayer/Code/Source/MultiplayerSystemComponent.cpp index 893e53fbf6..61dbf9289f 100644 --- a/Gems/Multiplayer/Code/Source/MultiplayerSystemComponent.cpp +++ b/Gems/Multiplayer/Code/Source/MultiplayerSystemComponent.cpp @@ -554,7 +554,7 @@ namespace Multiplayer m_tickFactor = 0.0f; m_lastReplicatedHostTimeMs = packet.GetHostTimeMs(); m_lastReplicatedHostFrameId = packet.GetHostFrameId(); - m_networkTime.AlterTime(m_lastReplicatedHostFrameId, m_lastReplicatedHostTimeMs, AzNetworking::InvalidConnectionId); + m_networkTime.ForceSetTime(m_lastReplicatedHostFrameId, m_lastReplicatedHostTimeMs); } for (AZStd::size_t i = 0; i < packet.GetEntityMessages().size(); ++i) @@ -861,7 +861,7 @@ namespace Multiplayer { m_tickFactor += deltaTime / serverRateSeconds; // Linear close to the origin, but asymptote at y = 1 - m_renderBlendFactor = AZStd::clamp(1.0f - (std::pow(cl_renderTickBlendBase, m_tickFactor)), 0.0f, 1.0f); + m_renderBlendFactor = AZStd::clamp(1.0f - (std::pow(cl_renderTickBlendBase, m_tickFactor)), 0.0f, m_tickFactor); AZLOG ( NET_Blending, diff --git a/Gems/Multiplayer/Code/Source/NetworkEntity/EntityReplication/EntityReplicationManager.cpp b/Gems/Multiplayer/Code/Source/NetworkEntity/EntityReplication/EntityReplicationManager.cpp index b8a77dab58..482d3a1ee8 100644 --- a/Gems/Multiplayer/Code/Source/NetworkEntity/EntityReplication/EntityReplicationManager.cpp +++ b/Gems/Multiplayer/Code/Source/NetworkEntity/EntityReplication/EntityReplicationManager.cpp @@ -524,6 +524,7 @@ namespace Multiplayer bool EntityReplicationManager::HandlePropertyChangeMessage ( + AzNetworking::IConnection* invokingConnection, EntityReplicator* entityReplicator, AzNetworking::PacketId packetId, NetEntityId netEntityId, @@ -558,6 +559,12 @@ namespace Multiplayer NetBindComponent* netBindComponent = replicatorEntity.GetNetBindComponent(); AZ_Assert(netBindComponent != nullptr, "No NetBindComponent"); + if (createEntity) + { + // Always set our invoking connectionId for any newly created entities, since this connection now 'owns' them from a rewind perspective + netBindComponent->SetOwningConnectionId(invokingConnection->GetConnectionId()); + } + const bool changeNetworkRole = (netBindComponent->GetNetEntityRole() != localNetworkRole); if (changeNetworkRole) { @@ -744,7 +751,7 @@ namespace Multiplayer bool EntityReplicationManager::HandleEntityUpdateMessage ( - [[maybe_unused]] AzNetworking::IConnection* invokingConnection, + AzNetworking::IConnection* invokingConnection, const AzNetworking::IPacketHeader& packetHeader, const NetworkEntityUpdateMessage& updateMessage ) @@ -794,7 +801,7 @@ namespace Multiplayer } // This may implicitly create a replicator for us - bool handled = HandlePropertyChangeMessage(entityReplicator, packetHeader.GetPacketId(), updateMessage.GetEntityId(), updateMessage.GetNetworkRole(), outputSerializer, prefabEntityId); + bool handled = HandlePropertyChangeMessage(invokingConnection, entityReplicator, packetHeader.GetPacketId(), updateMessage.GetEntityId(), updateMessage.GetNetworkRole(), outputSerializer, prefabEntityId); AZ_Assert(handled, "Failed to handle NetworkEntityUpdateMessage message"); return handled; @@ -1121,7 +1128,7 @@ namespace Multiplayer } } - bool EntityReplicationManager::HandleEntityMigration([[maybe_unused]] AzNetworking::IConnection* invokingConnection, EntityMigrationMessage& message) + bool EntityReplicationManager::HandleEntityMigration(AzNetworking::IConnection* invokingConnection, EntityMigrationMessage& message) { EntityReplicator* replicator = GetEntityReplicator(message.m_entityId); { @@ -1130,6 +1137,7 @@ namespace Multiplayer AzNetworking::TrackChangedSerializer outputSerializer(message.m_propertyUpdateData.GetBuffer(), static_cast(message.m_propertyUpdateData.GetSize())); if (!HandlePropertyChangeMessage ( + invokingConnection, replicator, AzNetworking::InvalidPacketId, message.m_entityId, diff --git a/Gems/Multiplayer/Code/Source/NetworkEntity/EntityReplication/EntityReplicationManager.h b/Gems/Multiplayer/Code/Source/NetworkEntity/EntityReplication/EntityReplicationManager.h index 9f5e743bbc..731a1a7556 100644 --- a/Gems/Multiplayer/Code/Source/NetworkEntity/EntityReplication/EntityReplicationManager.h +++ b/Gems/Multiplayer/Code/Source/NetworkEntity/EntityReplication/EntityReplicationManager.h @@ -136,6 +136,7 @@ namespace Multiplayer bool HandlePropertyChangeMessage ( + AzNetworking::IConnection* invokingConnection, EntityReplicator* entityReplicator, AzNetworking::PacketId packetId, NetEntityId netEntityId, diff --git a/Gems/Multiplayer/Code/Source/NetworkTime/NetworkTime.cpp b/Gems/Multiplayer/Code/Source/NetworkTime/NetworkTime.cpp index 7c19546fb6..db8d6bd2f7 100644 --- a/Gems/Multiplayer/Code/Source/NetworkTime/NetworkTime.cpp +++ b/Gems/Multiplayer/Code/Source/NetworkTime/NetworkTime.cpp @@ -70,6 +70,15 @@ namespace Multiplayer return (IsTimeRewound() && (rewindConnectionId == m_rewindingConnectionId)) ? m_unalteredFrameId : m_hostFrameId; } + void NetworkTime::ForceSetTime(HostFrameId frameId, AZ::TimeMs timeMs) + { + AZ_Assert(!IsTimeRewound(), "Forcibly setting network time is unsupported under a rewound time scope"); + m_unalteredFrameId = frameId; + m_hostFrameId = frameId; + m_hostTimeMs = timeMs; + m_rewindingConnectionId = AzNetworking::InvalidConnectionId; + } + void NetworkTime::AlterTime(HostFrameId frameId, AZ::TimeMs timeMs, AzNetworking::ConnectionId rewindConnectionId) { m_hostFrameId = frameId; diff --git a/Gems/Multiplayer/Code/Source/NetworkTime/NetworkTime.h b/Gems/Multiplayer/Code/Source/NetworkTime/NetworkTime.h index 6c211c816d..0278ddd2b0 100644 --- a/Gems/Multiplayer/Code/Source/NetworkTime/NetworkTime.h +++ b/Gems/Multiplayer/Code/Source/NetworkTime/NetworkTime.h @@ -33,6 +33,7 @@ namespace Multiplayer float GetHostBlendFactor() const override; AzNetworking::ConnectionId GetRewindingConnectionId() const override; HostFrameId GetHostFrameIdForRewindingConnection(AzNetworking::ConnectionId rewindConnectionId) const override; + void ForceSetTime(HostFrameId frameId, AZ::TimeMs timeMs) override; void AlterTime(HostFrameId frameId, AZ::TimeMs timeMs, AzNetworking::ConnectionId rewindConnectionId) override; void AlterBlendFactor(float blendFactor) override; void SyncEntitiesToRewindState(const AZ::Aabb& rewindVolume) override;