Merge pull request #2993 from aws-lumberyard-dev/MultiplayerDesyncsAndCorrectionFixes

Multiplayer desyncs and correction fixes
This commit is contained in:
kberg-amzn
2021-08-16 14:53:55 -07:00
committed by GitHub
17 changed files with 205 additions and 252 deletions
@@ -10,22 +10,9 @@
namespace AzNetworking
{
StringifySerializer::StringifySerializer(char delimeter, bool outputFieldNames, const AZStd::string& seperator)
: m_delimeter(delimeter)
, m_outputFieldNames(outputFieldNames)
, m_separator(seperator)
const StringifySerializer::ValueMap& StringifySerializer::GetValueMap() const
{
;
}
const AZStd::string& StringifySerializer::GetString() const
{
return m_string;
}
const StringifySerializer::StringMap& StringifySerializer::GetValueMap() const
{
return m_map;
return m_valueMap;
}
SerializerMode StringifySerializer::GetSerializerMode() const
@@ -137,22 +124,9 @@ namespace AzNetworking
template <typename T>
bool StringifySerializer::ProcessData(const char* name, const T& value)
{
// Only add delimeters after we have processed at least one element
if (!m_string.empty())
{
m_string += m_delimeter;
}
if (m_outputFieldNames)
{
m_string += m_prefix;
m_string += name;
m_string += m_separator;
}
AZ::CVarFixedString string = AZ::ConsoleTypeHelpers::ValueToString(value);
m_string += string.c_str();
m_map[m_prefix + name] = string.c_str();
const AZStd::string keyString = m_prefix + name;
AZ::CVarFixedString valueString = AZ::ConsoleTypeHelpers::ValueToString(value);
m_valueMap[keyString] = valueString.c_str();
return true;
}
}
@@ -20,17 +20,12 @@ namespace AzNetworking
{
public:
using StringMap = AZStd::map<AZStd::string, AZStd::string>;
using ValueMap = AZStd::map<AZStd::string, AZStd::string>;
StringifySerializer(char delimeter = ' ', bool outputFieldNames = true, const AZStd::string& seperator = "=");
StringifySerializer() = default;
// GetString
// After serializing objects, get the serialized values as a single string
const AZStd::string& GetString() const;
// GetValueMap
// After serializing objects, get the serialized values as key value pairs
const StringMap& GetValueMap() const;
//! After serializing objects, get the serialized values as a map of key/value pairs.
const ValueMap& GetValueMap() const;
// ISerializer interfaces
SerializerMode GetSerializerMode() const override;
@@ -62,15 +57,8 @@ namespace AzNetworking
template <typename T>
bool ProcessData(const char* name, const T& value);
private:
char m_delimeter;
bool m_outputFieldNames = true;
StringMap m_map;
AZStd::string m_string;
ValueMap m_valueMap;
AZStd::string m_prefix;
AZStd::string m_separator;
AZStd::deque<AZStd::size_t> m_prefixSizeStack;
};
}
@@ -10,11 +10,10 @@
#include <Source/AutoGen/LocalPredictionPlayerInputComponent.AutoComponent.h>
#include <Multiplayer/Components/NetBindComponent.h>
#include <AzNetworking/Serialization/StringifySerializer.h>
namespace Multiplayer
{
using CorrectionEvent = AZ::Event<>;
class LocalPredictionPlayerInputComponent
: public LocalPredictionPlayerInputComponentBase
{
@@ -41,8 +40,7 @@ namespace Multiplayer
(
AzNetworking::IConnection* invokingConnection,
const Multiplayer::NetworkInputArray& inputArray,
const AZ::HashValue32& stateHash,
const AzNetworking::PacketEncodingBuffer& clientState
const AZ::HashValue32& stateHash
) override;
void HandleSendMigrateClientInput
@@ -58,11 +56,6 @@ namespace Multiplayer
const AzNetworking::PacketEncodingBuffer& correction
) override;
//! Return true if we're currently replaying inputs after a correction.
//! If this value returns true, effects, audio, and other cosmetic triggers should be suppressed
//! @return true if we're within correction scope and replaying inputs
bool IsReplayingInput() const;
//! Return true if we're currently migrating from one host to another.
//! @return boolean true if we're currently migrating from one host to another
bool IsMigrating() const;
@@ -70,8 +63,6 @@ namespace Multiplayer
ClientInputId GetLastInputId() const;
HostFrameId GetInputFrameId(const NetworkInput& input) const;
void CorrectionEventAddHandle(CorrectionEvent::Handler& handler);
private:
void OnMigrateStart(ClientInputId migratedInputId);
@@ -79,6 +70,9 @@ namespace Multiplayer
void UpdateAutonomous(AZ::TimeMs deltaTimeMs);
void UpdateBankedTime(AZ::TimeMs deltaTimeMs);
using StateHistoryItem = AZStd::unique_ptr<AzNetworking::StringifySerializer>;
AZStd::map<ClientInputId, StateHistoryItem> m_predictiveStateHistory;
// Implicitly sorted player input history, back() is the input that corresponds to the latest client input Id
NetworkInputHistory m_inputHistory;
@@ -88,7 +82,6 @@ namespace Multiplayer
AZ::ScheduledEvent m_autonomousUpdateEvent; // Drives autonomous input collection
AZ::ScheduledEvent m_updateBankedTimeEvent; // Drives authority bank time updates
CorrectionEvent m_correctionEvent;
EntityMigrationStartEvent::Handler m_migrateStartHandler;
EntityMigrationEndEvent::Handler m_migrateEndHandler;
@@ -104,7 +97,6 @@ namespace Multiplayer
ClientInputId m_lastMigratedInputId = ClientInputId{ 0 }; // Used to resend inputs that were queued during a migration event
HostFrameId m_serverMigrateFrameId = InvalidHostFrameId;
bool m_replayingInput = false; // True if we're replaying inputs under a correction event (use this to suppress effects or audio)
bool m_allowMigrateClientInput = false; // True if this component was migrated, we will allow the client to send us migrated inputs (one time only)
};
}
@@ -36,6 +36,7 @@ namespace Multiplayer
using EntityMigrationEndEvent = AZ::Event<>;
using EntityServerMigrationEvent = AZ::Event<const ConstNetworkEntityHandle&, HostId, AzNetworking::ConnectionId>;
using EntityPreRenderEvent = AZ::Event<float, float>;
using EntityCorrectionEvent = AZ::Event<>;
//! @class NetBindComponent
//! @brief Component that provides net-binding to a networked entity.
@@ -87,9 +88,19 @@ namespace Multiplayer
AzNetworking::ConnectionId GetOwningConnectionId() const;
void SetAllowAutonomy(bool value);
MultiplayerComponentInputVector AllocateComponentInputs();
//! Return true if we're currently processing inputs.
//! @return true if we're within ProcessInput scope and writing to predictive state
bool IsProcessingInput() const;
//! Return true if we're currently replaying inputs after a correction.
//! If this value returns true, effects, audio, and other cosmetic triggers should be suppressed
//! @return true if we're within correction scope and replaying inputs
bool IsReprocessingInput() const;
void CreateInput(NetworkInput& networkInput, float deltaTime);
void ProcessInput(NetworkInput& networkInput, float deltaTime);
void ReprocessInput(NetworkInput& networkInput, float deltaTime);
bool HandleRpcMessage(AzNetworking::IConnection* invokingConnection, NetEntityRole remoteRole, NetworkEntityRpcMessage& message);
bool HandlePropertyChangeMessage(AzNetworking::ISerializer& serializer, bool notifyChanges = true);
@@ -108,6 +119,7 @@ namespace Multiplayer
void NotifyMigrationEnd();
void NotifyServerMigration(HostId hostId, AzNetworking::ConnectionId connectionId);
void NotifyPreRender(float deltaTime, float blendFactor);
void NotifyCorrection();
void AddEntityStopEventHandler(EntityStopEvent::Handler& eventHandler);
void AddEntityDirtiedEventHandler(EntityDirtiedEvent::Handler& eventHandler);
@@ -116,6 +128,7 @@ namespace Multiplayer
void AddEntityMigrationEndEventHandler(EntityMigrationEndEvent::Handler& eventHandler);
void AddEntityServerMigrationEventHandler(EntityServerMigrationEvent::Handler& eventHandler);
void AddEntityPreRenderEventHandler(EntityPreRenderEvent::Handler& eventHandler);
void AddEntityCorrectionEventHandler(EntityCorrectionEvent::Handler& handler);
bool SerializeEntityCorrection(AzNetworking::ISerializer& serializer);
@@ -165,6 +178,7 @@ namespace Multiplayer
EntityMigrationEndEvent m_entityMigrationEndEvent;
EntityServerMigrationEvent m_entityServerMigrationEvent;
EntityPreRenderEvent m_entityPreRenderEvent;
EntityCorrectionEvent m_entityCorrectionEvent;
AZ::Event<> m_onRemove;
RpcSendEvent::Handler m_handleLocalServerRpcMessageEventHandle;
AZ::Event<>::Handler m_handleMarkedDirty;
@@ -177,10 +191,11 @@ namespace Multiplayer
AzNetworking::ConnectionId m_owningConnectionId = AzNetworking::InvalidConnectionId;
bool m_isProcessingInput = false;
bool m_isMigrationDataValid = false;
bool m_needsToBeStopped = false;
bool m_allowAutonomy = false; // Set to true for the hosts controlled entity
bool m_isProcessingInput = false; // Set to true when we are processing input
bool m_isReprocessingInput = false; // Set to true when we are reprocessing input (during a correction)
bool m_isMigrationDataValid = false;
bool m_needsToBeStopped = false;
bool m_allowAutonomy = false; // Set to true for the hosts controlled entity
friend class NetworkEntityManager;
friend class EntityReplicationManager;
@@ -30,6 +30,7 @@ namespace Multiplayer
private:
void OnPreRender(float deltaTime, float blendFactor);
void OnCorrection();
void OnRotationChangedEvent(const AZ::Quaternion& rotation);
void OnTranslationChangedEvent(const AZ::Vector3& translation);
@@ -47,6 +48,7 @@ namespace Multiplayer
AZ::Event<uint8_t>::Handler m_resetCountEventHandler;
EntityPreRenderEvent::Handler m_entityPreRenderEventHandler;
EntityCorrectionEvent::Handler m_entityCorrectionEventHandler;
Multiplayer::HostFrameId m_targetHostFrameId = HostFrameId(0);
};
@@ -58,6 +58,11 @@ namespace Multiplayer
//! @return the HostFrameId taking into account the provided rewinding connectionId
virtual HostFrameId GetHostFrameIdForRewindingConnection(AzNetworking::ConnectionId rewindConnectionId) const = 0;
//! Forcibly sets the current network time to the provided frameId and game time in milliseconds.
//! @param frameId the new HostFrameId to use
//! @param timeMs the new HostTimeMs to use
virtual void ForceSetTime(HostFrameId frameId, AZ::TimeMs timeMs) = 0;
//! Alters the current HostFrameId and binds that alteration to the provided ConnectionId.
//! @param frameId the new HostFrameId to use
//! @param timeMs the new HostTimeMs to use
@@ -78,7 +78,7 @@ namespace Multiplayer
const HostFrameId frameTime = GetCurrentTimeForProperty();
if (frameTime < m_headTime)
{
AZ_Assert(false, "Trying to mutate a rewindable in the past");
AZ_Assert(false, "Trying to mutate a rewindable value in the past");
}
else if (m_headTime < frameTime)
{
@@ -643,16 +643,15 @@ bool {{ ClassName }}::Serialize{{ AutoComponentMacros.GetNetPropertiesSetName(Re
const uint32_t lastBit = static_cast<uint32_t>({{ AutoComponentMacros.GetNetPropertiesQualifiedPropertyDirtyEnum(Component.attrib['Name'], ReplicateFrom, ReplicateTo, Property, 'End') }});
{% endif %}
{% if Property.attrib['IsRewindable']|booleanTrue %}
AzNetworking::FixedSizeBitsetView deltaRecord(replicationRecord.m_{{ LowerFirst(AutoComponentMacros.GetNetPropertiesSetName(ReplicateFrom, ReplicateTo)) }}, firstBit, lastBit - firstBit + 1);
m_{{ LowerFirst(Property.attrib['Name']) }}.Serialize(serializer, deltaRecord);
{% else %}
if (deltaRecord.AnySet())
{
{% if Property.attrib['Container'] == 'Vector' %}
serializer.Serialize<AZStd::fixed_vector<{{ Property.attrib['Type'] }}, {{ Property.attrib['Count'] }}>>(m_{{ LowerFirst(Property.attrib['Name']) }}, "{{ LowerFirst(Property.attrib['Name']) }}");
serializer.Serialize<AZStd::fixed_vector<{{ Property.attrib['Type'] }}, {{ Property.attrib['Count'] }}>>(m_{{ LowerFirst(Property.attrib['Name']) }}, "{{ LowerFirst(Property.attrib['Name']) }}");
{% elif Property.attrib['Container'] == 'Array' %}
serializer.Serialize<AZStd::array<{{ Property.attrib['Type'] }}, {{ Property.attrib['Count'] }}>>(m_{{ LowerFirst(Property.attrib['Name']) }}, "{{ LowerFirst(Property.attrib['Name']) }}");
serializer.Serialize<AZStd::array<{{ Property.attrib['Type'] }}, {{ Property.attrib['Count'] }}>>(m_{{ LowerFirst(Property.attrib['Name']) }}, "{{ LowerFirst(Property.attrib['Name']) }}");
{% endif %}
{% endif %}
}
}
{% else %}
Multiplayer::SerializeNetworkPropertyHelper
@@ -22,7 +22,6 @@
<RemoteProcedure Name="SendClientInput" InvokeFrom="Autonomous" HandleOn="Authority" IsPublic="true" IsReliable="false" GenerateEventBindings="false" Description="Client to server move / input RPC">
<Param Type="Multiplayer::NetworkInputArray" Name="inputArray" />
<Param Type="AZ::HashValue32" Name="stateHash" />
<Param Type="AzNetworking::PacketEncodingBuffer" Name="clientState" Description="This is for debugging desyncs only; release games should not populate this parameter" />
</RemoteProcedure>
<RemoteProcedure Name="SendClientInputCorrection" InvokeFrom="Authority" HandleOn="Autonomous" IsPublic="true" IsReliable="false" GenerateEventBindings="false" Description="Autonomous proxy correction RPC">
@@ -21,28 +21,54 @@ namespace Multiplayer
AZ_CVAR(AZ::TimeMs, cl_MaxRewindHistoryMs, AZ::TimeMs{ 2000 }, nullptr, AZ::ConsoleFunctorFlags::Null, "Maximum number of milliseconds to keep for server correction rewind and replay");
#ifndef AZ_RELEASE_BUILD
AZ_CVAR(float, cl_DebugHackTimeMultiplier, 1.0f, nullptr, AZ::ConsoleFunctorFlags::Null, "Scalar value used to simulate clock hacking cheats for validating bank time system and anticheat");
AZ_CVAR(bool, cl_EnableDesyncDebugging, false, nullptr, AZ::ConsoleFunctorFlags::Null, "If enabled, debug logs will contain verbose information on detected state desyncs");
AZ_CVAR(bool, cl_EnableDesyncDebugging, true, nullptr, AZ::ConsoleFunctorFlags::Null, "If enabled, debug logs will contain verbose information on detected state desyncs");
AZ_CVAR(uint32_t, cl_PredictiveStateHistorySize, 120, nullptr, AZ::ConsoleFunctorFlags::Null, "Controls how many inputs of predictive state should be retained for debugging desyncs");
#endif
AZ_CVAR(bool, sv_ForceCorrections, false, nullptr, AZ::ConsoleFunctorFlags::Null, "If enabled, the server will force a correction for every input received for debugging");
AZ_CVAR(bool, sv_EnableCorrections, true, nullptr, AZ::ConsoleFunctorFlags::Null, "Enables server corrections on autonomous proxy desyncs");
AZ_CVAR(double, sv_MaxBankTimeWindowSec, 0.2, nullptr, AZ::ConsoleFunctorFlags::Null, "Maximum bank time we allow before we start rejecting autonomous proxy move inputs due to anticheat kicking in");
AZ_CVAR(double, sv_BankTimeDecay, 0.025, nullptr, AZ::ConsoleFunctorFlags::Null, "Amount to decay bank time by, in case of more permanent shifts in client latency");
AZ_CVAR(AZ::TimeMs, sv_MinCorrectionTimeMs, AZ::TimeMs{ 100 }, nullptr, AZ::ConsoleFunctorFlags::Null, "Minimum time to wait between sending out corrections in order to avoid flooding corrections on high-latency connections");
AZ_CVAR(AZ::TimeMs, sv_InputUpdateTimeMs, AZ::TimeMs{ 5 }, nullptr, AZ::ConsoleFunctorFlags::Null, "Minimum time between component updates");
// Debug helper functions
AZStd::string GetInputString(NetworkInput& input)
void PrintCorrectionDifferences(const AzNetworking::StringifySerializer& client, const AzNetworking::StringifySerializer& server)
{
AzNetworking::StringifySerializer serializer(',', false);
input.Serialize(serializer);
return serializer.GetString();
const auto& clientMap = client.GetValueMap();
const auto& serverMap = server.GetValueMap();
AzNetworking::StringifySerializer::ValueMap differences = clientMap;
for (auto iter = server.GetValueMap().begin(); iter != server.GetValueMap().end(); ++iter)
{
auto serverValueIter = clientMap.find(iter->first);
if (iter->second == differences[iter->first])
{
differences.erase(iter->first);
}
}
if (differences.empty())
{
AZLOG_ERROR("The hash mismatched, but no differences were found.")
}
for (auto iter = differences.begin(); iter != differences.end(); ++iter)
{
auto clientValueIter = clientMap.find(iter->first);
auto serverValueIter = serverMap.find(iter->first);
if (clientValueIter == clientMap.end() || serverValueIter == serverMap.end())
{
AZLOG_ERROR(" %s (Not found in server and/or client value map!)", iter->first.c_str());
continue;
}
AZLOG_ERROR(" %s Server=%s Client=%s", iter->first.c_str(), serverValueIter->second.c_str(), clientValueIter->second.c_str());
}
}
AZStd::string GetCorrectionDataString(NetBindComponent* netBindComponent)
inline double ConvertTimeMsToSeconds(AZ::TimeMs value)
{
AzNetworking::StringifySerializer serializer(',', false);
netBindComponent->SerializeEntityCorrection(serializer);
return serializer.GetString();
return static_cast<double>(static_cast<AZ::TimeMs>(value)) / 1000.0;
}
void LocalPredictionPlayerInputComponent::LocalPredictionPlayerInputComponent::Reflect(AZ::ReflectContext* context)
@@ -106,8 +132,7 @@ namespace Multiplayer
(
AzNetworking::IConnection* invokingConnection,
const Multiplayer::NetworkInputArray& inputArray,
const AZ::HashValue32& stateHash,
[[maybe_unused]] const AzNetworking::PacketEncodingBuffer& clientState
const AZ::HashValue32& stateHash
)
{
if (invokingConnection == nullptr)
@@ -131,7 +156,7 @@ namespace Multiplayer
}
const AZ::TimeMs currentTimeMs = AZ::GetElapsedTimeMs();
const double clientInputRateSec = static_cast<double>(static_cast<AZ::TimeMs>(cl_InputRateMs)) / 1000.0;
const double clientInputRateSec = ConvertTimeMsToSeconds(cl_InputRateMs);
m_lastInputReceivedTimeMs = currentTimeMs;
// Keep track of last inputs received, also allows us to update frame ids
@@ -156,8 +181,8 @@ namespace Multiplayer
if (m_clientBankedTime < sv_MaxBankTimeWindowSec)
{
// Client blends from previous frame to target so here we subtract blend factor to get to that state
const float blendFactor = AZStd::min(AZStd::max(0.f, input.GetHostBlendFactor()), 1.f);
const AZ::TimeMs blendMs = AZ::TimeMs(static_cast<float>(static_cast<AZ::TimeMs>(cl_InputRateMs)) * (1.f - blendFactor));
const float blendFactor = AZStd::min(AZStd::max(0.f, input.GetHostBlendFactor()), 1.0f);
const AZ::TimeMs blendMs = AZ::TimeMs(static_cast<float>(static_cast<AZ::TimeMs>(cl_InputRateMs)) * (1.0f - blendFactor));
m_clientBankedTime = AZStd::min(m_clientBankedTime + clientInputRateSec, (double)sv_MaxBankTimeWindowSec); // clamp to boundary
{
ScopedAlterTime scopedTime(input.GetHostFrameId(), input.GetHostTimeMs() - blendMs, input.GetHostBlendFactor(), invokingConnection->GetConnectionId());
@@ -179,7 +204,7 @@ namespace Multiplayer
}
}
if (sv_EnableCorrections && (currentTimeMs - m_lastCorrectionSentTimeMs > sv_MinCorrectionTimeMs))
if (sv_ForceCorrections || (sv_EnableCorrections && (currentTimeMs - m_lastCorrectionSentTimeMs > sv_MinCorrectionTimeMs)))
{
m_lastCorrectionSentTimeMs = currentTimeMs;
@@ -213,69 +238,6 @@ namespace Multiplayer
// Send correction
SendClientInputCorrection(GetLastInputId(), correction);
#ifndef AZ_RELEASE_BUILD
AZStd::string clientStateString;
AZStd::string serverStateString;
if (cl_EnableDesyncDebugging)
{
// In debug, show which states caused the correction
// Write in client state
AzNetworking::NetworkOutputSerializer clientStateSerializer(clientState.GetBuffer(), static_cast<uint32_t>(clientState.GetSize()));
GetNetBindComponent()->SerializeEntityCorrection(clientStateSerializer);
// Read out state values
AzNetworking::StringifySerializer clientValues;
GetNetBindComponent()->SerializeEntityCorrection(clientValues);
// Restore server state
AzNetworking::NetworkOutputSerializer serverStateSerializer(correction.GetBuffer(), static_cast<uint32_t>(correction.GetSize()));
GetNetBindComponent()->SerializeEntityCorrection(serverStateSerializer);
// Read out state values
AzNetworking::StringifySerializer serverValues;
GetNetBindComponent()->SerializeEntityCorrection(serverValues);
AZStd::map<AZStd::string, AZStd::pair<AZStd::string, AZStd::string>> mapComparison;
// put the server value in the first part of the pair
for (const auto& pair : serverValues.GetValueMap())
{
mapComparison[pair.first].first = pair.second;
}
// put the client value in the second part of the pair
for (const auto& pair : clientValues.GetValueMap())
{
mapComparison[pair.first].second = pair.second;
}
bool firstIt = true;
for (const auto& mapPair : mapComparison)
{
if (mapPair.second.first != mapPair.second.second)
{
if (!firstIt)
{
clientStateString += ",";
serverStateString += ",";
}
firstIt = false;
AZStd::string clientValue = mapPair.second.second.empty() ? "<no value>" : mapPair.second.second;
AZStd::string serverValue = mapPair.second.first.empty() ? "<no value>" : mapPair.second.first;
clientStateString += mapPair.first + "=" + clientValue;
serverStateString += mapPair.first + "=" + serverValue;
}
}
}
else
{
clientStateString = "available in debug only";
serverStateString = "available in debug only";
}
AZLOG_ERROR("** Autonomous proxy desync detected! ** clientState=[%s], serverState=[%s]", clientStateString.c_str(), serverStateString.c_str());
#endif
}
}
}
@@ -302,7 +264,7 @@ namespace Multiplayer
return;
}
const float clientInputRateSec = static_cast<float>(static_cast<AZ::TimeMs>(cl_InputRateMs)) / 1000.0;
const double clientInputRateSec = ConvertTimeMsToSeconds(cl_InputRateMs);
// Copy array so we can modify input ids
NetworkInputMigrationVector inputArrayCopy = inputArray;
@@ -317,14 +279,7 @@ namespace Multiplayer
ScopedAlterTime scopedTime(input.GetHostFrameId(), input.GetHostTimeMs(), input.GetHostBlendFactor(), invokingConnection->GetConnectionId());
GetNetBindComponent()->ProcessInput(input, clientInputRateSec);
AZLOG
(
NET_Prediction,
"Migrated InputId=%d - i=[%s] o=[%s]",
aznumeric_cast<int32_t>(input.GetClientInputId()),
GetInputString(input).c_str(),
GetCorrectionDataString(GetNetBindComponent()).c_str()
);
AZLOG(NET_Prediction, "Migrated InputId=%d", aznumeric_cast<int32_t>(input.GetClientInputId()));
// Don't bother checking for corrections here, the next regular input will trigger any corrections if necessary
// Also don't bother with any cheat detection here, because the input array is limited in size and at most and can only be sent once
@@ -334,7 +289,7 @@ namespace Multiplayer
void LocalPredictionPlayerInputComponentController::HandleSendClientInputCorrection
(
AzNetworking::IConnection* invokingConnection,
[[maybe_unused]] AzNetworking::IConnection* invokingConnection,
const Multiplayer::ClientInputId& inputId,
const AzNetworking::PacketEncodingBuffer& correction
)
@@ -357,15 +312,26 @@ namespace Multiplayer
// Apply the correction
AzNetworking::TrackChangedSerializer<AzNetworking::NetworkOutputSerializer> serializer(correction.GetBuffer(), static_cast<uint32_t>(correction.GetSize()));
GetNetBindComponent()->SerializeEntityCorrection(serializer);
m_correctionEvent.Signal();
GetNetBindComponent()->NotifyCorrection();
AZLOG
(
NET_Prediction,
"Corrected InputId=%d - o=[%s]",
aznumeric_cast<int32_t>(m_lastCorrectionInputId),
GetCorrectionDataString(GetNetBindComponent()).c_str()
);
#ifndef AZ_RELEASE_BUILD
if (cl_EnableDesyncDebugging)
{
AZLOG_INFO("** Autonomous Desync - Corrected clientInputId=%d ", aznumeric_cast<int32_t>(inputId));
auto iter = m_predictiveStateHistory.find(inputId);
if (iter != m_predictiveStateHistory.end())
{
// Read out state values
AzNetworking::StringifySerializer serverValues;
GetNetBindComponent()->SerializeEntityCorrection(serverValues);
PrintCorrectionDifferences(*iter->second, serverValues);
}
else
{
AZLOG_INFO("Received correction that is too old to diff, increase cl_PredictiveStateHistorySize");
}
}
#endif
const uint32_t inputHistorySize = static_cast<uint32_t>(m_inputHistory.Size());
const uint32_t historicalDelta = aznumeric_cast<uint32_t>(m_clientInputId - inputId); // Do not replay the move we just corrected, that was already processed by the server
@@ -373,46 +339,18 @@ namespace Multiplayer
// If this correction is for a move outside our input history window, just start replaying from the oldest move we have available
const uint32_t startReplayIndex = (inputHistorySize > historicalDelta) ? (inputHistorySize - historicalDelta) : 0;
// Flag that we are replaying inputs
struct ScopedReplayingInput
{
ScopedReplayingInput(LocalPredictionPlayerInputComponentController* instance)
: m_instance(instance)
{
m_instance->m_replayingInput = true;
}
~ScopedReplayingInput()
{
m_instance->m_replayingInput = false;
}
LocalPredictionPlayerInputComponentController* m_instance;
};
ScopedReplayingInput markReplayingInput(this);
const float clientInputRateSec = static_cast<float>(static_cast<AZ::TimeMs>(cl_InputRateMs)) / 1000.0;
const double clientInputRateSec = ConvertTimeMsToSeconds(cl_InputRateMs);
for (uint32_t replayIndex = startReplayIndex; replayIndex < inputHistorySize; ++replayIndex)
{
// Reprocess the input for this frame
NetworkInput& input = m_inputHistory[replayIndex];
ScopedAlterTime scopedTime(input.GetHostFrameId(), input.GetHostTimeMs(), input.GetHostBlendFactor(), invokingConnection->GetConnectionId());
GetNetBindComponent()->ProcessInput(input, clientInputRateSec);
GetNetBindComponent()->ReprocessInput(input, clientInputRateSec);
AZLOG
(
NET_Prediction,
"Replayed InputId=%d - i=[%s] o=[%s]",
aznumeric_cast<int32_t>(input.GetClientInputId()),
GetInputString(input).c_str(),
GetCorrectionDataString(GetNetBindComponent()).c_str()
);
AZLOG(NET_Prediction, "Replayed InputId=%d", aznumeric_cast<int32_t>(input.GetClientInputId()));
}
}
bool LocalPredictionPlayerInputComponentController::IsReplayingInput() const
{
return m_replayingInput;
}
bool LocalPredictionPlayerInputComponentController::IsMigrating() const
{
return m_lastMigratedInputId != ClientInputId{ 0 };
@@ -432,11 +370,6 @@ namespace Multiplayer
return (input.GetHostFrameId() == InvalidHostFrameId) ? m_serverMigrateFrameId : input.GetHostFrameId();
}
void LocalPredictionPlayerInputComponentController::CorrectionEventAddHandle(CorrectionEvent::Handler& handler)
{
handler.Connect(m_correctionEvent);
}
void LocalPredictionPlayerInputComponentController::OnMigrateStart(ClientInputId migratedInputId)
{
m_lastMigratedInputId = migratedInputId;
@@ -477,9 +410,9 @@ namespace Multiplayer
void LocalPredictionPlayerInputComponentController::UpdateAutonomous(AZ::TimeMs deltaTimeMs)
{
const double deltaTime = static_cast<double>(deltaTimeMs) / 1000.0;
const double inputRate = static_cast<double>(static_cast<AZ::TimeMs>(cl_InputRateMs)) / 1000.0;
const double maxRewindHistory = static_cast<double>(static_cast<AZ::TimeMs>(cl_MaxRewindHistoryMs)) / 1000.0;
const double deltaTime = ConvertTimeMsToSeconds(deltaTimeMs);
const double clientInputRateSec = ConvertTimeMsToSeconds(cl_InputRateMs);
const double maxRewindHistory = ConvertTimeMsToSeconds(cl_MaxRewindHistoryMs);
#ifndef AZ_RELEASE_BUILD
m_moveAccumulator += deltaTime * cl_DebugHackTimeMultiplier;
@@ -487,13 +420,13 @@ namespace Multiplayer
m_moveAccumulator += deltaTime;
#endif
const uint32_t maxClientInputs = inputRate > 0.0 ? static_cast<uint32_t>(maxRewindHistory / inputRate) : 0;
const uint32_t maxClientInputs = clientInputRateSec > 0.0 ? static_cast<uint32_t>(maxRewindHistory / clientInputRateSec) : 0;
IMultiplayer* multiplayer = GetMultiplayer();
INetworkTime* networkTime = GetNetworkTime();
while (m_moveAccumulator >= inputRate)
while (m_moveAccumulator >= clientInputRateSec)
{
m_moveAccumulator -= inputRate;
m_moveAccumulator -= clientInputRateSec;
++m_clientInputId;
NetworkInputArray inputArray(GetEntityHandle());
@@ -505,35 +438,17 @@ namespace Multiplayer
input.SetHostBlendFactor(multiplayer->GetCurrentBlendFactor());
// Allow components to form the input for this frame
GetNetBindComponent()->CreateInput(input, inputRate);
GetNetBindComponent()->CreateInput(input, clientInputRateSec);
// Process the input for this frame
GetNetBindComponent()->ProcessInput(input, inputRate);
GetNetBindComponent()->ProcessInput(input, clientInputRateSec);
AZLOG
(
NET_Prediction,
"Processed InutId=%d - i=[%s] o=[%s]",
aznumeric_cast<int32_t>(m_clientInputId),
GetInputString(input).c_str(),
GetCorrectionDataString(GetNetBindComponent()).c_str()
);
AZLOG(NET_Prediction, "Processed InputId=%d", aznumeric_cast<int32_t>(m_clientInputId));
// Generate a hash based on the current client predicted states
AzNetworking::HashSerializer hashSerializer;
GetNetBindComponent()->SerializeEntityCorrection(hashSerializer);
// In debug, send the entire client output state to the server to make it easier to debug desync issues
AzNetworking::PacketEncodingBuffer processInputResult;
#ifndef AZ_RELEASE_BUILD
if (cl_EnableDesyncDebugging)
{
AzNetworking::NetworkInputSerializer processInputResultSerializer(processInputResult.GetBuffer(), static_cast<uint32_t>(processInputResult.GetCapacity()));
GetNetBindComponent()->SerializeEntityCorrection(processInputResultSerializer);
processInputResult.Resize(processInputResultSerializer.GetSize());
}
#endif
// Save this input and discard move history outside our client rewind window
m_inputHistory.PushBack(input);
while (m_inputHistory.Size() > maxClientInputs)
@@ -552,10 +467,23 @@ namespace Multiplayer
inputArray[i] = m_inputHistory[historyIndex];
}
#ifndef AZ_RELEASE_BUILD
if (cl_EnableDesyncDebugging)
{
StateHistoryItem inputHistory = AZStd::make_unique<AzNetworking::StringifySerializer>();
while (m_predictiveStateHistory.size() > cl_PredictiveStateHistorySize)
{
m_predictiveStateHistory.erase(m_predictiveStateHistory.begin());
}
GetNetBindComponent()->SerializeEntityCorrection(*inputHistory);
m_predictiveStateHistory.emplace(m_clientInputId, AZStd::move(inputHistory));
}
#endif
// Send the input to server (only when we are not migrating)
if (!IsMigrating())
{
SendClientInput(inputArray, hashSerializer.GetHash(), processInputResult);
SendClientInput(inputArray, hashSerializer.GetHash());
}
}
}
@@ -563,7 +491,7 @@ namespace Multiplayer
void LocalPredictionPlayerInputComponentController::UpdateBankedTime(AZ::TimeMs deltaTimeMs)
{
const double deltaTime = static_cast<double>(deltaTimeMs) / 1000.0;
const double inputRate = static_cast<double>(static_cast<AZ::TimeMs>(cl_InputRateMs)) / 1000.0;
const double clientInputRateSec = static_cast<double>(static_cast<AZ::TimeMs>(cl_InputRateMs)) / 1000.0;
const double maxRewindHistory = static_cast<double>(static_cast<AZ::TimeMs>(cl_MaxRewindHistoryMs)) / 1000.0;
// Update banked time accumulator
@@ -577,18 +505,11 @@ namespace Multiplayer
NetworkInput& input = m_lastInputReceived[0];
{
ScopedAlterTime scopedTime(input.GetHostFrameId(), input.GetHostTimeMs(), DefaultBlendFactor, AzNetworking::InvalidConnectionId);
GetNetBindComponent()->ProcessInput(input, inputRate);
ScopedAlterTime scopedTime(input.GetHostFrameId(), input.GetHostTimeMs(), DefaultBlendFactor, GetNetBindComponent()->GetOwningConnectionId());
GetNetBindComponent()->ProcessInput(input, clientInputRateSec);
}
AZLOG
(
NET_Prediction,
"Forced InputId=%d - i=[%s] o=[%s]",
aznumeric_cast<int32_t>(input.GetClientInputId()),
GetInputString(input).c_str(),
GetCorrectionDataString(GetNetBindComponent()).c_str()
);
AZLOG(NET_Prediction, "Forced InputId=%d", aznumeric_cast<int32_t>(input.GetClientInputId()));
}
// Decay our bank time window, in case the remote endpoint has suffered a more persistent shift in latency, this should cause the client to eventually recover
@@ -267,10 +267,15 @@ namespace Multiplayer
return m_isProcessingInput;
}
bool NetBindComponent::IsReprocessingInput() const
{
return m_isReprocessingInput;
}
void NetBindComponent::CreateInput(NetworkInput& networkInput, float deltaTime)
{
// Only autonomous or authority runs this logic
AZ_Assert(m_netEntityRole == NetEntityRole::Autonomous || m_netEntityRole == NetEntityRole::Authority, "Incorrect network role for input creation");
// Only autonomous runs this logic
AZ_Assert(IsNetEntityRoleAutonomous(), "Incorrect network role for input creation");
for (MultiplayerComponent* multiplayerComponent : m_multiplayerInputComponentVector)
{
multiplayerComponent->GetController()->CreateInput(networkInput, deltaTime);
@@ -279,12 +284,21 @@ namespace Multiplayer
void NetBindComponent::ProcessInput(NetworkInput& networkInput, float deltaTime)
{
m_isProcessingInput = true;
// Only autonomous and authority runs this logic
AZ_Assert((NetworkRoleHasController(m_netEntityRole)), "Incorrect network role for input processing");
for (MultiplayerComponent* multiplayerComponent : m_multiplayerInputComponentVector)
{
multiplayerComponent->GetController()->ProcessInput(networkInput, deltaTime);
}
m_isProcessingInput = false;
}
void NetBindComponent::ReprocessInput(NetworkInput& networkInput, float deltaTime)
{
m_isReprocessingInput = true;
ProcessInput(networkInput, deltaTime);
m_isReprocessingInput = false;
}
bool NetBindComponent::HandleRpcMessage(AzNetworking::IConnection* invokingConnection, NetEntityRole remoteRole, NetworkEntityRpcMessage& message)
@@ -394,6 +408,11 @@ namespace Multiplayer
m_entityPreRenderEvent.Signal(deltaTime, blendFactor);
}
void NetBindComponent::NotifyCorrection()
{
m_entityCorrectionEvent.Signal();
}
void NetBindComponent::AddEntityStopEventHandler(EntityStopEvent::Handler& eventHandler)
{
eventHandler.Connect(m_entityStopEvent);
@@ -429,6 +448,11 @@ namespace Multiplayer
eventHandler.Connect(m_entityPreRenderEvent);
}
void NetBindComponent::AddEntityCorrectionEventHandler(EntityCorrectionEvent::Handler& eventHandler)
{
eventHandler.Connect(m_entityCorrectionEvent);
}
bool NetBindComponent::SerializeEntityCorrection(AzNetworking::ISerializer& serializer)
{
m_predictableRecord.ResetConsumedBits();
@@ -656,6 +680,7 @@ namespace Multiplayer
MultiplayerComponent* multiplayerComponent = azrtti_cast<MultiplayerComponent*>(component);
if (multiplayerComponent != nullptr)
{
multiplayerComponent->SetOwningConnectionId(m_owningConnectionId);
m_multiplayerInputComponentVector.push_back(multiplayerComponent);
}
}
@@ -31,6 +31,7 @@ namespace Multiplayer
, m_scaleEventHandler([this](float scale) { OnScaleChangedEvent(scale); })
, m_resetCountEventHandler([this](const uint8_t&) { OnResetCountChangedEvent(); })
, m_entityPreRenderEventHandler([this](float deltaTime, float blendFactor) { OnPreRender(deltaTime, blendFactor); })
, m_entityCorrectionEventHandler([this]() { OnCorrection(); })
{
;
}
@@ -47,6 +48,7 @@ namespace Multiplayer
ScaleAddEvent(m_scaleEventHandler);
ResetCountAddEvent(m_resetCountEventHandler);
GetNetBindComponent()->AddEntityPreRenderEventHandler(m_entityPreRenderEventHandler);
GetNetBindComponent()->AddEntityCorrectionEventHandler(m_entityCorrectionEventHandler);
// When coming into relevance, reset all blending factors so we don't interpolate to our start position
OnResetCountChangedEvent();
@@ -119,6 +121,18 @@ namespace Multiplayer
}
}
void NetworkTransformComponent::OnCorrection()
{
// Snap to latest
OnResetCountChangedEvent();
// Hard set the entities transform
if (!GetTransformComponent()->GetWorldTM().IsClose(m_targetTransform))
{
GetTransformComponent()->SetWorldTM(m_targetTransform);
}
}
NetworkTransformComponentController::NetworkTransformComponentController(NetworkTransformComponent& parent)
: NetworkTransformComponentControllerBase(parent)
@@ -554,7 +554,7 @@ namespace Multiplayer
m_tickFactor = 0.0f;
m_lastReplicatedHostTimeMs = packet.GetHostTimeMs();
m_lastReplicatedHostFrameId = packet.GetHostFrameId();
m_networkTime.AlterTime(m_lastReplicatedHostFrameId, m_lastReplicatedHostTimeMs, AzNetworking::InvalidConnectionId);
m_networkTime.ForceSetTime(m_lastReplicatedHostFrameId, m_lastReplicatedHostTimeMs);
}
for (AZStd::size_t i = 0; i < packet.GetEntityMessages().size(); ++i)
@@ -861,7 +861,7 @@ namespace Multiplayer
{
m_tickFactor += deltaTime / serverRateSeconds;
// Linear close to the origin, but asymptote at y = 1
m_renderBlendFactor = AZStd::clamp(1.0f - (std::pow(cl_renderTickBlendBase, m_tickFactor)), 0.0f, 1.0f);
m_renderBlendFactor = AZStd::clamp(1.0f - (std::pow(cl_renderTickBlendBase, m_tickFactor)), 0.0f, m_tickFactor);
AZLOG
(
NET_Blending,
@@ -524,6 +524,7 @@ namespace Multiplayer
bool EntityReplicationManager::HandlePropertyChangeMessage
(
AzNetworking::IConnection* invokingConnection,
EntityReplicator* entityReplicator,
AzNetworking::PacketId packetId,
NetEntityId netEntityId,
@@ -558,6 +559,12 @@ namespace Multiplayer
NetBindComponent* netBindComponent = replicatorEntity.GetNetBindComponent();
AZ_Assert(netBindComponent != nullptr, "No NetBindComponent");
if (createEntity)
{
// Always set our invoking connectionId for any newly created entities, since this connection now 'owns' them from a rewind perspective
netBindComponent->SetOwningConnectionId(invokingConnection->GetConnectionId());
}
const bool changeNetworkRole = (netBindComponent->GetNetEntityRole() != localNetworkRole);
if (changeNetworkRole)
{
@@ -744,7 +751,7 @@ namespace Multiplayer
bool EntityReplicationManager::HandleEntityUpdateMessage
(
[[maybe_unused]] AzNetworking::IConnection* invokingConnection,
AzNetworking::IConnection* invokingConnection,
const AzNetworking::IPacketHeader& packetHeader,
const NetworkEntityUpdateMessage& updateMessage
)
@@ -794,7 +801,7 @@ namespace Multiplayer
}
// This may implicitly create a replicator for us
bool handled = HandlePropertyChangeMessage(entityReplicator, packetHeader.GetPacketId(), updateMessage.GetEntityId(), updateMessage.GetNetworkRole(), outputSerializer, prefabEntityId);
bool handled = HandlePropertyChangeMessage(invokingConnection, entityReplicator, packetHeader.GetPacketId(), updateMessage.GetEntityId(), updateMessage.GetNetworkRole(), outputSerializer, prefabEntityId);
AZ_Assert(handled, "Failed to handle NetworkEntityUpdateMessage message");
return handled;
@@ -1121,7 +1128,7 @@ namespace Multiplayer
}
}
bool EntityReplicationManager::HandleEntityMigration([[maybe_unused]] AzNetworking::IConnection* invokingConnection, EntityMigrationMessage& message)
bool EntityReplicationManager::HandleEntityMigration(AzNetworking::IConnection* invokingConnection, EntityMigrationMessage& message)
{
EntityReplicator* replicator = GetEntityReplicator(message.m_entityId);
{
@@ -1130,6 +1137,7 @@ namespace Multiplayer
AzNetworking::TrackChangedSerializer<AzNetworking::NetworkOutputSerializer> outputSerializer(message.m_propertyUpdateData.GetBuffer(), static_cast<uint32_t>(message.m_propertyUpdateData.GetSize()));
if (!HandlePropertyChangeMessage
(
invokingConnection,
replicator,
AzNetworking::InvalidPacketId,
message.m_entityId,
@@ -136,6 +136,7 @@ namespace Multiplayer
bool HandlePropertyChangeMessage
(
AzNetworking::IConnection* invokingConnection,
EntityReplicator* entityReplicator,
AzNetworking::PacketId packetId,
NetEntityId netEntityId,
@@ -70,6 +70,15 @@ namespace Multiplayer
return (IsTimeRewound() && (rewindConnectionId == m_rewindingConnectionId)) ? m_unalteredFrameId : m_hostFrameId;
}
void NetworkTime::ForceSetTime(HostFrameId frameId, AZ::TimeMs timeMs)
{
AZ_Assert(!IsTimeRewound(), "Forcibly setting network time is unsupported under a rewound time scope");
m_unalteredFrameId = frameId;
m_hostFrameId = frameId;
m_hostTimeMs = timeMs;
m_rewindingConnectionId = AzNetworking::InvalidConnectionId;
}
void NetworkTime::AlterTime(HostFrameId frameId, AZ::TimeMs timeMs, AzNetworking::ConnectionId rewindConnectionId)
{
m_hostFrameId = frameId;
@@ -33,6 +33,7 @@ namespace Multiplayer
float GetHostBlendFactor() const override;
AzNetworking::ConnectionId GetRewindingConnectionId() const override;
HostFrameId GetHostFrameIdForRewindingConnection(AzNetworking::ConnectionId rewindConnectionId) const override;
void ForceSetTime(HostFrameId frameId, AZ::TimeMs timeMs) override;
void AlterTime(HostFrameId frameId, AZ::TimeMs timeMs, AzNetworking::ConnectionId rewindConnectionId) override;
void AlterBlendFactor(float blendFactor) override;
void SyncEntitiesToRewindState(const AZ::Aabb& rewindVolume) override;