A number of fixes to timeout and disconnect handling

Signed-off-by: kberg-amzn <karlberg@amazon.com>
This commit is contained in:
kberg-amzn
2021-11-03 14:17:20 -07:00
parent a4e0d69e83
commit 5b734b9d41
16 changed files with 104 additions and 279 deletions
@@ -50,7 +50,7 @@ namespace AZ
}
else
{
AZLOG_WARN("ScheduledEventHandle event pointer doesn't match to the pointer of handle to the event.");
//AZLOG_WARN("ScheduledEventHandle event pointer doesn't match to the pointer of handle to the event.");
}
}
return false; // Event has been deleted, so the handle class must be deleted after this function.
@@ -13,7 +13,9 @@
<Member Type="AzNetworking::DisconnectReason" Name="disconnectReason" Init="AzNetworking::DisconnectReason::None" />
</Packet>
<Packet Name="HeartbeatPacket" Desc="This packet is used to keep an established connection alive" />
<Packet Name="HeartbeatPacket" Desc="This packet is used to keep an established connection alive">
<Member Type="bool" Name="requestResponse" Init="false" />
</Packet>
<Packet Name="FragmentedPacket" Desc="This packet is used to segment a packet that exceeds a connections MTU">
<Member Type="AzNetworking::SequenceId" Name="unfragmentedSequence" Init="AzNetworking::InvalidSequenceId" />
@@ -27,13 +27,11 @@ namespace AzNetworking
ConnectionId connectionId,
const IpAddress& remoteAddress,
TcpNetworkInterface& networkInterface,
TcpSocket& socket,
TimeoutId timeoutId
TcpSocket& socket
)
: IConnection(connectionId, remoteAddress)
, m_networkInterface(networkInterface)
, m_socket(socket.CloneAndTakeOwnership())
, m_timeoutId(timeoutId)
, m_state(m_socket->IsOpen() ? ConnectionState::Connecting : ConnectionState::Disconnected)
, m_connectionRole(ConnectionRole::Acceptor)
, m_registeredSocketFd(InvalidSocketFd)
@@ -163,13 +161,6 @@ namespace AzNetworking
break;
}
TimeoutQueue::TimeoutItem* timeoutItem = m_networkInterface.m_connectionTimeoutQueue.RetrieveItem(GetTimeoutId());
if (timeoutItem == nullptr)
{
return true;
}
timeoutItem->UpdateTimeoutTime(startTimeMs);
NetworkOutputSerializer serializer(buffer.GetBuffer(), static_cast<uint32_t>(buffer.GetSize()));
if (m_state == ConnectionState::Connecting)
{
@@ -38,14 +38,12 @@ namespace AzNetworking
//! @param remoteAddress IP address of the remote endpoint
//! @param networkInterface TcpNetworkInterface that owns this connection instance
//! @param socket TCP socket to take ownership of and use for sending and receiving data
//! @param timeoutId timeout identifier of this connection instance
TcpConnection
(
ConnectionId connectionId,
const IpAddress& remoteAddress,
TcpNetworkInterface& networkInterface,
TcpSocket& socket,
TimeoutId timeoutId
TcpSocket& socket
);
//! Construct a new socket with optional encryption, used when initiating a new connection
@@ -69,14 +67,6 @@ namespace AzNetworking
//! @return the TcpSocket bound to this TcpConnection
TcpSocket* GetTcpSocket() const;
//! Sets the timeout identifier for this TcpConnection.
//! @param timeoutId the timeout identifier to use for this TcpConnection
void SetTimeoutId(TimeoutId timeoutId);
//! Returns the timeout identifier for this TcpConnection.
//! @return the timeout identifier for this TcpConnection
TimeoutId GetTimeoutId() const;
//! Returns true if this connection instance is in an open state, and is capable of actively sending and receiving packets.
//! @return boolean true if this connection instance is in an open state
bool IsOpen() const;
@@ -142,7 +132,6 @@ namespace AzNetworking
AZStd::unique_ptr<TcpSocket> m_socket;
AZStd::unique_ptr<ICompressor> m_compressor;
TimeoutId m_timeoutId;
PacketId m_lastSentPacketId = InvalidPacketId;
ConnectionState m_state = ConnectionState::Disconnected;
ConnectionRole m_connectionRole = ConnectionRole::Connector;
@@ -15,16 +15,6 @@ namespace AzNetworking
return m_socket.get();
}
inline void TcpConnection::SetTimeoutId(TimeoutId timeoutId)
{
m_timeoutId = timeoutId;
}
inline TimeoutId TcpConnection::GetTimeoutId() const
{
return m_timeoutId;
}
inline bool TcpConnection::IsOpen() const
{
return m_socket->IsOpen();
@@ -21,16 +21,11 @@ namespace AzNetworking
static const bool net_TcpUseEncryption = false;
#endif
AZ_CVAR(bool, net_TcpTimeoutConnections, true, nullptr, AZ::ConsoleFunctorFlags::DontReplicate, "Boolean value on whether we should timeout Tcp connections");
AZ_CVAR(AZ::TimeMs, net_TcpHeartbeatTimeMs, AZ::TimeMs{ 2 * 1000 }, nullptr, AZ::ConsoleFunctorFlags::Null, "Tcp connection heartbeat frequency");
AZ_CVAR(AZ::TimeMs, net_TcpDefaultTimeoutMs, AZ::TimeMs{ 10 * 1000 }, nullptr, AZ::ConsoleFunctorFlags::Null, "Time in milliseconds before we timeout an idle Tcp connection");
TcpNetworkInterface::TcpNetworkInterface(AZ::Name name, IConnectionListener& connectionListener, TrustZone trustZone, TcpListenThread& listenThread)
: m_name(name)
, m_trustZone(trustZone)
, m_connectionListener(connectionListener)
, m_listenThread(listenThread)
, m_timeoutMs(net_TcpDefaultTimeoutMs)
{
;
}
@@ -98,8 +93,6 @@ namespace AzNetworking
}
AZLOG_INFO("Adding new socket %d", static_cast<int32_t>(tcpSocket->GetSocketFd()));
const TimeoutId newTimeoutId = m_connectionTimeoutQueue.RegisterItem(static_cast<uint64_t>(tcpSocket->GetSocketFd()), net_TcpHeartbeatTimeMs);
connection->SetTimeoutId(newTimeoutId);
connection->SendReliablePacket(CorePackets::InitiateConnectionPacket());
m_connectionListener.OnConnect(connection.get());
m_connectionSet.AddConnection(AZStd::move(connection));
@@ -110,12 +103,6 @@ namespace AzNetworking
{
const AZ::TimeMs startTimeMs = AZ::GetElapsedTimeMs();
// Time out any stale connections
{
ConnectionTimeoutFunctor functor(*this);
m_connectionTimeoutQueue.UpdateTimeouts(functor);
}
AcceptNewConnections();
auto readCallback = [this, startTimeMs](SocketFd socketFd) { HandleConnectionRecv(socketFd, startTimeMs); };
@@ -258,8 +245,7 @@ namespace AzNetworking
return;
}
AZLOG(NET_TcpTraffic, "Adding new socket %d", static_cast<int32_t>(tcpSocket.GetSocketFd()));
const TimeoutId timeoutId = m_connectionTimeoutQueue.RegisterItem(static_cast<uint64_t>(tcpSocket.GetSocketFd()), m_timeoutMs);
AZStd::unique_ptr<TcpConnection> connection = AZStd::make_unique<TcpConnection>(connectionId, remoteAddress, *this, tcpSocket, timeoutId);
AZStd::unique_ptr<TcpConnection> connection = AZStd::make_unique<TcpConnection>(connectionId, remoteAddress, *this, tcpSocket);
AZ_Assert(connection->GetConnectionRole() == ConnectionRole::Acceptor, "Invalid role for connection");
GetConnectionListener().OnConnect(connection.get());
m_connectionSet.AddConnection(AZStd::move(connection));
@@ -286,7 +272,6 @@ namespace AzNetworking
m_pendingRemoves.resize_no_construct(0);
}
TcpNetworkInterface::PendingConnection::PendingConnection(SocketFd socketFd, uint32_t remoteIpAddress, uint16_t remotePort, uint16_t listenPort)
: m_socketFd(socketFd)
, m_remoteIpAddress(remoteIpAddress)
@@ -295,34 +280,4 @@ namespace AzNetworking
{
;
}
TcpNetworkInterface::ConnectionTimeoutFunctor::ConnectionTimeoutFunctor(TcpNetworkInterface& networkInterface)
: m_networkInterface(networkInterface)
{
;
}
TimeoutResult TcpNetworkInterface::ConnectionTimeoutFunctor::HandleTimeout(TimeoutQueue::TimeoutItem& item)
{
const SocketFd socketFd = static_cast<SocketFd>(item.m_userData);
TcpConnection* tcpConnection = m_networkInterface.m_connectionSet.GetConnection(socketFd);
if (tcpConnection == nullptr)
{
// We've already deleted this connection
return TimeoutResult::Delete;
}
if (tcpConnection->GetConnectionRole() == ConnectionRole::Connector)
{
tcpConnection->SendReliablePacket(CorePackets::HeartbeatPacket());
}
else if (net_TcpTimeoutConnections && (m_networkInterface.GetTimeoutMs() > AZ::TimeMs{ 0 }))
{
tcpConnection->Disconnect(DisconnectReason::Timeout, TerminationEndpoint::Local);
return TimeoutResult::Delete;
}
return TimeoutResult::Refresh;
}
}
@@ -137,16 +137,6 @@ namespace AzNetworking
AZ_DISABLE_COPY_MOVE(TcpNetworkInterface);
struct ConnectionTimeoutFunctor final
: public ITimeoutHandler
{
ConnectionTimeoutFunctor(TcpNetworkInterface& networkInterface);
TimeoutResult HandleTimeout(TimeoutQueue::TimeoutItem& item) override;
private:
AZ_DISABLE_COPY_MOVE(ConnectionTimeoutFunctor);
TcpNetworkInterface& m_networkInterface;
};
struct PendingRemove
{
SocketFd m_socketFd;
@@ -162,7 +152,6 @@ namespace AzNetworking
TcpSocketManager m_tcpSocketManager;
AZ::ThreadSafeDeque<PendingConnection> m_pendingConnections;
AZStd::vector<PendingRemove> m_pendingRemoves;
TimeoutQueue m_connectionTimeoutQueue;
TcpListenThread& m_listenThread;
friend class TcpConnection; // For access to private RequestDisconnect() method
@@ -79,7 +79,7 @@ namespace AzNetworking
AZLOG(NET_Acks, "Unacked packet count exceeded, sending client heartbeat (curr %u : max %u)", m_unackedPacketCount, static_cast<uint32_t>(net_UdpMaxUnackedPacketCount));
// This simply times out unreliable chunks that haven't completed within our timeout delay
m_fragmentQueue.Update();
SendUnreliablePacket(CorePackets::HeartbeatPacket());
SendUnreliablePacket(CorePackets::HeartbeatPacket(false));
}
}
@@ -289,7 +289,10 @@ namespace AzNetworking
{
return PacketDispatchResult::Failure;
}
// Do nothing, we've already processed our ack packets
if (packet.GetRequestResponse())
{
SendUnreliablePacket(CorePackets::HeartbeatPacket(false));
}
return PacketDispatchResult::Success;
}
break;
@@ -136,19 +136,19 @@ namespace AzNetworking
AZ_DISABLE_COPY_MOVE(UdpConnection);
UdpNetworkInterface& m_networkInterface;
UdpPacketTracker m_packetTracker;
UdpReliableQueue m_reliableQueue;
UdpFragmentQueue m_fragmentQueue;
ConnectionState m_state = ConnectionState::Disconnected;
ConnectionRole m_connectionRole = ConnectionRole::Connector;
DtlsEndpoint m_dtlsEndpoint;
UdpPacketTracker m_packetTracker;
UdpReliableQueue m_reliableQueue;
UdpFragmentQueue m_fragmentQueue;
ConnectionState m_state = ConnectionState::Disconnected;
ConnectionRole m_connectionRole = ConnectionRole::Connector;
DtlsEndpoint m_dtlsEndpoint;
AZ::TimeMs m_lastSentPacketMs;
uint32_t m_unackedPacketCount = 0;
uint32_t m_connectionMtu = MaxUdpTransmissionUnit;
TimeoutId m_timeoutId;
uint32_t m_timeoutCounter = 0;
int32_t m_timeoutCounter = 0;
};
}
@@ -31,7 +31,7 @@ namespace AzNetworking
AZ_CVAR(bool, net_UdpTimeoutConnections, true, nullptr, AZ::ConsoleFunctorFlags::DontReplicate, "Boolean value on whether we should timeout Udp connections");
AZ_CVAR(AZ::TimeMs, net_UdpPacketTimeSliceMs, AZ::TimeMs{ 8 }, nullptr, AZ::ConsoleFunctorFlags::DontReplicate, "The number of milliseconds to allow for packet processing");
AZ_CVAR(AZ::TimeMs, net_UdpHeartbeatTimeMs, AZ::TimeMs{ 2 * 1000 }, nullptr, AZ::ConsoleFunctorFlags::Null, "Udp connection heartbeat frequency");
AZ_CVAR(int32_t, net_UdpUnackedHeartbeats, 3, nullptr, AZ::ConsoleFunctorFlags::Null, "The number of heartbeats to attempt to send to keep a connection alive before giving up");
AZ_CVAR(AZ::TimeMs, net_UdpDefaultTimeoutMs, AZ::TimeMs{ 10 * 1000 }, nullptr, AZ::ConsoleFunctorFlags::Null, "Time in milliseconds before we timeout an idle Udp connection");
AZ_CVAR(AZ::TimeMs, net_MinPacketTimeoutMs, AZ::TimeMs{ 200 }, nullptr, AZ::ConsoleFunctorFlags::DontReplicate, "Minimum time to wait before timing out an unacked packet");
AZ_CVAR(int32_t, net_MaxTimeoutsPerFrame, 1000, nullptr, AZ::ConsoleFunctorFlags::DontReplicate, "Maximum number of packet timeouts to allow to process in a single frame");
@@ -139,7 +139,8 @@ namespace AzNetworking
}
const ConnectionId connectionId = m_connectionSet.GetNextConnectionId();
const TimeoutId timeoutId = m_connectionTimeoutQueue.RegisterItem(aznumeric_cast<uint64_t>(connectionId), m_timeoutMs);
const AZ::TimeMs timeoutTimeMs = m_timeoutMs / static_cast<AZ::TimeMs>(static_cast<int32_t>(net_UdpUnackedHeartbeats));
const TimeoutId timeoutId = m_connectionTimeoutQueue.RegisterItem(aznumeric_cast<uint64_t>(connectionId), timeoutTimeMs);
AZStd::unique_ptr<UdpConnection> connection = AZStd::make_unique<UdpConnection>(connectionId, remoteAddress, *this, ConnectionRole::Connector);
UdpPacketEncodingBuffer dtlsData;
@@ -277,6 +278,7 @@ namespace AzNetworking
}
timeoutItem->UpdateTimeoutTime(startTimeMs);
connection->m_timeoutCounter = 0;
PacketDispatchResult handledPacket = PacketDispatchResult::Failure;
if (header.GetPacketType() < aznumeric_cast<PacketType>(CorePackets::PacketType::MAX))
@@ -319,16 +321,10 @@ namespace AzNetworking
const AZ::TimeMs receiveTimeMs = AZ::GetElapsedTimeMs() - startTimeMs;
// Time out any stale client connections
{
ConnectionTimeoutFunctor functor(*this);
m_connectionTimeoutQueue.UpdateTimeouts(functor);
}
m_connectionTimeoutQueue.UpdateTimeouts([this](TimeoutQueue::TimeoutItem& item) { return HandleConnectionTimeout(item); });
// Time out any packets that haven't been acked within our timeout window
{
PacketTimeoutFunctor functor(*this);
m_packetTimeoutQueue.UpdateTimeouts(functor, static_cast<int32_t>(net_MaxTimeoutsPerFrame));
}
m_packetTimeoutQueue.UpdateTimeouts([this](TimeoutQueue::TimeoutItem& item) { return HandlePacketTimeout(item); }, static_cast<int32_t>(net_MaxTimeoutsPerFrame));
// Delete any connections we've disconnected
for (RemovedConnection& removedConnection : m_removedConnections)
@@ -709,21 +705,14 @@ namespace AzNetworking
{
// Packets involved in handshake are InitiateConnection, ConnectionHandshake and FragmentedPackets of ConnectionHandshake
return packetType == aznumeric_cast<PacketType>(CorePackets::PacketType::InitiateConnectionPacket) ||
packetType == aznumeric_cast<PacketType>(CorePackets::PacketType::ConnectionHandshakePacket) ||
(packetType == aznumeric_cast<PacketType>(CorePackets::PacketType::FragmentedPacket) && endpoint.IsConnecting());
packetType == aznumeric_cast<PacketType>(CorePackets::PacketType::ConnectionHandshakePacket) ||
(packetType == aznumeric_cast<PacketType>(CorePackets::PacketType::FragmentedPacket) && endpoint.IsConnecting());
}
UdpNetworkInterface::ConnectionTimeoutFunctor::ConnectionTimeoutFunctor(UdpNetworkInterface& networkInterface)
: m_networkInterface(networkInterface)
{
;
}
TimeoutResult UdpNetworkInterface::ConnectionTimeoutFunctor::HandleTimeout(TimeoutQueue::TimeoutItem& item)
TimeoutResult UdpNetworkInterface::HandleConnectionTimeout(TimeoutQueue::TimeoutItem& item)
{
const ConnectionId connectionId = ConnectionId(aznumeric_cast<uint32_t>(item.m_userData));
UdpConnection* udpConnection = static_cast<UdpConnection*>(m_networkInterface.m_connectionSet.GetConnection(connectionId));
UdpConnection* udpConnection = static_cast<UdpConnection*>(m_connectionSet.GetConnection(connectionId));
if (udpConnection == nullptr)
{
@@ -731,22 +720,23 @@ namespace AzNetworking
return TimeoutResult::Delete;
}
if (udpConnection->GetConnectionState() == ConnectionState::Connecting)
if ((udpConnection->GetConnectionState() == ConnectionState::Connecting)
&& udpConnection->GetDtlsEndpoint().IsConnecting())
{
if (udpConnection->GetDtlsEndpoint().IsConnecting())
{
// DTLS prefers we resend data lost over the wire with fresh SSL IDs so account for that here
UdpPacketEncodingBuffer dtlsData;
udpConnection->ProcessHandshakeData(dtlsData);
return TimeoutResult::Refresh;
}
// DTLS prefers we resend data lost over the wire with fresh SSL IDs so account for that here
UdpPacketEncodingBuffer dtlsData;
udpConnection->ProcessHandshakeData(dtlsData);
return TimeoutResult::Refresh;
}
if (udpConnection->GetConnectionRole() == ConnectionRole::Connector)
if ((udpConnection->GetConnectionRole() == ConnectionRole::Connector)
&& (udpConnection->m_timeoutCounter < net_UdpUnackedHeartbeats))
{
udpConnection->SendUnreliablePacket(CorePackets::HeartbeatPacket());
// Set the request response flag to true since we want a response to keep the connection alive
udpConnection->SendUnreliablePacket(CorePackets::HeartbeatPacket(true));
++udpConnection->m_timeoutCounter;
}
else if (net_UdpTimeoutConnections && (m_networkInterface.GetTimeoutMs() > AZ::TimeMs{ 0 }))
else if (net_UdpTimeoutConnections && (GetTimeoutMs() > AZ::TimeMs{ 0 }))
{
udpConnection->Disconnect(DisconnectReason::Timeout, TerminationEndpoint::Local);
return TimeoutResult::Delete;
@@ -755,19 +745,13 @@ namespace AzNetworking
return TimeoutResult::Refresh;
}
UdpNetworkInterface::PacketTimeoutFunctor::PacketTimeoutFunctor(UdpNetworkInterface& networkInterface)
: m_networkInterface(networkInterface)
{
;
}
TimeoutResult UdpNetworkInterface::PacketTimeoutFunctor::HandleTimeout(TimeoutQueue::TimeoutItem& item)
TimeoutResult UdpNetworkInterface::HandlePacketTimeout(TimeoutQueue::TimeoutItem& item)
{
ConnectionId connectionId;
PacketId packetId;
ReliabilityType reliability;
DecodeTimeoutId(item.m_userData, connectionId, packetId, reliability);
UdpConnection* connection = static_cast<UdpConnection*>(m_networkInterface.m_connectionSet.GetConnection(connectionId));
UdpConnection* connection = static_cast<UdpConnection*>(m_connectionSet.GetConnection(connectionId));
if (connection == nullptr)
{
@@ -782,16 +766,14 @@ namespace AzNetworking
case PacketTimeoutResult::Acked:
// Packet was already acked, just discard this timeout entry
return TimeoutResult::Delete;
case PacketTimeoutResult::Pending:
// Packet timed out before we received any info about it's sequence from the remote endpoint
// The connection latency may have increased, and our Rtt metrics may still be adjusting..
// Just throw it back into the timeout queue
return TimeoutResult::Refresh;
case PacketTimeoutResult::Lost:
// Packet timed out and was not acked, so we consider it lost
m_networkInterface.m_connectionListener.OnPacketLost(connection, packetId);
m_connectionListener.OnPacketLost(connection, packetId);
break;
}
@@ -149,34 +149,24 @@ namespace AzNetworking
//! @param endpoint whether the disconnection was initiated locally or remotely
void RequestDisconnect(UdpConnection* connection, DisconnectReason reason, TerminationEndpoint endpoint);
//! Internal helper to check if a packet's type is for connection handshake
//! Internal helper to check if a packet's type is for connection handshake.
//! @param endpoint DTLS endpoint participating in the handshake
//! @param packetType type of the packet
//! @return if the packet is for handshake
bool IsHandshakePacket(const DtlsEndpoint& endpoint, AzNetworking::PacketType packetType) const;
//! Internal helper to manage connection timeout behaviour.
//! @param item the timeout item corresponding to the timed out connection
//! @return whether to delete or persist the timeout item
TimeoutResult HandleConnectionTimeout(TimeoutQueue::TimeoutItem& item);
//! Internal helper to manage packet timeout behaviour.
//! @param item the timeout item corresponding to the timed out packet
//! @return whether to delete or persist the timeout item
TimeoutResult HandlePacketTimeout(TimeoutQueue::TimeoutItem& item);
AZ_DISABLE_COPY_MOVE(UdpNetworkInterface);
struct ConnectionTimeoutFunctor final
: public ITimeoutHandler
{
ConnectionTimeoutFunctor(UdpNetworkInterface& networkInterface);
TimeoutResult HandleTimeout(TimeoutQueue::TimeoutItem& item) override;
private:
AZ_DISABLE_COPY_MOVE(ConnectionTimeoutFunctor);
UdpNetworkInterface& m_networkInterface;
};
struct PacketTimeoutFunctor final
: public ITimeoutHandler
{
PacketTimeoutFunctor(UdpNetworkInterface& networkInterface);
TimeoutResult HandleTimeout(TimeoutQueue::TimeoutItem& item) override;
private:
AZ_DISABLE_COPY_MOVE(PacketTimeoutFunctor);
UdpNetworkInterface& m_networkInterface;
};
AZ::Name m_name;
TrustZone m_trustZone;
uint16_t m_port = 0;
@@ -1107,7 +1107,6 @@ namespace Multiplayer
void MultiplayerSystemComponent::OnAutonomousEntityReplicatorCreated()
{
m_autonomousEntityReplicatorCreatedHandler.Disconnect();
//m_networkEntityManager.GetNetworkEntityAuthorityTracker()->ResetTimeoutTime(AZ::TimeMs{ 2000 });
m_clientMigrationEndEvent.Signal();
}
@@ -155,7 +155,6 @@ namespace Multiplayer
AZ_CONSOLEFUNC(MultiplayerSystemComponent, DumpStats, AZ::ConsoleFunctorFlags::Null, "Dumps stats for the current multiplayer session");
AzNetworking::INetworkInterface* m_networkInterface = nullptr;
AzNetworking::INetworkInterface* m_networkEditorInterface = nullptr;
AZ::ConsoleCommandInvokedEvent::Handler m_consoleCommandHandler;
AZ::ThreadSafeDeque<AZStd::string> m_cvarCommands;
@@ -11,6 +11,7 @@
#include <Multiplayer/NetworkEntity/INetworkEntityManager.h>
#include <AzCore/Console/IConsole.h>
#include <AzCore/Console/ILogger.h>
#include <AzCore/EBus/IEventScheduler.h>
#include <AzNetworking/Utilities/NetworkCommon.h>
#include <AzNetworking/Serialization/NetworkOutputSerializer.h>
@@ -33,37 +34,21 @@ namespace Multiplayer
AZLOG
(
NET_AuthTracker,
"AuthTracker: Removing timeout for networkEntityId %llu from %s, new owner is %s",
"AuthTracker: Removing timeout for networkEntityId %llu, new owner is %s",
aznumeric_cast<AZ::u64>(entityHandle.GetNetEntityId()),
timeoutData->second.m_previousOwner.GetString().c_str(),
newOwner.GetString().c_str()
);
m_timeoutDataMap.erase(timeoutData);
ret = true;
}
auto iter = m_entityAuthorityMap.find(entityHandle.GetNetEntityId());
if (iter != m_entityAuthorityMap.end())
{
AZLOG
(
NET_AuthTracker,
"AuthTracker: Assigning networkEntityId %llu from %s to %s",
aznumeric_cast<AZ::u64>(entityHandle.GetNetEntityId()),
iter->second.back().GetString().c_str(),
newOwner.GetString().c_str()
);
}
else
{
AZLOG
(
NET_AuthTracker,
"AuthTracker: Assigning networkEntityId %llu to %s",
aznumeric_cast<AZ::u64>(entityHandle.GetNetEntityId()),
newOwner.GetString().c_str()
);
}
AZLOG
(
NET_AuthTracker,
"AuthTracker: Assigning networkEntityId %llu to %s",
aznumeric_cast<AZ::u64>(entityHandle.GetNetEntityId()),
newOwner.GetString().c_str()
);
m_entityAuthorityMap[entityHandle.GetNetEntityId()].push_back(newOwner);
return ret;
@@ -103,14 +88,41 @@ namespace Multiplayer
{
AZ_Assert
(
(m_timeoutDataMap.find(entityHandle.GetNetEntityId()) == m_timeoutDataMap.end()) ||
(m_timeoutDataMap[entityHandle.GetNetEntityId()].m_previousOwner == previousOwner),
m_timeoutDataMap.find(entityHandle.GetNetEntityId()) == m_timeoutDataMap.end(),
"Trying to add something twice to the timeout map, this is unexpected"
);
m_timeoutQueue.RegisterItem(aznumeric_cast<uint64_t>(entityHandle.GetNetEntityId()), net_EntityMigrationTimeoutMs);
TimeoutData& timeoutData = m_timeoutDataMap[entityHandle.GetNetEntityId()];
timeoutData.m_entityHandle = entityHandle;
timeoutData.m_previousOwner = previousOwner;
m_timeoutDataMap.insert(entityHandle.GetNetEntityId());
AZ::Interface<AZ::IEventScheduler>::Get()->AddCallback([this, netEntityId = entityHandle.GetNetEntityId(), previousOwner]
{
auto timeoutData = m_timeoutDataMap.find(netEntityId);
if (timeoutData != m_timeoutDataMap.end())
{
m_timeoutDataMap.erase(timeoutData);
ConstNetworkEntityHandle entityHandle = m_networkEntityManager.GetEntity(netEntityId);
if (auto entity = entityHandle.GetEntity())
{
NetEntityRole networkRole = NetEntityRole::InvalidRole;
NetBindComponent* netBindComponent = entityHandle.GetNetBindComponent();
if (netBindComponent != nullptr)
{
networkRole = netBindComponent->GetNetEntityRole();
}
if (networkRole != NetEntityRole::Authority)
{
AZLOG_ERROR
(
"Timed out entity id %llu during migration previous owner %s, removing it",
aznumeric_cast<AZ::u64>(entityHandle.GetNetEntityId()),
previousOwner.GetString().c_str()
);
m_networkEntityManager.MarkForRemoval(entityHandle);
}
}
}
},
AZ::Name("Entity authority removal functor"),
net_EntityMigrationTimeoutMs
);
}
else
{
@@ -127,18 +139,6 @@ namespace Multiplayer
}
HostId NetworkEntityAuthorityTracker::GetEntityAuthorityManager(ConstNetworkEntityHandle entityHandle) const
{
HostId hostId = GetEntityAuthorityManagerInternal(entityHandle);
AZ_Assert(hostId != InvalidHostId, "Unable to determine manager for entity");
return hostId;
}
bool NetworkEntityAuthorityTracker::DoesEntityHaveOwner(ConstNetworkEntityHandle entityHandle) const
{
return InvalidHostId != GetEntityAuthorityManagerInternal(entityHandle);
}
HostId NetworkEntityAuthorityTracker::GetEntityAuthorityManagerInternal(ConstNetworkEntityHandle entityHandle) const
{
if (auto localEnt = entityHandle.GetEntity())
{
@@ -167,52 +167,8 @@ namespace Multiplayer
return InvalidHostId;
}
NetworkEntityAuthorityTracker::TimeoutData::TimeoutData(ConstNetworkEntityHandle entityHandle, const HostId& previousOwner)
: m_entityHandle(entityHandle)
, m_previousOwner(previousOwner)
bool NetworkEntityAuthorityTracker::DoesEntityHaveOwner(ConstNetworkEntityHandle entityHandle) const
{
;
}
NetworkEntityAuthorityTracker::NetworkEntityTimeoutFunctor::NetworkEntityTimeoutFunctor
(
NetworkEntityAuthorityTracker& networkEntityAuthorityTracker,
INetworkEntityManager& networkEntityManager
)
: m_networkEntityAuthorityTracker(networkEntityAuthorityTracker)
, m_networkEntityManager(networkEntityManager)
{
;
}
AzNetworking::TimeoutResult NetworkEntityAuthorityTracker::NetworkEntityTimeoutFunctor::HandleTimeout(AzNetworking::TimeoutQueue::TimeoutItem& item)
{
const NetEntityId netEntityId = aznumeric_cast<NetEntityId>(item.m_userData);
auto timeoutData = m_networkEntityAuthorityTracker.m_timeoutDataMap.find(netEntityId);
if (timeoutData != m_networkEntityAuthorityTracker.m_timeoutDataMap.end())
{
m_networkEntityAuthorityTracker.m_timeoutDataMap.erase(timeoutData);
ConstNetworkEntityHandle entityHandle = m_networkEntityManager.GetEntity(netEntityId);
if (auto entity = entityHandle.GetEntity())
{
NetEntityRole networkRole = NetEntityRole::InvalidRole;
NetBindComponent* netBindComponent = entityHandle.GetNetBindComponent();
if (netBindComponent != nullptr)
{
networkRole = netBindComponent->GetNetEntityRole();
}
if (networkRole != NetEntityRole::Authority)
{
AZLOG_ERROR
(
"Timed out entity id %llu during migration previous owner %s, removing it",
aznumeric_cast<AZ::u64>(entityHandle.GetNetEntityId()),
timeoutData->second.m_previousOwner.GetString().c_str()
);
m_networkEntityManager.MarkForRemoval(entityHandle);
}
}
}
return AzNetworking::TimeoutResult::Delete;
return InvalidHostId != GetEntityAuthorityManager(entityHandle);
}
}
@@ -29,37 +29,13 @@ namespace Multiplayer
HostId GetEntityAuthorityManager(ConstNetworkEntityHandle entityHandle) const;
private:
HostId GetEntityAuthorityManagerInternal(ConstNetworkEntityHandle entityHandle) const;
NetworkEntityAuthorityTracker& operator= (const NetworkEntityAuthorityTracker&) = delete;
struct TimeoutData final
{
TimeoutData() = default;
TimeoutData(ConstNetworkEntityHandle entityHandle, const HostId& previousOwner);
ConstNetworkEntityHandle m_entityHandle;
HostId m_previousOwner = InvalidHostId;
};
struct NetworkEntityTimeoutFunctor final
: public AzNetworking::ITimeoutHandler
{
NetworkEntityTimeoutFunctor(NetworkEntityAuthorityTracker& networkEntityAuthorityTracker, INetworkEntityManager& m_networkEntityManager);
AzNetworking::TimeoutResult HandleTimeout(AzNetworking::TimeoutQueue::TimeoutItem& item) override;
private:
AZ_DISABLE_COPY_MOVE(NetworkEntityTimeoutFunctor);
NetworkEntityAuthorityTracker& m_networkEntityAuthorityTracker;
INetworkEntityManager& m_networkEntityManager;
};
using TimeoutDataMap = AZStd::unordered_map<NetEntityId, TimeoutData>;
using TimeoutDataMap = AZStd::unordered_set<NetEntityId>;
using EntityAuthorityMap = AZStd::unordered_map<NetEntityId, AZStd::vector<HostId>>;
TimeoutDataMap m_timeoutDataMap;
EntityAuthorityMap m_entityAuthorityMap;
INetworkEntityManager& m_networkEntityManager;
AzNetworking::TimeoutQueue m_timeoutQueue;
};
}
@@ -241,6 +241,10 @@ namespace Multiplayer
{
AZ::Entity* entity = it->second;
NetBindComponent* netBindComponent = m_networkEntityTracker.GetNetBindComponent(entity);
if (netBindComponent == nullptr)
{
continue;
}
AZ::Aabb entityBounds = AZ::Interface<AzFramework::IEntityBoundsUnion>::Get()->GetEntityWorldBoundsUnion(entity->GetId());
entityBounds.Expand(AZ::Vector3(0.01f));
if (netBindComponent->GetNetEntityRole() == NetEntityRole::Authority)