Integrating latest 47acbe8

Code/Framework/AzNetworking/AzNetworking/UdpTransport/DtlsEndpoint.cpp

@@ -12,6 +12,7 @@
 
 #include <AzNetworking/UdpTransport/DtlsEndpoint.h>
 #include <AzNetworking/UdpTransport/DtlsSocket.h>
+#include <AzNetworking/UdpTransport/UdpConnection.h>
 #include <AzNetworking/Utilities/EncryptionCommon.h>
 #include <AzNetworking/Utilities/NetworkIncludes.h>
 #include <AzCore/Console/IConsole.h>
@@ -57,14 +58,8 @@ namespace AzNetworking
         return result;
     }
 
-    DtlsEndpoint::ConnectResult DtlsEndpoint::Accept(const DtlsSocket& socket, const IpAddress& address, const UdpPacketEncodingBuffer& dtlsData)
+    DtlsEndpoint::ConnectResult DtlsEndpoint::Accept(const DtlsSocket& socket, const IpAddress& address)
     {
-        if (dtlsData.GetSize() <= 0)
-        {
-            AZLOG_WARN("Encryption is enabled on accepting endpoint, but connector provided an empty DTLS handshake blob.  Check that encryption is properly disabled on *BOTH* endpoints");
-            return DtlsEndpoint::ConnectResult::Failed;
-        }
-
         const ConnectResult result = ConstructEndpointInternal(socket, address);
 #if AZ_TRAIT_USE_OPENSSL
         if (result != ConnectResult::Failed)
@@ -72,10 +67,7 @@ namespace AzNetworking
             // This SSL should be configured to accept connections
             SSL_set_accept_state(m_sslSocket);
             m_state = HandshakeState::Accepting;
-            const uint8_t* encryptedData = dtlsData.GetBuffer();
-            const uint32_t encryptedSize = dtlsData.GetSize();
-            BIO_write(m_readBio, encryptedData, encryptedSize);
-            return CompleteHandshake(socket);
+            return ConnectResult::Pending;
         }
 #endif
         return result;
@@ -88,49 +80,53 @@ namespace AzNetworking
              || (m_state == HandshakeState::Failed)); // In all cases caller should call CompleteHandshake() next and check the return value
     }
 
-    DtlsEndpoint::ConnectResult DtlsEndpoint::CompleteHandshake(const UdpSocket& socket)
+    DtlsEndpoint::ConnectResult DtlsEndpoint::ProcessHandshakeData([[maybe_unused]] UdpConnection& connection, [[maybe_unused]] const UdpPacketEncodingBuffer& dtlsData)
     {
-        UdpPacketEncodingBuffer responseData;
-        const ConnectResult result = PerformHandshakeInternal(responseData);
-        if ((result != ConnectResult::Failed) && (responseData.GetSize() > 0))
+        ConnectResult result = ConnectResult::Failed;
+#if AZ_TRAIT_USE_OPENSSL
+        UdpPacketEncodingBuffer outDtlsData;
+        if (dtlsData.GetSize() > 0)
         {
-            struct sockaddr_in dest;
-            memset(&dest, 0, sizeof(dest));
-            dest.sin_family = AF_INET;
-            dest.sin_addr.s_addr = m_address.GetAddress(ByteOrder::Network);
-            dest.sin_port = m_address.GetPort(ByteOrder::Network);
-            sendto(static_cast<int32_t>(socket.GetSocketFd()), reinterpret_cast<char*>(responseData.GetBuffer()), responseData.GetSize(), 0, (sockaddr*)&dest, sizeof(dest));
-            AZLOG(NET_DebugDtls, "Replying to DTLS handshake datagram, %u bytes", static_cast<int32_t>(responseData.GetSize()));
+            const uint8_t* encryptedData = dtlsData.GetBuffer();
+            const uint32_t encryptedSize = dtlsData.GetSize();
+            BIO_write(m_readBio, encryptedData, encryptedSize);
         }
+        DtlsEndpoint::HandshakeState prevState = m_state;
+        result = PerformHandshakeInternal(outDtlsData);
+
+        // Pass along any remaining handshake data
+        // If we're the connecting endpoint and the handshake is complete, both sides are encrypted and this isn't necessary so skip it
+        bool continueHandshake = prevState != DtlsEndpoint::HandshakeState::Connecting || m_state != DtlsEndpoint::HandshakeState::Complete;
+        if (outDtlsData.GetSize() > 0 && continueHandshake)
+        {
+            CorePackets::ConnectionHandshakePacket handshakePacket = CorePackets::ConnectionHandshakePacket();
+            handshakePacket.SetHandshakeBuffer(outDtlsData);
+            // SSL prefers we handle resend by reobtaining data through SSL so we use unreliable and resend on timeout
+            connection.SendUnreliablePacket(handshakePacket);
+        }
+#endif
 
         return result;
     }
 
-    const uint8_t* DtlsEndpoint::DecodePacket
-    (
-        [[maybe_unused]] const UdpSocket& socket,
-        [[maybe_unused]] const uint8_t* encryptedData,
-        [[maybe_unused]] int32_t encryptedSize,
-        [[maybe_unused]] uint8_t* outDecodedData,
-        [[maybe_unused]] int32_t& outDecodedSize
-    )
+    const uint8_t* DtlsEndpoint::DecodePacket([[maybe_unused]] UdpConnection& connection, const uint8_t* encryptedData, int32_t encryptedSize, uint8_t* outDecodedData, int32_t& outDecodedSize)
     {
-        if (m_sslSocket == nullptr)
+        if (m_sslSocket == nullptr || IsConnecting())
         {
             // If the ssl socket is nullptr, it means encryption is not enabled, just passthrough the received data
+            // If the socket is connecting/handshaking, it means we can't yet encrypt, just passthrough the received data
             outDecodedSize = encryptedSize;
             return encryptedData;
         }
 #if AZ_TRAIT_USE_OPENSSL
-        BIO_write(m_readBio, encryptedData, encryptedSize);
-        if (IsConnecting())
-        {
-            CompleteHandshake(socket);
-            outDecodedSize = 0;
-        }
-        // CompleteHandshake() above may have failed and destroyed the SSL context, check here that state is valid so we don't crash on SSL_read
+
+        int32_t bioWriteSize = BIO_write(m_readBio, encryptedData, encryptedSize);
         if (m_state != HandshakeState::Failed)
         {
+            if (bioWriteSize != encryptedSize)
+            {
+                AZLOG_ERROR("BIO did not write as many bytes as provided");
+            }
             outDecodedSize = SSL_read(m_sslSocket, outDecodedData, encryptedSize);
         }
 #endif
@@ -175,13 +171,6 @@ namespace AzNetworking
 
         ConnectResult connectResult = ConnectResult::Pending;
 #if AZ_TRAIT_USE_OPENSSL
-        if (SSL_is_init_finished(m_sslSocket))
-        {
-            const char* stateString = GetEnumString(m_state);
-            AZLOG(NET_DebugDtls, "dtls handshake is completed, unblocking connection for game traffic, prior state: %s", stateString);
-            m_state = HandshakeState::Complete;
-            connectResult = ConnectResult::Complete;
-        }
         ERR_clear_error();
         const int32_t result = SSL_do_handshake(m_sslSocket);
         if (result <= 0)
@@ -200,6 +189,14 @@ namespace AzNetworking
             }
         }
 
+        if (SSL_is_init_finished(m_sslSocket))
+        {
+            const char* stateString = GetEnumString(m_state);
+            AZLOG(NET_DebugDtls, "dtls handshake is completed, unblocking connection for game traffic, prior state: %s", stateString);
+            m_state = HandshakeState::Complete;
+            connectResult = ConnectResult::Complete;
+        }
+
         // Need to do this... connection negotiation may have left data in the write bio that we need to send out
         if (BIO_ctrl_pending(m_writeBio) > 0)
         {

Code/Framework/AzNetworking/AzNetworking/UdpTransport/DtlsEndpoint.h

@@ -22,6 +22,7 @@ typedef struct bio_st BIO;
 
 namespace AzNetworking
 {
+    class UdpConnection;
     class UdpSocket;
     class DtlsSocket;
 
@@ -61,30 +62,30 @@ namespace AzNetworking
         ConnectResult Connect(const DtlsSocket& socket, const IpAddress& address, UdpPacketEncodingBuffer& outDtlsData);
 
         //! Accepts a connection from the remote encrypted endpoint.
-        //! @param socket   the dtls socket being used for data transmission
-        //! @param address  the address of the remote endpoint connecting to us
-        //! @param dtlsData data buffer containing the initial dtls handshake packet
+        //! @param socket      the dtls socket being used for data transmission
+        //! @param address     the address of the remote endpoint connecting to us
         //! @return a connect result specifying whether the connection is still pending, failed, or complete
-        ConnectResult Accept(const DtlsSocket& socket, const IpAddress& address, const UdpPacketEncodingBuffer& dtlsData);
+        ConnectResult Accept(const DtlsSocket& socket, const IpAddress& address);
 
         //! Returns whether or not the endpoint is still negotiating the dtls handshake.
         //! @return true if the endpoint is still in a connecting state
         bool IsConnecting() const;
 
         //! Attempts to complete the dtls handshake and establish an encrypted connection.
-        //! @param socket the dtls socket being used for data transmission
+        //! @param connection the UDP connection being used for data transmission
+        //! @param dtlsData    data buffer containing dtls handshake packet
         //! @return a connect result specifying whether the connection is still pending, failed, or complete
-        ConnectResult CompleteHandshake(const UdpSocket& socket);
+        ConnectResult ProcessHandshakeData(UdpConnection& connection, const UdpPacketEncodingBuffer& dtlsData);
 
         //! If the endpoint has encryption enabled, this will decrypt the transmitted data and return the result.
         //! @note sizes have to be signed since OpenSSL often returns negative values to represent error results
-        //! @param socket         the DTLS socket being used for data transmission
+        //! @param connection     the UDP connection being used for data transmission
         //! @param encryptedData  the potentially encrypted data received from the socket
         //! @param encryptedSize  the size of the received raw data
         //! @param outDecodedData an appropriately sized output buffer to store decrypted data
         //! @param outDecodedSize the size of the output buffer
         //! @return pointer to the decoded data
-        const uint8_t* DecodePacket(const UdpSocket& socket, const uint8_t* encryptedData, int32_t encryptedSize, uint8_t* outDecodedData, int32_t& outDecodedSize);
+        const uint8_t* DecodePacket(UdpConnection& connection, const uint8_t* encryptedData, int32_t encryptedSize, uint8_t* outDecodedData, int32_t& outDecodedSize);
 
     private:
 

Code/Framework/AzNetworking/AzNetworking/UdpTransport/DtlsSocket.cpp

@@ -10,6 +10,7 @@
 *
 */
 
+#include <AzNetworking/Framework/INetworkInterface.h>
 #include <AzNetworking/UdpTransport/DtlsSocket.h>
 #include <AzCore/Console/ILogger.h>
 
@@ -35,9 +36,9 @@ namespace AzNetworking
         return dtlsEndpoint.Connect(*this, address, outDtlsData);
     }
 
-    DtlsEndpoint::ConnectResult DtlsSocket::AcceptDtlsEndpoint(DtlsEndpoint& dtlsEndpoint, const IpAddress& address, const UdpPacketEncodingBuffer& dtlsData) const
+    DtlsEndpoint::ConnectResult DtlsSocket::AcceptDtlsEndpoint(DtlsEndpoint& dtlsEndpoint, const IpAddress& address) const
     {
-        return dtlsEndpoint.Accept(*this, address, dtlsData);
+        return dtlsEndpoint.Accept(*this, address);
     }
 
     bool DtlsSocket::Open(uint16_t port, CanAcceptConnections canAccept, TrustZone trustZone)
@@ -91,6 +92,11 @@ namespace AzNetworking
         // Write out the packet we were requested to send
         const int32_t sentBytesRaw = SSL_write(dtlsEndpoint.m_sslSocket, data, size);
         const int32_t sentBytesEnc = BIO_read(dtlsEndpoint.m_writeBio, encrpytedSendBuffer, sizeof(encrpytedSendBuffer));
+
+        // Track encryption metrics
+        m_sentBytesEncryptionInflation += aznumeric_cast<uint32_t>(sentBytesEnc - aznumeric_cast<int32_t>(size));
+        m_sentPacketsEncrypted++;
+
         return UdpSocket::SendInternal(address, encrpytedSendBuffer, sentBytesEnc, encrypt, dtlsEndpoint);
 #else
         return 0;

Code/Framework/AzNetworking/AzNetworking/UdpTransport/DtlsSocket.h

@@ -46,9 +46,8 @@ namespace AzNetworking
         //! Accepts an encryption socket wrapper.
         //! @param dtlsEndpoint the encryption wrapper instance to create a connection over
         //! @param address      the IP address of the endpoint to connect to
-        //! @param dtlsData     data buffer containing the dtls handshake packet
         //! @return a connect result specifying whether the connection is still pending, failed, or complete
-        DtlsEndpoint::ConnectResult AcceptDtlsEndpoint(DtlsEndpoint& dtlsEndpoint, const IpAddress& address, const UdpPacketEncodingBuffer& dtlsData) const override;
+        DtlsEndpoint::ConnectResult AcceptDtlsEndpoint(DtlsEndpoint& dtlsEndpoint, const IpAddress& address) const override;
 
         //! Opens the UDP socket on the given port.
         //! @param port      the port number to open the UDP socket on, 0 will bind to any available port

Code/Framework/AzNetworking/AzNetworking/UdpTransport/UdpConnection.cpp

@@ -66,9 +66,9 @@ namespace AzNetworking
         }
     }
 
-    DtlsEndpoint::ConnectResult UdpConnection::CompleteHandshake()
+    DtlsEndpoint::ConnectResult UdpConnection::ProcessHandshakeData(const UdpPacketEncodingBuffer& dtlsData)
     {
-        const DtlsEndpoint::ConnectResult result = m_dtlsEndpoint.CompleteHandshake(*(m_networkInterface.m_socket));
+        const DtlsEndpoint::ConnectResult result = m_dtlsEndpoint.ProcessHandshakeData(*this, dtlsData);
         if (result == DtlsEndpoint::ConnectResult::Failed)
         {
             Disconnect(DisconnectReason::NetworkError, TerminationEndpoint::Local);
@@ -249,6 +249,27 @@ namespace AzNetworking
         }
         break;
 
+        case CorePackets::PacketType::ConnectionHandshakePacket:
+        {
+            AZLOG(NET_CorePackets, "Received core packet %s", "ConnectionHandshakePacket");
+            CorePackets::ConnectionHandshakePacket packet;
+            if (!serializer.Serialize(packet, "Packet"))
+            {
+                return false;
+            }
+
+            if (m_state != ConnectionState::Connected)
+            {
+                if (ProcessHandshakeData(packet.GetHandshakeBuffer()) == DtlsEndpoint::ConnectResult::Complete)
+                {
+                    m_state = ConnectionState::Connected;
+                }
+            }
+
+            return true;
+        }
+        break;
+
         case CorePackets::PacketType::TerminateConnectionPacket:
         {
             AZLOG(NET_CorePackets, "Received core packet %s", "TerminateConnection");

Code/Framework/AzNetworking/AzNetworking/UdpTransport/UdpConnection.h

@@ -38,6 +38,7 @@ namespace AzNetworking
     class UdpConnection
     :   public IConnection
     {
+        friend class UdpFragmentQueue;
         friend class UdpNetworkInterface;
 
     public:
@@ -50,9 +51,10 @@ namespace AzNetworking
         UdpConnection(ConnectionId connectionId, const IpAddress& remoteAddress, UdpNetworkInterface& networkInterface, ConnectionRole connectionRole);
         ~UdpConnection() override;
 
-        //! Helper to complete dtls handshake logic on a newly established connection
+        //! Helper to exchange dtls handshake data during connection handshake
+        //! @param dtlsData    data buffer containing dtls handshake packet
         //! @return the current result code for the dtls handshake operation, failed, pending, or complete
-        DtlsEndpoint::ConnectResult CompleteHandshake();
+        DtlsEndpoint::ConnectResult ProcessHandshakeData(const UdpPacketEncodingBuffer& dtlsData);
 
         //! Updates the connection heartbeat if active.
         //! @param currentTimeMs current wall clock time in milliseconds

Code/Framework/AzNetworking/AzNetworking/UdpTransport/UdpFragmentQueue.cpp

@@ -155,7 +155,17 @@ namespace AzNetworking
             }
         }
         connection->GetPacketTracker().ProcessReceived(connection, header);
-        return connectionListener.OnPacketReceived(connection, header, networkSerializer);
+        bool handledPacket = false;
+        if (header.GetPacketType() < aznumeric_cast<PacketType>(CorePackets::PacketType::MAX))
+        {
+            handledPacket = connection->HandleCorePacket(connectionListener, header, networkSerializer);
+        }
+        else
+        {
+            handledPacket = connectionListener.OnPacketReceived(connection, header, networkSerializer);
+        }
+
+        return handledPacket;
     }
 
     TimeoutResult UdpFragmentQueue::HandleTimeout(TimeoutQueue::TimeoutItem& item)

Code/Framework/AzNetworking/AzNetworking/UdpTransport/UdpFragmentQueue.h

@@ -31,6 +31,7 @@ namespace AzNetworking
     class UdpFragmentQueue
         : public ITimeoutHandler
     {
+
     public:
 
         //! Updates the UdpFragmentQueue timeout queue.

Code/Framework/AzNetworking/AzNetworking/UdpTransport/UdpNetworkInterface.cpp

@@ -10,6 +10,7 @@
 *
 */
 
+#include <AzNetworking/Framework/INetworking.h>
 #include <AzNetworking/UdpTransport/UdpNetworkInterface.h>
 #include <AzNetworking/UdpTransport/UdpConnection.h>
 #include <AzNetworking/UdpTransport/DtlsSocket.h>
@@ -17,7 +18,6 @@
 #include <AzNetworking/Serialization/NetworkInputSerializer.h>
 #include <AzNetworking/Serialization/NetworkOutputSerializer.h>
 #include <AzNetworking/Framework/ICompressor.h>
-#include <AzNetworking/Utilities/CompressionCommon.h>
 #include <AzNetworking/Utilities/NetworkCommon.h>
 #include <AzCore/Console/IConsole.h>
 #include <AzCore/Console/ILogger.h>
@@ -26,8 +26,11 @@ namespace AzNetworking
 {
 #if AZ_TRAIT_USE_OPENSSL
     AZ_CVAR(bool, net_UdpUseEncryption, false, nullptr, AZ::ConsoleFunctorFlags::DontReplicate, "Enable encryption on Udp based connections");
+    AZ_CVAR(uint32_t, net_SslInflationOverhead, 32, nullptr, AZ::ConsoleFunctorFlags::DontReplicate, "A SSL fudge overhead value to take out of fragmented packet payloads");
+
 #else
     static const bool net_UdpUseEncryption = false;
+    static const uint32_t net_SslInflationOverhead = 0;
 #endif
 
     AZ_CVAR(bool, net_UdpTimeoutConnections, true, nullptr, AZ::ConsoleFunctorFlags::DontReplicate, "Boolean value on whether we should timeout Udp connections");
@@ -64,8 +67,8 @@ namespace AzNetworking
         , m_readerThread(readerThread)
     {
         const AZ::CVarFixedString compressor = static_cast<AZ::CVarFixedString>(net_UdpCompressor);
-        const char* compressorName = compressor.c_str();
-        m_compressor = CreateCompressor(compressorName);
+        const AZ::Name compressorName = AZ::Name(compressor);
+        m_compressor = AZ::Interface<INetworking>::Get()->CreateCompressor(compressorName);
     }
 
     UdpNetworkInterface::~UdpNetworkInterface()
@@ -122,7 +125,7 @@ namespace AzNetworking
     {
         if (!m_socket->IsOpen())
         {
-            m_socket->Open(m_port, UdpSocket::CanAcceptConnections::True, m_trustZone);
+            m_socket->Open(m_port, UdpSocket::CanAcceptConnections::False, m_trustZone);
             m_readerThread.RegisterSocket(m_socket.get());
         }
 
@@ -137,7 +140,12 @@ namespace AzNetworking
         connection->m_state = ConnectionState::Connecting;
         connection->SetConnectionMtu(MaxUdpTransmissionUnit);
         connection->SetTimeoutId(timeoutId);
-        connection->SendReliablePacket(CorePackets::InitiateConnectionPacket());
+
+        // Signal the connection attempt
+        CorePackets::InitiateConnectionPacket connectPacket = CorePackets::InitiateConnectionPacket();
+        connectPacket.SetHandshakeBuffer(dtlsData);
+        connection->SendReliablePacket(connectPacket);
+
         m_connectionListener.OnConnect(connection.get());
         m_connectionSet.AddConnection(AZStd::move(connection));
         return connectionId;
@@ -150,10 +158,6 @@ namespace AzNetworking
             return;
         }
 
-#ifdef ENABLE_LATENCY_DEBUG
-        m_socket->ProcessDeferredPackets();
-#endif
-
         const AZ::TimeMs startTimeMs = AZ::GetElapsedTimeMs();
         const UdpReaderThread::ReceivedPackets* packets = m_readerThread.GetReceivedPackets(m_socket.get());
         if (packets == nullptr)
@@ -191,7 +195,7 @@ namespace AzNetworking
 
             int32_t decodedPacketSize = 0;
             m_decryptBuffer.Resize(m_decryptBuffer.GetCapacity());
-            const uint8_t* decodedPacketData = connection->GetDtlsEndpoint().DecodePacket(*m_socket, packet.m_buffer, packet.m_receivedBytes, m_decryptBuffer.GetBuffer(), decodedPacketSize);
+            const uint8_t* decodedPacketData = connection->GetDtlsEndpoint().DecodePacket(*connection, packet.m_buffer, packet.m_receivedBytes, m_decryptBuffer.GetBuffer(), decodedPacketSize);
             m_decryptBuffer.Resize(decodedPacketSize);
 
             if (decodedPacketSize == 0)
@@ -201,8 +205,7 @@ namespace AzNetworking
             }
             else if (decodedPacketSize < 0)
             {
-                // Something bad happened on the SSL read and we're now invalid, we should disconnect
-                connection->Disconnect(DisconnectReason::SslFailure, TerminationEndpoint::Local);
+                // Late unencrypted handshake packets or just random garbage can show up, discard and continue
                 continue;
             }
 
@@ -233,8 +236,8 @@ namespace AzNetworking
                 }
                 decodedPacketData = m_decompressBuffer.GetBuffer();
                 decodedPacketSize = m_decompressBuffer.GetSize();
-                GetMetrics().m_recvBytesUncompressed += decodedPacketSize;
             }
+            GetMetrics().m_recvBytesUncompressed += decodedPacketSize;
 
             TimeoutQueue::TimeoutItem* timeoutItem = m_connectionTimeoutQueue.RetrieveItem(connection->GetTimeoutId());
             if (timeoutItem == nullptr)
@@ -273,11 +276,19 @@ namespace AzNetworking
                 if (handledPacket)
                 {
                     connection->UpdateHeartbeat(currentTimeMs);
-                    if (connection->GetConnectionState() == ConnectionState::Connecting)
+                    if (connection->GetConnectionState() == ConnectionState::Connecting && !connection->GetDtlsEndpoint().IsConnecting())
                     {
+                        // Connection is realized once a packet is received and socket handshake is verified complete
                         connection->m_state = ConnectionState::Connected;
                     }
                 }
+                else if (m_socket->IsEncrypted() && connection->GetDtlsEndpoint().IsConnecting() &&
+                    !IsHandshakePacket(connection->GetDtlsEndpoint(), header.GetPacketType()))
+                {
+                    // It's possible for one side to finish its half of the handshake and start sending encrypted data
+                    // If it's not an expected unencrypted type then skip it for now
+                    continue;
+                }
                 else if (connection->GetConnectionState() != ConnectionState::Disconnecting)
                 {
                     connection->Disconnect(DisconnectReason::StreamError, TerminationEndpoint::Local);
@@ -309,6 +320,8 @@ namespace AzNetworking
         // Update metrics
         GetMetrics().m_sendPackets = m_socket->GetSentPackets();
         GetMetrics().m_sendBytes = m_socket->GetSentBytes();
+        GetMetrics().m_sendPacketsEncrypted = m_socket->GetSentPacketsEncrypted();
+        GetMetrics().m_sendBytesEncryptionInflation = m_socket->GetSentBytesEncryptionInflation();
         GetMetrics().m_recvTimeMs += receiveTimeMs;
         GetMetrics().m_recvPackets = m_socket->GetRecvPackets();
         GetMetrics().m_recvBytes = m_socket->GetRecvBytes();
@@ -410,8 +423,7 @@ namespace AzNetworking
 
         // The ordering inside this function is incredibly important and fragile
         const IpAddress& address = connection.GetRemoteAddress();
-        // We don't want to compress the initial InitiateConnectionPacket
-        // We can use this to transmit compression and encryption details in the future
+        // We don't want to compress the initial InitiateConnectionPacket, ConnectionHandshakePackets or FragmentedPackets of those two
         const bool shouldCompress = packet.GetPacketType() != aznumeric_cast<PacketType>(CorePackets::PacketType::InitiateConnectionPacket);
 
         if (address.GetAddress(ByteOrder::Host) == 0)
@@ -427,6 +439,29 @@ namespace AzNetworking
         UdpPacketHeader header(connection.GetPacketTracker(), packet.GetPacketType(), reliableSequence);
         const PacketId localPacketId = header.GetPacketId();
 
+        // If it's a reliable packet, make sure our reliable queue knows about it now because we might need to drop it if our connection is
+        // not set up
+        if (reliabilityType == ReliabilityType::Reliable)
+        {
+            if (!connection.PrepareReliablePacketForSend(localPacketId, reliableSequence, packet))
+            {
+                connection.Disconnect(DisconnectReason::ReliableQueueFull, TerminationEndpoint::Local);
+            }
+        }
+
+        // If we're still connecting, only transmit packets related to establishing connection and queue the rest for later
+        // This implicitly enforces that the only FragmentedPackets sent here are of ConnectionHandshakePacket
+        // Other large packets are simply queued before they are fragmented
+        if (connection.GetDtlsEndpoint().IsConnecting() && !IsHandshakePacket(connection.GetDtlsEndpoint(), packet.GetPacketType()))
+        {
+            // IMPORTANT that we register with the timeout queue here, otherwise we don't have the timer to pop for reliable packets
+            RegisterWithTimeoutQueue(connection.GetConnectionId(), localPacketId, reliabilityType, connection.GetMetrics());
+            AZLOG(
+                NET_DebugDtls, "Connection is still in handshake negotiation, blocking packet send for packet type %d",
+                (int)packet.GetPacketType());
+            return localPacketId;
+        }
+
         UdpPacketEncodingBuffer buffer;
         {
             buffer.Resize(buffer.GetCapacity());
@@ -457,11 +492,12 @@ namespace AzNetworking
         uint32_t packetSize = buffer.GetSize();
         uint8_t* packetData = buffer.GetBuffer();
 
-        // If the packet doesn't fit within our MTU, break it up
-        if (packetSize > connection.GetConnectionMtu())
+        // If the packet doesn't fit within our MTU (minus potential SSL encryption overhead), break it up
+        if (packetSize > connection.GetConnectionMtu() - net_SslInflationOverhead)
         {
             // Each fragmented packet we send adds an extra fragmented packet header, need to deduct that from our chunk size, otherwise we infinitely loop
-            const uint32_t chunkSize = connection.GetConnectionMtu() - net_FragmentedHeaderOverhead;
+            // SSL encryption can also inflate our payload so we pre-emptively deduct an estimated tax
+            const uint32_t chunkSize = connection.GetConnectionMtu() - net_FragmentedHeaderOverhead - net_SslInflationOverhead;
             const uint32_t numChunks = (packetSize + chunkSize - 1) / chunkSize; // We want to round up on the remainder
             const uint8_t* chunkStart = packetData;
             const SequenceId fragmentedSequence = connection.m_fragmentQueue.GetNextFragmentedSequenceId();
@@ -529,30 +565,9 @@ namespace AzNetworking
             aznumeric_cast<uint32_t>(header.GetSequenceWindow())
         );
 
-        // If it's a reliable packet, make sure our reliable queue knows about it now because we might need to drop it if our connection is not set up
-        if (reliabilityType == ReliabilityType::Reliable)
-        {
-            if (!connection.PrepareReliablePacketForSend(localPacketId, reliableSequence, packet))
-            {
-                connection.Disconnect(DisconnectReason::ReliableQueueFull, TerminationEndpoint::Local);
-            }
-        }
-
-        // Okay, if we're still connecting, and the packet we're trying to send is not a retransmitted initiate connection packet, return an error, don't send yet
-        if (connection.GetDtlsEndpoint().IsConnecting() && (packet.GetPacketType() != aznumeric_cast<PacketType>(CorePackets::PacketType::InitiateConnectionPacket)))
-        {
-            const DtlsEndpoint::ConnectResult result = connection.CompleteHandshake();
-            if (result != DtlsEndpoint::ConnectResult::Complete) // DTLS handshake still in progress
-            {
-                // IMPORTANT that we register with the timeout queue here, otherwise we don't have the timer to pop for reliable packets
-                RegisterWithTimeoutQueue(connection.GetConnectionId(), localPacketId, reliabilityType, connection.GetMetrics());
-                AZLOG(NET_DebugDtls, "Connection is still in handshake negotiation, blocking packet send for packet type %d", (int)packet.GetPacketType());
-                return localPacketId;
-            }
-        }
-
         AZLOG(NET_DebugDtls, "Connection is sending packet type %d", aznumeric_cast<int32_t>(packet.GetPacketType()));
-        const bool shouldEncrypt = packet.GetPacketType() == aznumeric_cast<PacketType>(CorePackets::PacketType::InitiateConnectionPacket);
+        // If we're not connected then we're still handshaking and require packets to be unencrypted
+        const bool shouldEncrypt = !IsHandshakePacket(connection.GetDtlsEndpoint(), packet.GetPacketType());
         if (m_socket->Send(address, packetData, packetSize, shouldEncrypt, connection.GetDtlsEndpoint(), connection.GetConnectionQuality()))
         {
             RegisterWithTimeoutQueue(connection.GetConnectionId(), localPacketId, reliabilityType, connection.GetMetrics());
@@ -626,11 +641,10 @@ namespace AzNetworking
 
         AZLOG(Debug_UdpConnect, "Accepted new Udp Connection");
         AZStd::unique_ptr<UdpConnection> connection = AZStd::make_unique<UdpConnection>(connectionId, connectPacket.m_address, *this, ConnectionRole::Acceptor);
-        UdpPacketEncodingBuffer dtlsData;
-        m_socket->AcceptDtlsEndpoint(connection->GetDtlsEndpoint(), connectPacket.m_address, dtlsData);
+        DtlsEndpoint::ConnectResult result = m_socket->AcceptDtlsEndpoint(connection->GetDtlsEndpoint(), connectPacket.m_address);
 
-        // We're accepting this connection, so we can immediately transition to a connected state
-        connection->m_state = ConnectionState::Connected;
+        // Transition state based on our how our socket resolved
+        connection->m_state = result == DtlsEndpoint::ConnectResult::Complete ? ConnectionState::Connected : ConnectionState::Connecting;
         connection->SetTimeoutId(timeoutId);
         m_connectionListener.OnConnect(connection.get());
         m_connectionSet.AddConnection(AZStd::move(connection));
@@ -646,6 +660,14 @@ namespace AzNetworking
         m_removedConnections.emplace_back(RemovedConnection{ connection, reason, endpoint });
     }
 
+    bool UdpNetworkInterface::IsHandshakePacket(const DtlsEndpoint& endpoint, PacketType packetType) const
+    {
+        // Packets involved in handshake are InitiateConnection, ConnectionHandshake and FragmentedPackets of ConnectionHandshake
+        return packetType == aznumeric_cast<PacketType>(CorePackets::PacketType::InitiateConnectionPacket) ||
+            packetType == aznumeric_cast<PacketType>(CorePackets::PacketType::ConnectionHandshakePacket) ||
+            (packetType == aznumeric_cast<PacketType>(CorePackets::PacketType::FragmentedPacket) && endpoint.IsConnecting());
+    }
+
 
     UdpNetworkInterface::ConnectionTimeoutFunctor::ConnectionTimeoutFunctor(UdpNetworkInterface& networkInterface)
         : m_networkInterface(networkInterface)
@@ -668,7 +690,9 @@ namespace AzNetworking
         {
             if (udpConnection->GetDtlsEndpoint().IsConnecting())
             {
-                udpConnection->CompleteHandshake();
+                // DTLS prefers we resend data lost over the wire with fresh SSL IDs so account for that here
+                UdpPacketEncodingBuffer dtlsData;
+                udpConnection->ProcessHandshakeData(dtlsData);
                 return TimeoutResult::Refresh;
             }
         }

Code/Framework/AzNetworking/AzNetworking/UdpTransport/UdpNetworkInterface.h

@@ -104,6 +104,12 @@ namespace AzNetworking
         //! @param endpoint   whether the disconnection was initiated locally or remotely
         void RequestDisconnect(UdpConnection* connection, DisconnectReason reason, TerminationEndpoint endpoint);
 
+        //! Internal helper to check if a packet's type is for connection handshake
+        //! @param  endpoint   DTLS endpoint participating in the handshake
+        //! @param  packetType type of the packet
+        //! @return if the packet is for handshake
+        bool IsHandshakePacket(const DtlsEndpoint& endpoint, AzNetworking::PacketType packetType) const;
+
         AZ_DISABLE_COPY_MOVE(UdpNetworkInterface);
 
         struct ConnectionTimeoutFunctor final

Code/Framework/AzNetworking/AzNetworking/UdpTransport/UdpSocket.cpp

@@ -18,6 +18,9 @@
 #include <AzCore/Console/IConsole.h>
 #include <AzCore/Console/ILogger.h>
 #include <AzCore/std/algorithm.h>
+#include <AzCore/EBus/IEventScheduler.h>
+#include <AzCore/EBus/ScheduledEvent.h>
+#include <AzCore/Interface/Interface.h>
 
 namespace AzNetworking
 {
@@ -41,13 +44,8 @@ namespace AzNetworking
         return DtlsEndpoint::ConnectResult::Complete;
     }
 
-    DtlsEndpoint::ConnectResult UdpSocket::AcceptDtlsEndpoint(DtlsEndpoint&, const IpAddress&, const UdpPacketEncodingBuffer& dtlsData) const
+    DtlsEndpoint::ConnectResult UdpSocket::AcceptDtlsEndpoint(DtlsEndpoint&, const IpAddress&) const
     {
-        if (dtlsData.GetSize() > 0)
-        {
-            AZLOG_WARN("Encryption is disabled on accepting endpoint, but connector provided a DTLS handshake blob.  Check that encryption is properly disabled on *BOTH* endpoints");
-            return DtlsEndpoint::ConnectResult::Failed;
-        }
         // No-op, no encryption wrapper required
         return DtlsEndpoint::ConnectResult::Complete;
     }
@@ -108,28 +106,6 @@ namespace AzNetworking
         m_socketFd = InvalidSocketFd;
     }
 
-#ifdef ENABLE_LATENCY_DEBUG
-    //! Checks packets deferred by latency debug and sends any that have passed their latency threshold
-    void UdpSocket::ProcessDeferredPackets()
-    {
-        const AZ::TimeMs currTimeMs = AZ::GetElapsedTimeMs();
-        while (m_sendBuffer.size() > 0)
-        {
-            const DeferredData& sendData = m_sendBuffer.front();
-            if (sendData.m_timeDeferredMs >= currTimeMs)
-            {
-                SendInternal(sendData.m_address, sendData.m_dataBuffer.GetBuffer(), sendData.m_dataBuffer.GetSize(), sendData.m_encrypt, *sendData.m_dtlsEndpoint);
-                m_sendBuffer.pop_front();
-            }
-            else
-            {
-                // Send buffer is sorted on append so all subsequent elements should also have time remaining
-                break;
-            }
-        }
-    }
-#endif
-
     int32_t UdpSocket::Send
     (
         const IpAddress& address,
@@ -146,17 +122,6 @@ namespace AzNetworking
         AZ_Assert(address.GetAddress(ByteOrder::Host) != 0, "Invalid address");
         AZ_Assert(address.GetPort(ByteOrder::Host) != 0, "Invalid address");
 
-#ifdef ENABLE_LATENCY_DEBUG
-        if ((connectionQuality.m_latencyMs > AZ::TimeMs{ 0 }) || (connectionQuality.m_varianceMs > AZ::TimeMs{ 0 }))
-        {
-            const AZ::TimeMs jitterMs = aznumeric_cast<AZ::TimeMs>(m_random.GetRandom()) % (connectionQuality.m_varianceMs / aznumeric_cast<AZ::TimeMs>(2));
-            const AZ::TimeMs currTimeMs = AZ::GetElapsedTimeMs();
-            const AZ::TimeMs deferTimeMs = (connectionQuality.m_latencyMs / aznumeric_cast<AZ::TimeMs>(2)) + jitterMs;
-            m_sendBuffer.push_back(DeferredData(currTimeMs + deferTimeMs, address, data, size, encrypt, dtlsEndpoint));
-            std::sort(m_sendBuffer.begin(), m_sendBuffer.end());
-        }
-#endif
-
         if (!IsOpen())
         {
             return 0;
@@ -193,6 +158,18 @@ namespace AzNetworking
                 AZLOG_ERROR("Failed to write to socket (%d:%s)", error, GetNetworkErrorDesc(error));
             }
         }
+#ifdef ENABLE_LATENCY_DEBUG
+        else if ((connectionQuality.m_latencyMs > AZ::TimeMs{ 0 }) || (connectionQuality.m_varianceMs > AZ::TimeMs{ 0 }))
+        {
+            const AZ::TimeMs jitterMs = aznumeric_cast<AZ::TimeMs>(m_random.GetRandom()) % (connectionQuality.m_varianceMs / aznumeric_cast<AZ::TimeMs>(2));
+            const AZ::TimeMs currTimeMs = AZ::GetElapsedTimeMs();
+            const AZ::TimeMs deferTimeMs = (connectionQuality.m_latencyMs / aznumeric_cast<AZ::TimeMs>(2)) + jitterMs;
+
+            DeferredData deferred = DeferredData(address, data, size, encrypt, dtlsEndpoint);
+            AZ::Interface<AZ::IEventScheduler>::Get()->AddCallback([&, deferredData = deferred]
+                    { SendInternalDeferred(deferredData); }, AZ::Name("Deferred packet"), deferTimeMs);
+        }
+#endif
 
         m_sentPackets++;
         m_sentBytes += sentBytes;
@@ -253,7 +230,7 @@ namespace AzNetworking
     }
 
     int32_t UdpSocket::SendInternal(const IpAddress& address, const uint8_t* data, uint32_t size,
-        [[maybe_unused]] bool dontEncypt, [[maybe_unused]] DtlsEndpoint& dtlsEndpoint) const
+        [[maybe_unused]] bool encrypt, [[maybe_unused]] DtlsEndpoint& dtlsEndpoint) const
     {
         sockaddr_in destAddr;
         memset(&destAddr, 0, sizeof(destAddr));
@@ -262,4 +239,11 @@ namespace AzNetworking
         destAddr.sin_port = address.GetPort(ByteOrder::Network);
         return sendto(static_cast<int32_t>(m_socketFd), reinterpret_cast<const char*>(data), size, 0, (sockaddr*)&destAddr, sizeof(destAddr));
     }
+
+#ifdef ENABLE_LATENCY_DEBUG
+    int32_t UdpSocket::SendInternalDeferred(const DeferredData& data) const
+    {
+        return SendInternal(data.m_address, data.m_dataBuffer.GetBuffer(), data.m_dataBuffer.GetSize(), data.m_encrypt, *data.m_dtlsEndpoint);
+    }
+#endif
 }

Code/Framework/AzNetworking/AzNetworking/UdpTransport/UdpSocket.h

@@ -23,6 +23,12 @@
 #   define ENABLE_LATENCY_DEBUG 1
 #endif
 
+namespace AZ
+{
+    // Forwards
+    class ScheduledEvent;
+}
+
 namespace AzNetworking
 {
     // Forwards
@@ -57,9 +63,8 @@ namespace AzNetworking
         //! Accepts an encryption socket wrapper.
         //! @param dtlsEndpoint the encryption wrapper instance to create a connection over
         //! @param address      the IP address of the endpoint to connect to
-        //! @param dtlsData     data buffer containing the dtls handshake packet
         //! @return a connect result specifying whether the connection is still pending, failed, or complete
-        virtual DtlsEndpoint::ConnectResult AcceptDtlsEndpoint(DtlsEndpoint& dtlsEndpoint, const IpAddress& address, const UdpPacketEncodingBuffer& dtlsData) const;
+        virtual DtlsEndpoint::ConnectResult AcceptDtlsEndpoint(DtlsEndpoint& dtlsEndpoint, const IpAddress& address) const;
 
         //! Opens the UDP socket on the given port.
         //! @param port      the port number to open the UDP socket on, 0 will bind to any available port
@@ -71,11 +76,6 @@ namespace AzNetworking
         //! Closes an open socket.
         virtual void Close();
 
-#ifdef ENABLE_LATENCY_DEBUG
-        //! Checks packets deferred by latency debug and sends any that have passed their latency threshold
-        virtual void ProcessDeferredPackets();
-#endif
-
         //! Returns true if the UDP socket is currently in an open state.
         //! @return boolean true if the socket is in a connected state
         bool IsOpen() const;
@@ -105,12 +105,20 @@ namespace AzNetworking
         //! @return the total number of packets sent on this socket
         uint32_t GetSentPackets() const;
 
-        //! Returns the total number of bytes received on this socket.
-        //! @return the total number of bytes received on this socket
+        //! Returns the total number of bytes sent on this socket.
+        //! @return the total number of bytes sent on this socket
         uint32_t GetSentBytes() const;
 
-        //! Returns the total number of packets sent on this socket.
-        //! @return the total number of packets sent on this socket
+        //! Returns the total number of encrypted packets sent on this socket.
+        //! @return the total number of encrypted packets sent on this socket
+        uint32_t GetSentPacketsEncrypted() const;
+
+        //! Returns the total number of additional bytes sent on this socket due to SSL encryption.
+        //! @return the total number of additional bytes sent on this socket due to SSL encryption
+        uint32_t GetSentBytesEncryptionInflation() const;
+
+        //! Returns the total number of packets received on this socket.
+        //! @return the total number of packets received on this socket
         uint32_t GetRecvPackets() const;
 
         //! Returns the total number of bytes received on this socket.
@@ -119,6 +127,9 @@ namespace AzNetworking
 
     protected:
 
+        mutable uint32_t m_sentPacketsEncrypted = 0;
+        mutable uint32_t m_sentBytesEncryptionInflation = 0;
+
         virtual int32_t SendInternal(const IpAddress& address, const uint8_t* data, uint32_t size, bool encrypt, DtlsEndpoint& dtlsEndpoint) const;
 
     private:
@@ -132,29 +143,24 @@ namespace AzNetworking
 #ifdef ENABLE_LATENCY_DEBUG
         struct DeferredData
         {
-            DeferredData(AZ::TimeMs timeDeferredMs, const IpAddress& address, const uint8_t* data, uint32_t size, bool encrypt, DtlsEndpoint& dtlsEndpoint)
-                : m_timeDeferredMs(timeDeferredMs)
-                , m_address(address)
+            DeferredData(const IpAddress& address, const uint8_t* data, uint32_t size, bool encrypt, DtlsEndpoint& dtlsEndpoint)
+                : m_address(address)
                 , m_encrypt(encrypt)
                 , m_dtlsEndpoint(&dtlsEndpoint)
             {
                 m_dataBuffer.CopyValues(data, size);
             }
 
-            bool operator <(const DeferredData& rhs) const 
-            {
-                return m_timeDeferredMs < rhs.m_timeDeferredMs;
-            }
-
-            AZ::TimeMs m_timeDeferredMs;
-            IpAddress m_address;
             bool m_encrypt;
-            DtlsEndpoint* m_dtlsEndpoint;
-            UdpPacketEncodingBuffer m_dataBuffer;
+            DtlsEndpoint* m_dtlsEndpoint = nullptr;
+            AZ::ScheduledEvent* m_owningEvent = nullptr;
+            IpAddress m_address;
+            // Deferred UDP packets should have gone through UDP Fragmentation already so ChunkBuffer is sufficient size
+            ChunkBuffer m_dataBuffer;
         };
 
-        mutable AZStd::deque<DeferredData> m_sendBuffer;
-        mutable AZStd::deque<DeferredData> m_recvBuffer;
+        int32_t SendInternalDeferred(const DeferredData& data) const;
+
         mutable AZ::SimpleLcgRandom m_random;
 #endif
     };

Code/Framework/AzNetworking/AzNetworking/UdpTransport/UdpSocket.inl

@@ -34,6 +34,16 @@ namespace AzNetworking
         return m_sentBytes;
     }
 
+    inline uint32_t UdpSocket::GetSentPacketsEncrypted() const
+    {
+        return m_sentPacketsEncrypted;
+    }
+
+    inline uint32_t UdpSocket::GetSentBytesEncryptionInflation() const
+    {
+        return m_sentBytesEncryptionInflation;
+    }
+
     inline uint32_t UdpSocket::GetRecvPackets() const
     {
         return m_recvPackets;