SIG/Network - Migrated legacy multiplayer per entity analytics over to O3DE Multiplayer Gem

Added a new ImGui menu under Multiplayer: "Multiplayer Entity Stats"
This commit is contained in:
Olex Lozitskiy
2021-08-10 15:55:10 -04:00
committed by GitHub
15 changed files with 941 additions and 17 deletions
+1 -1
View File
@@ -79,7 +79,7 @@ ly_add_target(
# The "Multiplayer" target is used by clients and servers, Debug is used only on clients.
ly_create_alias(NAME Multiplayer.Clients NAMESPACE Gem TARGETS Gem::Multiplayer Gem::Multiplayer.Debug)
ly_create_alias(NAME Multiplayer.Servers NAMESPACE Gem TARGETS Gem::Multiplayer)
ly_create_alias(NAME Multiplayer.Servers NAMESPACE Gem TARGETS Gem::Multiplayer Gem::Multiplayer.Debug)
if (PAL_TRAIT_BUILD_HOST_TOOLS)
ly_add_target(
@@ -105,14 +105,14 @@ namespace Multiplayer
template <typename TYPE>
inline void SerializeNetworkPropertyHelper
(
AzNetworking::ISerializer& serializer,
bool modifyRecord,
AzNetworking::FixedSizeBitsetView& bitset,
int32_t bitIndex,
TYPE& value,
const char* name,
NetComponentId componentId,
PropertyIndex propertyIndex,
AzNetworking::ISerializer& serializer,
bool modifyRecord,
AzNetworking::FixedSizeBitsetView& bitset,
int32_t bitIndex,
TYPE& value,
const char* name,
NetComponentId componentId,
PropertyIndex propertyIndex,
MultiplayerStats& stats
)
{
@@ -0,0 +1,28 @@
/*
* Copyright (c) Contributors to the Open 3D Engine Project.
* For complete copyright and license terms please see the LICENSE at the root of this distribution.
*
* SPDX-License-Identifier: Apache-2.0 OR MIT
*
*/
#pragma once
namespace Multiplayer
{
//! @class IMultiplayerDebug
//! @brief IMultiplayerDebug provides access to multiplayer debug overlays
class IMultiplayerDebug
{
public:
AZ_RTTI(IMultiplayerDebug, "{C5EB7F3A-E19F-4921-A604-C9BDC910123C}");
virtual ~IMultiplayerDebug() = default;
//! Enables printing of debug text over entities that have significant amount of traffic.
virtual void ShowEntityBandwidthDebugOverlay() = 0;
//! Disables printing of debug text over entities that have significant amount of traffic.
virtual void HideEntityBandwidthDebugOverlay() = 0;
};
}
@@ -50,10 +50,13 @@ namespace Multiplayer
AZStd::vector<ComponentStats> m_componentStats;
void ReserveComponentStats(NetComponentId netComponentId, uint16_t propertyCount, uint16_t rpcCount);
void RecordEntitySerializeStart(AzNetworking::SerializerMode mode, AZ::EntityId entityId, const char* entityName);
void RecordComponentSerializeEnd(AzNetworking::SerializerMode mode, NetComponentId netComponentId);
void RecordEntitySerializeStop(AzNetworking::SerializerMode mode, AZ::EntityId entityId, const char* entityName);
void RecordPropertySent(NetComponentId netComponentId, PropertyIndex propertyId, uint32_t totalBytes);
void RecordPropertyReceived(NetComponentId netComponentId, PropertyIndex propertyId, uint32_t totalBytes);
void RecordRpcSent(NetComponentId netComponentId, RpcIndex rpcId, uint32_t totalBytes);
void RecordRpcReceived(NetComponentId netComponentId, RpcIndex rpcId, uint32_t totalBytes);
void RecordRpcSent(AZ::EntityId entityId, const char* entityName, NetComponentId netComponentId, RpcIndex rpcId, uint32_t totalBytes);
void RecordRpcReceived(AZ::EntityId entityId, const char* entityName, NetComponentId netComponentId, RpcIndex rpcId, uint32_t totalBytes);
void TickStats(AZ::TimeMs metricFrameTimeMs);
Metric CalculateComponentPropertyUpdateSentMetrics(NetComponentId netComponentId) const;
@@ -64,5 +67,31 @@ namespace Multiplayer
Metric CalculateTotalPropertyUpdateRecvMetrics() const;
Metric CalculateTotalRpcsSentMetrics() const;
Metric CalculateTotalRpcsRecvMetrics() const;
struct Events
{
AZ::Event<AzNetworking::SerializerMode, AZ::EntityId, const char*> m_entitySerializeStart;
AZ::Event<AzNetworking::SerializerMode, NetComponentId> m_componentSerializeEnd;
AZ::Event<AzNetworking::SerializerMode, AZ::EntityId, const char*> m_entitySerializeStop;
AZ::Event<NetComponentId, PropertyIndex, uint32_t> m_propertySent;
AZ::Event<NetComponentId, PropertyIndex, uint32_t> m_propertyReceived;
AZ::Event<AZ::EntityId, const char*, NetComponentId, RpcIndex, uint32_t> m_rpcSent;
AZ::Event<AZ::EntityId, const char*, NetComponentId, RpcIndex, uint32_t> m_rpcReceived;
};
Events m_events;
struct EventHandlers
{
AZ::Event<AzNetworking::SerializerMode, AZ::EntityId, const char*>::Handler m_entitySerializeStart;
AZ::Event<AzNetworking::SerializerMode, NetComponentId>::Handler m_componentSerializeEnd;
AZ::Event<AzNetworking::SerializerMode, AZ::EntityId, const char*>::Handler m_entitySerializeStop;
AZ::Event<NetComponentId, PropertyIndex, uint32_t>::Handler m_propertySent;
AZ::Event<NetComponentId, PropertyIndex, uint32_t>::Handler m_propertyReceived;
AZ::Event<AZ::EntityId, const char*, NetComponentId, RpcIndex, uint32_t>::Handler m_rpcSent;
AZ::Event<AZ::EntityId, const char*, NetComponentId, RpcIndex, uint32_t>::Handler m_rpcReceived;
};
void ConnectHandlers(EventHandlers& handlers);
};
}
@@ -447,12 +447,19 @@ namespace Multiplayer
bool NetBindComponent::SerializeStateDeltaMessage(ReplicationRecord& replicationRecord, AzNetworking::ISerializer& serializer)
{
auto& stats = GetMultiplayer()->GetStats();
stats.RecordEntitySerializeStart(serializer.GetSerializerMode(), GetEntityId(), GetEntity()->GetName().c_str());
bool success = true;
for (auto iter = m_multiplayerSerializationComponentVector.begin(); iter != m_multiplayerSerializationComponentVector.end(); ++iter)
{
success &= (*iter)->SerializeStateDeltaMessage(replicationRecord, serializer);
stats.RecordComponentSerializeEnd(serializer.GetSerializerMode(), (*iter)->GetNetComponentId());
}
stats.RecordEntitySerializeStop(serializer.GetSerializerMode(), GetEntityId(), GetEntity()->GetName().c_str());
return success;
}
@@ -0,0 +1,202 @@
/*
* Copyright (c) Contributors to the Open 3D Engine Project.
* For complete copyright and license terms please see the LICENSE at the root of this distribution.
*
* SPDX-License-Identifier: Apache-2.0 OR MIT
*
*/
#include "MultiplayerDebugByteReporter.h"
#include <iomanip> // for std::setfill
#include <sstream>
#include <AzCore/std/sort.h>
namespace Multiplayer
{
MultiplayerDebugByteReporter::MultiplayerDebugByteReporter()
{
MultiplayerDebugByteReporter::Reset();
}
void MultiplayerDebugByteReporter::ReportBytes(size_t byteSize)
{
m_count++;
m_totalBytes += byteSize;
m_totalBytesThisSecond += byteSize;
m_minBytes = AZStd::min(m_minBytes, byteSize);
m_maxBytes = AZStd::max(m_maxBytes, byteSize);
}
void MultiplayerDebugByteReporter::AggregateBytes(size_t byteSize)
{
m_aggregateBytes += byteSize;
}
void MultiplayerDebugByteReporter::ReportAggregateBytes()
{
ReportBytes(m_aggregateBytes);
m_aggregateBytes = 0;
}
float MultiplayerDebugByteReporter::GetAverageBytes() const
{
if (m_count == 0)
{
return 0.0f;
}
return aznumeric_cast<float>(m_totalBytes) / aznumeric_cast<float>(m_count);
}
size_t MultiplayerDebugByteReporter::GetMaxBytes() const
{
return m_maxBytes;
}
size_t MultiplayerDebugByteReporter::GetMinBytes() const
{
return m_minBytes;
}
size_t MultiplayerDebugByteReporter::GetTotalBytes() const
{
return m_totalBytes;
}
float MultiplayerDebugByteReporter::GetKbitsPerSecond()
{
const auto now = AZStd::chrono::monotonic_clock::now();
// Check the amount of time elapsed and update totals if necessary.
// Time here is measured in whole seconds from the epoch, providing synchronization in
// reporting intervals across all byte reporters.
const AZStd::chrono::seconds nowSeconds = AZStd::chrono::duration_cast<AZStd::chrono::seconds>(now.time_since_epoch());
const AZStd::chrono::seconds secondsSinceLastUpdate = nowSeconds -
AZStd::chrono::duration_cast<AZStd::chrono::seconds>(m_lastUpdateTime.time_since_epoch());
if (secondsSinceLastUpdate.count())
{
// normalize over elapsed milliseconds
constexpr int k_millisecondsPerSecond = 1000;
const auto msSinceLastUpdate = AZStd::chrono::duration_cast<AZStd::chrono::milliseconds>(now - m_lastUpdateTime);
m_totalBytesLastSecond = k_millisecondsPerSecond * aznumeric_cast<float>(m_totalBytesThisSecond) / aznumeric_cast<float>(msSinceLastUpdate.count());
m_totalBytesThisSecond = 0;
m_lastUpdateTime = now;
}
constexpr float bitsPerByte = 8.0f;
constexpr int bitsPerKilobit = 1024;
return bitsPerByte * m_totalBytesLastSecond / bitsPerKilobit;
}
void MultiplayerDebugByteReporter::Combine(const MultiplayerDebugByteReporter& other)
{
m_count += other.m_count;
m_totalBytes += other.m_totalBytes;
m_totalBytesThisSecond += other.m_totalBytesThisSecond;
m_minBytes = AZStd::GetMin(m_minBytes, other.m_minBytes);
m_maxBytes = AZStd::GetMax(m_maxBytes, other.m_maxBytes);
}
void MultiplayerDebugByteReporter::Reset()
{
m_count = 0;
m_totalBytes = 0;
m_totalBytesThisSecond = 0;
m_totalBytesLastSecond = 0;
m_minBytes = std::numeric_limits<decltype(m_minBytes)>::max();
m_maxBytes = 0;
m_aggregateBytes = 0;
}
void MultiplayerDebugComponentReporter::ReportField(const char* fieldName, size_t byteSize)
{
MultiplayerDebugByteReporter::AggregateBytes(byteSize);
m_fieldReports[fieldName].ReportBytes(byteSize);
}
void MultiplayerDebugComponentReporter::ReportFragmentEnd()
{
MultiplayerDebugByteReporter::ReportAggregateBytes();
m_componentDirtyBytes.ReportAggregateBytes();
}
AZStd::vector<MultiplayerDebugComponentReporter::Report> MultiplayerDebugComponentReporter::GetFieldReports()
{
AZStd::vector<Report> copy;
for (auto field = m_fieldReports.begin(); field != m_fieldReports.end(); ++field)
{
copy.emplace_back(field->first, &field->second);
}
auto sortByFrequency = [](const Report& a, const Report& b)
{
return a.second->GetTotalCount() > b.second->GetTotalCount();
};
AZStd::sort(copy.begin(), copy.end(), sortByFrequency);
return copy;
}
void MultiplayerDebugComponentReporter::Combine(const MultiplayerDebugComponentReporter& other)
{
MultiplayerDebugByteReporter::Combine(other);
for (const auto& fieldIterator : other.m_fieldReports)
{
m_fieldReports[fieldIterator.first].Combine(fieldIterator.second);
}
m_componentDirtyBytes.Combine(other.m_componentDirtyBytes);
}
void MultiplayerDebugEntityReporter::ReportField(AZ::u32 index, const char* componentName,
const char* fieldName, size_t byteSize)
{
if (m_currentComponentReport == nullptr)
{
std::stringstream component;
component << "[" << std::setw(2) << std::setfill('0') << aznumeric_cast<int>(index) << "]" << " " << componentName;
m_currentComponentReport = &m_componentReports[component.str().c_str()];
}
m_currentComponentReport->ReportField(fieldName, byteSize);
MultiplayerDebugByteReporter::AggregateBytes(byteSize);
}
void MultiplayerDebugEntityReporter::ReportFragmentEnd()
{
if (m_currentComponentReport)
{
m_currentComponentReport->ReportFragmentEnd();
m_currentComponentReport = nullptr;
}
MultiplayerDebugByteReporter::ReportAggregateBytes();
}
void MultiplayerDebugEntityReporter::Combine(const MultiplayerDebugEntityReporter& other)
{
MultiplayerDebugByteReporter::Combine(other);
for (const auto& componentIterator : other.m_componentReports)
{
m_componentReports[componentIterator.first].Combine(componentIterator.second);
}
SetEntityName(other.GetEntityName());
}
void MultiplayerDebugEntityReporter::Reset()
{
MultiplayerDebugByteReporter::Reset();
m_componentReports.clear();
}
AZStd::map<AZStd::string, MultiplayerDebugComponentReporter>& MultiplayerDebugEntityReporter::GetComponentReports()
{
return m_componentReports;
}
}
@@ -0,0 +1,96 @@
/*
* Copyright (c) Contributors to the Open 3D Engine Project.
* For complete copyright and license terms please see the LICENSE at the root of this distribution.
*
* SPDX-License-Identifier: Apache-2.0 OR MIT
*
*/
#pragma once
#include <AzCore/std/string/string.h>
#include <AzCore/std/chrono/chrono.h>
#include <AzCore/std/containers/map.h>
#include <AzCore/std/containers/vector.h>
namespace Multiplayer
{
class MultiplayerDebugByteReporter
{
public:
MultiplayerDebugByteReporter();
virtual ~MultiplayerDebugByteReporter() = default;
void ReportBytes(size_t byteSize);
void AggregateBytes(size_t byteSize);
void ReportAggregateBytes();
float GetAverageBytes() const;
size_t GetMaxBytes() const;
size_t GetMinBytes() const;
size_t GetTotalBytes() const;
float GetKbitsPerSecond();
void Combine(const MultiplayerDebugByteReporter& other);
virtual void Reset();
size_t GetTotalCount() const { return m_count; }
private:
size_t m_count;
size_t m_totalBytes;
size_t m_totalBytesThisSecond;
float m_totalBytesLastSecond;
size_t m_minBytes;
size_t m_maxBytes;
size_t m_aggregateBytes;
AZStd::chrono::monotonic_clock::time_point m_lastUpdateTime;
};
class MultiplayerDebugComponentReporter final
: public MultiplayerDebugByteReporter
{
public:
MultiplayerDebugComponentReporter() = default;
void ReportField(const char* fieldName, size_t byteSize);
void ReportFragmentEnd();
using Report = AZStd::pair<AZStd::string, MultiplayerDebugByteReporter*>;
AZStd::vector<Report> GetFieldReports();
void Combine(const MultiplayerDebugComponentReporter& other);
private:
AZStd::map<AZStd::string, MultiplayerDebugByteReporter> m_fieldReports;
MultiplayerDebugByteReporter m_componentDirtyBytes;
};
class MultiplayerDebugEntityReporter final
: public MultiplayerDebugByteReporter
{
public:
MultiplayerDebugEntityReporter() = default;
void ReportField(AZ::u32 index, const char* componentName, const char* fieldName, size_t byteSize);
void ReportFragmentEnd();
void Combine(const MultiplayerDebugEntityReporter& other);
void Reset() override;
const char* GetEntityName() const { return m_entityName.c_str(); }
void SetEntityName(const char* entityName)
{
// copying because the entity might go away
m_entityName = entityName;
}
AZStd::map<AZStd::string, MultiplayerDebugComponentReporter>& GetComponentReports();
private:
MultiplayerDebugComponentReporter* m_currentComponentReport = nullptr;
AZStd::map<AZStd::string, MultiplayerDebugComponentReporter> m_componentReports;
AZStd::string m_entityName;
};
}
@@ -0,0 +1,389 @@
/*
* Copyright (c) Contributors to the Open 3D Engine Project.
* For complete copyright and license terms please see the LICENSE at the root of this distribution.
*
* SPDX-License-Identifier: Apache-2.0 OR MIT
*
*/
#include "MultiplayerDebugPerEntityReporter.h"
#include <AzCore/Component/TransformBus.h>
#include <AzCore/Math/ToString.h>
#include <AzFramework/Entity/EntityDebugDisplayBus.h>
#include <Multiplayer/IMultiplayer.h>
#if defined(IMGUI_ENABLED)
#include <imgui/imgui.h>
#endif
AZ_CVAR(float, net_DebugEntities_ShowAboveKbps, 1.f, nullptr, AZ::ConsoleFunctorFlags::Null,
"Prints bandwidth on network entities with higher kpbs than this value");
AZ_CVAR(float, net_DebugEntities_WarnAboveKbps, 10.f, nullptr, AZ::ConsoleFunctorFlags::Null,
"Prints bandwidth on network entities with higher kpbs than this value");
AZ_CVAR(AZ::Color, net_DebugEntities_WarningColor, AZ::Colors::Red, nullptr, AZ::ConsoleFunctorFlags::Null,
"If true, prints debug text over entities that use a considerable amount of network traffic");
AZ_CVAR(AZ::Color, net_DebugEntities_BelowWarningColor, AZ::Colors::Grey, nullptr, AZ::ConsoleFunctorFlags::Null,
"If true, prints debug text over entities that use a considerable amount of network traffic");
namespace Multiplayer
{
#if defined(IMGUI_ENABLED)
static const ImVec4 k_ImGuiTomato = ImVec4(1.0f, 0.4f, 0.3f, 1.0f);
static const ImVec4 k_ImGuiKhaki = ImVec4(0.9f, 0.8f, 0.5f, 1.0f);
static const ImVec4 k_ImGuiCyan = ImVec4(0.5f, 1.0f, 1.0f, 1.0f);
static const ImVec4 k_ImGuiDusk = ImVec4(0.7f, 0.7f, 1.0f, 1.0f);
static const ImVec4 k_ImGuiWhite = ImVec4(1.0f, 1.0f, 1.0f, 1.0f);
// --------------------------------------------------------------------------------------------
template <typename Reporter>
bool ReplicatedStateTreeNode(const AZStd::string& name, Reporter& report, const ImVec4& color, int depth = 0)
{
const int defaultPadAmount = 55;
const int depthReduction = 3;
ImGui::PushStyleColor(ImGuiCol_Text, color);
const bool expanded = ImGui::TreeNode(name.c_str(),
"%-*s %7.2f kbps %7.2f B Avg. %4zu B Max %10zu B Payload",
defaultPadAmount - depthReduction * depth,
name.c_str(),
report.GetKbitsPerSecond(),
report.GetAverageBytes(),
report.GetMaxBytes(),
report.GetTotalBytes());
ImGui::PopStyleColor();
return expanded;
}
// --------------------------------------------------------------------------------------------
void DisplayReplicatedStateReport(AZStd::map<AZStd::string, MultiplayerDebugComponentReporter>& componentReports, float kbpsWarn, float maxWarn)
{
for (auto& componentPair : componentReports)
{
ImGui::Separator();
MultiplayerDebugComponentReporter& componentReport = componentPair.second;
if (ReplicatedStateTreeNode(componentPair.first, componentReport, k_ImGuiCyan, 1))
{
ImGui::Separator();
ImGui::Columns(6, "replicated_field_columns");
ImGui::NextColumn();
ImGui::Text("kbps");
ImGui::NextColumn();
ImGui::Text("Avg. Bytes");
ImGui::NextColumn();
ImGui::Text("Min Bytes");
ImGui::NextColumn();
ImGui::Text("Max Bytes");
ImGui::NextColumn();
ImGui::Text("Total Bytes");
ImGui::NextColumn();
auto fieldReports = componentReport.GetFieldReports();
for (auto& fieldPair : fieldReports)
{
MultiplayerDebugByteReporter& fieldReport = *fieldPair.second;
const float kbitsLastSecond = fieldReport.GetKbitsPerSecond();
const ImVec4* textColor = &k_ImGuiWhite;
if (aznumeric_cast<float>(fieldReport.GetMaxBytes()) > maxWarn)
{
textColor = &k_ImGuiKhaki;
}
if (kbitsLastSecond > kbpsWarn)
{
textColor = &k_ImGuiTomato;
}
ImGui::PushStyleColor(ImGuiCol_Text, *textColor);
ImGui::Text("%s", fieldPair.first.c_str());
ImGui::NextColumn();
ImGui::Text("%.2f", kbitsLastSecond);
ImGui::NextColumn();
ImGui::Text("%.2f", fieldReport.GetAverageBytes());
ImGui::NextColumn();
ImGui::Text("%zu", fieldReport.GetMinBytes());
ImGui::NextColumn();
ImGui::Text("%zu", fieldReport.GetMaxBytes());
ImGui::NextColumn();
ImGui::Text("%zu", fieldReport.GetTotalBytes());
ImGui::NextColumn();
ImGui::PopStyleColor();
}
ImGui::Columns(1);
ImGui::TreePop();
}
}
}
#endif
MultiplayerDebugPerEntityReporter::MultiplayerDebugPerEntityReporter()
: m_updateDebugOverlay([this]() { UpdateDebugOverlay(); }, AZ::Name("UpdateDebugPerEntityOverlay"))
{
m_updateDebugOverlay.Enqueue(AZ::TimeMs{ 0 }, true);
m_eventHandlers.m_entitySerializeStart = decltype(m_eventHandlers.m_entitySerializeStart)([this](AzNetworking::SerializerMode mode, AZ::EntityId entityId, const char* entityName)
{
RecordEntitySerializeStart(mode, entityId, entityName);
});
m_eventHandlers.m_componentSerializeEnd = decltype(m_eventHandlers.m_componentSerializeEnd)([this](AzNetworking::SerializerMode mode,
NetComponentId netComponentId)
{
RecordComponentSerializeEnd(mode, netComponentId);
});
m_eventHandlers.m_entitySerializeStop = decltype(m_eventHandlers.m_entitySerializeStop)([this](AzNetworking::SerializerMode mode, AZ::EntityId entityId, const char* entityName)
{
RecordEntitySerializeStop(mode, entityId, entityName);
});
m_eventHandlers.m_propertySent = decltype(m_eventHandlers.m_propertySent)([this](NetComponentId netComponentId,
PropertyIndex propertyId, uint32_t totalBytes)
{
RecordPropertySent(netComponentId, propertyId, totalBytes);
});
m_eventHandlers.m_propertyReceived = decltype(m_eventHandlers.m_propertyReceived)([this](NetComponentId netComponentId,
PropertyIndex propertyId, uint32_t totalBytes)
{
RecordPropertyReceived(netComponentId, propertyId, totalBytes);
});
m_eventHandlers.m_rpcSent = decltype(m_eventHandlers.m_rpcSent)([this](AZ::EntityId entityId, const char* entityName,
NetComponentId netComponentId,
RpcIndex rpcId, uint32_t totalBytes)
{
RecordRpcSent(entityId, entityName, netComponentId, rpcId, totalBytes);
});
m_eventHandlers.m_rpcReceived = decltype(m_eventHandlers.m_rpcReceived)([this](AZ::EntityId entityId, const char* entityName,
NetComponentId netComponentId,
RpcIndex rpcId, uint32_t totalBytes)
{
RecordRpcSent(entityId, entityName, netComponentId, rpcId, totalBytes);
});
GetMultiplayer()->GetStats().ConnectHandlers(m_eventHandlers);
}
// --------------------------------------------------------------------------------------------
void MultiplayerDebugPerEntityReporter::OnImGuiUpdate()
{
#if defined(IMGUI_ENABLED)
static ImGuiTextFilter filter;
filter.Draw();
if (ImGui::CollapsingHeader("Receiving Entities"))
{
for (AZStd::pair<AZ::EntityId, MultiplayerDebugEntityReporter>& entityPair : m_receivingEntityReports)
{
if (!filter.PassFilter(entityPair.second.GetEntityName()))
{
continue;
}
ImGui::Separator();
if (ReplicatedStateTreeNode(entityPair.second.GetEntityName(), entityPair.second, k_ImGuiDusk))
{
DisplayReplicatedStateReport(entityPair.second.GetComponentReports(), m_replicatedStateKbpsWarn, m_replicatedStateMaxSizeWarn);
ImGui::TreePop();
}
}
}
if (ImGui::CollapsingHeader("Sending Entities"))
{
for (AZStd::pair<AZ::EntityId, MultiplayerDebugEntityReporter>& entityPair : m_sendingEntityReports)
{
const char* name = entityPair.second.GetEntityName();
if (!filter.PassFilter(name))
{
continue;
}
ImGui::Separator();
if (ReplicatedStateTreeNode(name, entityPair.second, k_ImGuiDusk))
{
DisplayReplicatedStateReport(entityPair.second.GetComponentReports(), m_replicatedStateKbpsWarn, m_replicatedStateMaxSizeWarn);
ImGui::TreePop();
}
}
}
#endif
}
void MultiplayerDebugPerEntityReporter::RecordEntitySerializeStart(AzNetworking::SerializerMode mode,
[[maybe_unused]] AZ::EntityId entityId, [[maybe_unused]] const char* entityName)
{
switch (mode)
{
case AzNetworking::SerializerMode::ReadFromObject:
m_currentSendingEntityReport.Reset();
m_currentSendingEntityReport.SetEntityName(entityName);
break;
case AzNetworking::SerializerMode::WriteToObject:
m_currentReceivingEntityReport.Reset();
m_currentReceivingEntityReport.SetEntityName(entityName);
break;
}
}
void MultiplayerDebugPerEntityReporter::RecordComponentSerializeEnd(AzNetworking::SerializerMode mode, [[maybe_unused]] NetComponentId
netComponentId)
{
switch (mode)
{
case AzNetworking::SerializerMode::ReadFromObject:
m_currentSendingEntityReport.ReportFragmentEnd();
break;
case AzNetworking::SerializerMode::WriteToObject:
m_currentReceivingEntityReport.ReportFragmentEnd();
break;
}
}
void MultiplayerDebugPerEntityReporter::RecordEntitySerializeStop(AzNetworking::SerializerMode mode,
[[maybe_unused]] AZ::EntityId entityId, [[maybe_unused]] const char* entityName)
{
switch (mode)
{
case AzNetworking::SerializerMode::ReadFromObject:
m_sendingEntityReports[entityId].Combine(m_currentSendingEntityReport);
break;
case AzNetworking::SerializerMode::WriteToObject:
m_receivingEntityReports[entityId].Combine(m_currentReceivingEntityReport);
break;
}
}
void MultiplayerDebugPerEntityReporter::RecordPropertySent(
NetComponentId netComponentId,
PropertyIndex propertyId,
uint32_t totalBytes)
{
if (const MultiplayerComponentRegistry* componentRegistry = GetMultiplayerComponentRegistry())
{
m_currentSendingEntityReport.ReportField(static_cast<AZ::u32>(netComponentId),
componentRegistry->GetComponentName(netComponentId),
componentRegistry->GetComponentPropertyName(netComponentId, propertyId), totalBytes);
}
}
void MultiplayerDebugPerEntityReporter::RecordPropertyReceived(
NetComponentId netComponentId,
PropertyIndex propertyId,
uint32_t totalBytes)
{
if (const MultiplayerComponentRegistry* componentRegistry = GetMultiplayerComponentRegistry())
{
m_currentReceivingEntityReport.ReportField(static_cast<AZ::u32>(netComponentId),
componentRegistry->GetComponentName(netComponentId),
componentRegistry->GetComponentPropertyName(netComponentId, propertyId), totalBytes);
}
}
void MultiplayerDebugPerEntityReporter::RecordRpcSent(AZ::EntityId entityId, const char* entityName, NetComponentId netComponentId,
RpcIndex rpcId, uint32_t totalBytes)
{
if (const MultiplayerComponentRegistry* componentRegistry = GetMultiplayerComponentRegistry())
{
// MultiplayerDebugByteReporter requires a
RecordEntitySerializeStart(AzNetworking::SerializerMode::ReadFromObject, entityId, entityName);
m_currentSendingEntityReport.ReportField(static_cast<AZ::u32>(netComponentId),
componentRegistry->GetComponentName(netComponentId),
componentRegistry->GetComponentRpcName(netComponentId, rpcId), totalBytes);
RecordComponentSerializeEnd(AzNetworking::SerializerMode::ReadFromObject, netComponentId);
RecordEntitySerializeStop(AzNetworking::SerializerMode::ReadFromObject, entityId, entityName);
}
}
void MultiplayerDebugPerEntityReporter::RecordRpcReceived(
AZ::EntityId entityId, const char* entityName,
NetComponentId netComponentId,
RpcIndex rpcId,
uint32_t totalBytes)
{
if (const MultiplayerComponentRegistry* componentRegistry = GetMultiplayerComponentRegistry())
{
RecordEntitySerializeStart(AzNetworking::SerializerMode::WriteToObject, entityId, entityName);
m_currentReceivingEntityReport.ReportField(static_cast<AZ::u32>(netComponentId),
componentRegistry->GetComponentName(netComponentId),
componentRegistry->GetComponentRpcName(netComponentId, rpcId), totalBytes);
RecordComponentSerializeEnd(AzNetworking::SerializerMode::WriteToObject, netComponentId);
RecordEntitySerializeStop(AzNetworking::SerializerMode::WriteToObject, entityId, entityName);
}
}
void MultiplayerDebugPerEntityReporter::UpdateDebugOverlay()
{
m_networkEntitiesTraffic.clear();
// Merging up and down traffic to provide a unified debug text per entity
for (AZStd::pair<AZ::EntityId, MultiplayerDebugEntityReporter>& entityPair : m_receivingEntityReports)
{
m_networkEntitiesTraffic[entityPair.first].m_name = entityPair.second.GetEntityName();
m_networkEntitiesTraffic[entityPair.first].m_down = entityPair.second.GetKbitsPerSecond();
}
for (AZStd::pair<AZ::EntityId, MultiplayerDebugEntityReporter>& entityPair : m_sendingEntityReports)
{
m_networkEntitiesTraffic[entityPair.first].m_name = entityPair.second.GetEntityName();
m_networkEntitiesTraffic[entityPair.first].m_up = entityPair.second.GetKbitsPerSecond();
}
if (m_debugDisplay == nullptr)
{
AzFramework::DebugDisplayRequestBus::BusPtr debugDisplayBus;
AzFramework::DebugDisplayRequestBus::Bind(debugDisplayBus, AzFramework::g_defaultSceneEntityDebugDisplayId);
m_debugDisplay = AzFramework::DebugDisplayRequestBus::FindFirstHandler(debugDisplayBus);
}
const AZ::u32 stateBefore = m_debugDisplay->GetState();
for (const AZStd::pair<AZ::EntityId, NetworkEntityTraffic>& networkEntity : m_networkEntitiesTraffic)
{
if (networkEntity.second.m_down < net_DebugEntities_ShowAboveKbps && networkEntity.second.m_up < net_DebugEntities_ShowAboveKbps)
{
continue;
}
if (networkEntity.second.m_down > net_DebugEntities_WarnAboveKbps || networkEntity.second.m_up > net_DebugEntities_WarnAboveKbps)
{
m_debugDisplay->SetColor(net_DebugEntities_WarningColor);
}
else
{
m_debugDisplay->SetColor(net_DebugEntities_BelowWarningColor);
}
if (networkEntity.second.m_down > net_DebugEntities_ShowAboveKbps && networkEntity.second.m_up > net_DebugEntities_ShowAboveKbps)
{
azsprintf(m_statusBuffer, "[%s] %.0f down / %0.f up (kbps)", networkEntity.second.m_name,
networkEntity.second.m_down, networkEntity.second.m_up);
}
else if (networkEntity.second.m_down > net_DebugEntities_ShowAboveKbps)
{
azsprintf(m_statusBuffer, "[%s] %.0f down (kbps)", networkEntity.second.m_name, networkEntity.second.m_down);
}
else
{
azsprintf(m_statusBuffer, "[%s] %.0f up (kbps)", networkEntity.second.m_name, networkEntity.second.m_up);
}
AZ::Vector3 entityPosition = AZ::Vector3::CreateZero();
AZ::TransformBus::EventResult(entityPosition, networkEntity.first, &AZ::TransformBus::Events::GetWorldTranslation);
if (entityPosition.IsZero() == false)
{
constexpr bool centerText = true;
m_debugDisplay->DrawTextLabel(entityPosition, 1.0f, m_statusBuffer, centerText, 0, 0);
}
}
m_debugDisplay->SetState(stateBefore);
}
}
@@ -0,0 +1,72 @@
/*
* Copyright (c) Contributors to the Open 3D Engine Project.
* For complete copyright and license terms please see the LICENSE at the root of this distribution.
*
* SPDX-License-Identifier: Apache-2.0 OR MIT
*
*/
#pragma once
#include "MultiplayerDebugByteReporter.h"
#include <AzCore/Component/EntityId.h>
#include <AzCore/EBus/ScheduledEvent.h>
#include <AzFramework/Entity/EntityDebugDisplayBus.h>
#include <Multiplayer/MultiplayerStats.h>
#include <Multiplayer/MultiplayerTypes.h>
namespace Multiplayer
{
/**
* \brief Multiplayer traffic live analysis tool via ImGui.
*/
class MultiplayerDebugPerEntityReporter
{
public:
MultiplayerDebugPerEntityReporter();
//! main update loop
void OnImGuiUpdate();
//! Event handlers
// @{
void RecordEntitySerializeStart(AzNetworking::SerializerMode mode, AZ::EntityId entityId, const char* entityName);
void RecordComponentSerializeEnd(AzNetworking::SerializerMode mode, NetComponentId netComponentId);
void RecordEntitySerializeStop(AzNetworking::SerializerMode mode, AZ::EntityId entityId, const char* entityName);
void RecordPropertySent(NetComponentId netComponentId, PropertyIndex propertyId, uint32_t totalBytes);
void RecordPropertyReceived(NetComponentId netComponentId, PropertyIndex propertyId, uint32_t totalBytes);
void RecordRpcSent(AZ::EntityId entityId, const char* entityName, NetComponentId netComponentId, RpcIndex rpcId, uint32_t totalBytes);
void RecordRpcReceived(AZ::EntityId entityId, const char* entityName, NetComponentId netComponentId, RpcIndex rpcId, uint32_t totalBytes);
// }@
//! Draws bandwidth text over entities
void UpdateDebugOverlay();
private:
AZ::ScheduledEvent m_updateDebugOverlay;
MultiplayerStats::EventHandlers m_eventHandlers;
AZStd::map<AZ::EntityId, MultiplayerDebugEntityReporter> m_sendingEntityReports{};
MultiplayerDebugEntityReporter m_currentSendingEntityReport;
AZStd::map<AZ::EntityId, MultiplayerDebugEntityReporter> m_receivingEntityReports{};
MultiplayerDebugEntityReporter m_currentReceivingEntityReport;
float m_replicatedStateKbpsWarn = 10.f;
float m_replicatedStateMaxSizeWarn = 30.f;
char m_statusBuffer[100] = {};
struct NetworkEntityTraffic
{
const char* m_name = nullptr;
float m_up = 0.f;
float m_down = 0.f;
};
AZStd::unordered_map<AZ::EntityId, NetworkEntityTraffic> m_networkEntitiesTraffic;
AzFramework::DebugDisplayRequests* m_debugDisplay = nullptr;
};
}
@@ -13,6 +13,11 @@
#include <AzNetworking/Framework/INetworkInterface.h>
#include <Multiplayer/IMultiplayer.h>
void OnDebugEntities_ShowBandwidth_Changed(const bool& showBandwidth);
AZ_CVAR(bool, net_DebugEntities_ShowBandwidth, false, &OnDebugEntities_ShowBandwidth_Changed, AZ::ConsoleFunctorFlags::Null,
"If true, prints bandwidth values over entities that use a considerable amount of network traffic");
namespace Multiplayer
{
void MultiplayerDebugSystemComponent::Reflect(AZ::ReflectContext* context)
@@ -47,6 +52,17 @@ namespace Multiplayer
ImGui::ImGuiUpdateListenerBus::Handler::BusDisconnect();
#endif
}
void MultiplayerDebugSystemComponent::ShowEntityBandwidthDebugOverlay()
{
m_reporter = AZStd::make_unique<MultiplayerDebugPerEntityReporter>();
}
void MultiplayerDebugSystemComponent::HideEntityBandwidthDebugOverlay()
{
m_reporter.reset();
}
#ifdef IMGUI_ENABLED
void MultiplayerDebugSystemComponent::OnImGuiMainMenuUpdate()
{
@@ -54,6 +70,7 @@ namespace Multiplayer
{
ImGui::Checkbox("Networking Stats", &m_displayNetworkingStats);
ImGui::Checkbox("Multiplayer Stats", &m_displayMultiplayerStats);
ImGui::Checkbox("Multiplayer Entity Stats", &m_displayPerEntityStats);
ImGui::EndMenu();
}
}
@@ -432,6 +449,36 @@ namespace Multiplayer
DrawMultiplayerStats();
}
}
if (m_displayPerEntityStats)
{
if (ImGui::Begin("Multiplayer Per Entity Stats", &m_displayPerEntityStats, ImGuiWindowFlags_AlwaysAutoResize))
{
// This overrides @net_DebugNetworkEntity_ShowBandwidth value
if (m_reporter == nullptr)
{
ShowEntityBandwidthDebugOverlay();
}
if (m_reporter)
{
m_reporter->OnImGuiUpdate();
}
}
}
}
#endif
}
void OnDebugEntities_ShowBandwidth_Changed(const bool& showBandwidth)
{
if (showBandwidth)
{
AZ::Interface<Multiplayer::IMultiplayerDebug>::Get()->ShowEntityBandwidthDebugOverlay();
}
else
{
AZ::Interface<Multiplayer::IMultiplayerDebug>::Get()->HideEntityBandwidthDebugOverlay();
}
}
@@ -9,6 +9,9 @@
#pragma once
#include <AzCore/Component/Component.h>
#include <AzCore/Interface/Interface.h>
#include <Debug/MultiplayerDebugPerEntityReporter.h>
#include <Multiplayer/IMultiplayerDebug.h>
#ifdef IMGUI_ENABLED
# include <imgui/imgui.h>
@@ -19,6 +22,7 @@ namespace Multiplayer
{
class MultiplayerDebugSystemComponent final
: public AZ::Component
, public AZ::Interface<IMultiplayerDebug>::Registrar
#ifdef IMGUI_ENABLED
, public ImGui::ImGuiUpdateListenerBus::Handler
#endif
@@ -29,7 +33,7 @@ namespace Multiplayer
static void Reflect(AZ::ReflectContext* context);
static void GetProvidedServices(AZ::ComponentDescriptor::DependencyArrayType& provided);
static void GetRequiredServices(AZ::ComponentDescriptor::DependencyArrayType& required);
static void GetIncompatibleServices(AZ::ComponentDescriptor::DependencyArrayType& incompatbile);
static void GetIncompatibleServices(AZ::ComponentDescriptor::DependencyArrayType& incompatible);
~MultiplayerDebugSystemComponent() override = default;
@@ -39,6 +43,12 @@ namespace Multiplayer
void Deactivate() override;
//! @}
//! IMultiplayerDebug overrides
//! @{
void ShowEntityBandwidthDebugOverlay() override;
void HideEntityBandwidthDebugOverlay() override;
//! @}
#ifdef IMGUI_ENABLED
//! ImGui::ImGuiUpdateListenerBus overrides
//! @{
@@ -49,5 +59,8 @@ namespace Multiplayer
private:
bool m_displayNetworkingStats = false;
bool m_displayMultiplayerStats = false;
bool m_displayPerEntityStats = false;
AZStd::unique_ptr<MultiplayerDebugPerEntityReporter> m_reporter;
};
}
@@ -29,6 +29,21 @@ namespace Multiplayer
m_componentStats[netComponentIndex].m_rpcsRecv.resize(rpcCount);
}
void MultiplayerStats::RecordEntitySerializeStart(AzNetworking::SerializerMode mode, AZ::EntityId entityId, const char* entityName)
{
m_events.m_entitySerializeStart.Signal(mode, entityId, entityName);
}
void MultiplayerStats::RecordComponentSerializeEnd(AzNetworking::SerializerMode mode, NetComponentId netComponentId)
{
m_events.m_componentSerializeEnd.Signal(mode, netComponentId);
}
void MultiplayerStats::RecordEntitySerializeStop(AzNetworking::SerializerMode mode, AZ::EntityId entityId, const char* entityName)
{
m_events.m_entitySerializeStop.Signal(mode, entityId, entityName);
}
void MultiplayerStats::RecordPropertySent(NetComponentId netComponentId, PropertyIndex propertyId, uint32_t totalBytes)
{
const uint16_t netComponentIndex = aznumeric_cast<uint16_t>(netComponentId);
@@ -37,6 +52,8 @@ namespace Multiplayer
m_componentStats[netComponentIndex].m_propertyUpdatesSent[propertyIndex].m_totalBytes += totalBytes;
m_componentStats[netComponentIndex].m_propertyUpdatesSent[propertyIndex].m_callHistory[m_recordMetricIndex]++;
m_componentStats[netComponentIndex].m_propertyUpdatesSent[propertyIndex].m_byteHistory[m_recordMetricIndex] += totalBytes;
m_events.m_propertySent.Signal(netComponentId, propertyId, totalBytes);
}
void MultiplayerStats::RecordPropertyReceived(NetComponentId netComponentId, PropertyIndex propertyId, uint32_t totalBytes)
@@ -47,9 +64,11 @@ namespace Multiplayer
m_componentStats[netComponentIndex].m_propertyUpdatesRecv[propertyIndex].m_totalBytes += totalBytes;
m_componentStats[netComponentIndex].m_propertyUpdatesRecv[propertyIndex].m_callHistory[m_recordMetricIndex]++;
m_componentStats[netComponentIndex].m_propertyUpdatesRecv[propertyIndex].m_byteHistory[m_recordMetricIndex] += totalBytes;
m_events.m_propertyReceived.Signal(netComponentId, propertyId, totalBytes);
}
void MultiplayerStats::RecordRpcSent(NetComponentId netComponentId, RpcIndex rpcId, uint32_t totalBytes)
void MultiplayerStats::RecordRpcSent(AZ::EntityId entityId, const char* entityName, NetComponentId netComponentId, RpcIndex rpcId, uint32_t totalBytes)
{
const uint16_t netComponentIndex = aznumeric_cast<uint16_t>(netComponentId);
const uint16_t rpcIndex = aznumeric_cast<uint16_t>(rpcId);
@@ -57,9 +76,11 @@ namespace Multiplayer
m_componentStats[netComponentIndex].m_rpcsSent[rpcIndex].m_totalBytes += totalBytes;
m_componentStats[netComponentIndex].m_rpcsSent[rpcIndex].m_callHistory[m_recordMetricIndex]++;
m_componentStats[netComponentIndex].m_rpcsSent[rpcIndex].m_byteHistory[m_recordMetricIndex] += totalBytes;
m_events.m_rpcSent.Signal(entityId, entityName, netComponentId, rpcId, totalBytes);
}
void MultiplayerStats::RecordRpcReceived(NetComponentId netComponentId, RpcIndex rpcId, uint32_t totalBytes)
void MultiplayerStats::RecordRpcReceived(AZ::EntityId entityId, const char* entityName, NetComponentId netComponentId, RpcIndex rpcId, uint32_t totalBytes)
{
const uint16_t netComponentIndex = aznumeric_cast<uint16_t>(netComponentId);
const uint16_t rpcIndex = aznumeric_cast<uint16_t>(rpcId);
@@ -67,6 +88,8 @@ namespace Multiplayer
m_componentStats[netComponentIndex].m_rpcsRecv[rpcIndex].m_totalBytes += totalBytes;
m_componentStats[netComponentIndex].m_rpcsRecv[rpcIndex].m_callHistory[m_recordMetricIndex]++;
m_componentStats[netComponentIndex].m_rpcsRecv[rpcIndex].m_byteHistory[m_recordMetricIndex] += totalBytes;
m_events.m_rpcReceived.Signal(entityId, entityName, netComponentId, rpcId, totalBytes);
}
void MultiplayerStats::TickStats(AZ::TimeMs metricFrameTimeMs)
@@ -186,4 +209,15 @@ namespace Multiplayer
}
return result;
}
void MultiplayerStats::ConnectHandlers(EventHandlers& handlers)
{
handlers.m_entitySerializeStart.Connect(m_events.m_entitySerializeStart);
handlers.m_componentSerializeEnd.Connect(m_events.m_componentSerializeEnd);
handlers.m_entitySerializeStop.Connect(m_events.m_entitySerializeStop);
handlers.m_propertySent.Connect(m_events.m_propertySent);
handlers.m_propertyReceived.Connect(m_events.m_propertyReceived);
handlers.m_rpcSent.Connect(m_events.m_rpcSent);
handlers.m_rpcReceived.Connect(m_events.m_rpcReceived);
}
}
@@ -441,7 +441,8 @@ namespace Multiplayer
{
// Received rpc metrics, log rpc sent, number of bytes, and the componentId/rpcId for bandwidth metrics
MultiplayerStats& stats = GetMultiplayer()->GetStats();
stats.RecordRpcSent(entityRpcMessage.GetComponentId(), entityRpcMessage.GetRpcIndex(), entityRpcMessage.GetEstimatedSerializeSize());
stats.RecordRpcSent(GetEntityHandle().GetEntity()->GetId(), GetEntityHandle().GetEntity()->GetName().c_str(),
entityRpcMessage.GetComponentId(), entityRpcMessage.GetRpcIndex(), entityRpcMessage.GetEstimatedSerializeSize());
m_replicationManager.AddDeferredRpcMessage(entityRpcMessage);
}
@@ -515,7 +516,7 @@ namespace Multiplayer
&& (GetRemoteNetworkRole() == NetEntityRole::Server))
{
// We are on a server, and we received this message from another server, therefore we should forward this to our autonomous player
// This can occur if we've recently migrated
// This can occur if we've recently migrated
result = RpcValidationResult::ForwardToAutonomous;
}
}
@@ -624,7 +625,8 @@ namespace Multiplayer
{
// Received rpc metrics, log rpc received, time spent, number of bytes, and the componentId/rpcId for bandwidth metrics
MultiplayerStats& stats = GetMultiplayer()->GetStats();
stats.RecordRpcReceived(entityRpcMessage.GetComponentId(), entityRpcMessage.GetRpcIndex(), entityRpcMessage.GetEstimatedSerializeSize());
stats.RecordRpcReceived(GetEntityHandle().GetEntity()->GetId(), GetEntityHandle().GetEntity()->GetName().c_str(),
entityRpcMessage.GetComponentId(), entityRpcMessage.GetRpcIndex(), entityRpcMessage.GetEstimatedSerializeSize());
if (!m_netBindComponent)
{
@@ -7,6 +7,10 @@
#
set(FILES
Source/Debug/MultiplayerDebugByteReporter.cpp
Source/Debug/MultiplayerDebugByteReporter.h
Source/Debug/MultiplayerDebugPerEntityReporter.cpp
Source/Debug/MultiplayerDebugPerEntityReporter.h
Source/Debug/MultiplayerDebugModule.cpp
Source/Debug/MultiplayerDebugModule.h
Source/Debug/MultiplayerDebugSystemComponent.cpp
@@ -8,6 +8,7 @@
set(FILES
Include/Multiplayer/IMultiplayer.h
Include/Multiplayer/IMultiplayerDebug.h
Include/Multiplayer/IMultiplayerTools.h
Include/Multiplayer/MultiplayerConstants.h
Include/Multiplayer/MultiplayerStats.h