From 9a15b7cadc4c4432a31c5e1ea09e51d0bcce196d Mon Sep 17 00:00:00 2001 From: amzn-sj Date: Wed, 3 Nov 2021 11:18:57 -0700 Subject: [PATCH 01/34] 1. Fix "Open Editor" button not launching Editor on Mac. 2. Update LaunchAssetProcessor() paths on Mac. 3. LaunchAssetProcessor() uses ProcessWatcher wrappers. 4. SDK Launcher registers the engine when launching. Signed-off-by: amzn-sj --- .../Asset/AssetSystemComponentHelper_Mac.cpp | 32 ++++++++++-------- .../BundleLauncher/O3DE_SDK_Launcher.cpp | 6 ++++ .../Platform/Linux/ProjectUtils_linux.cpp | 5 +++ .../Platform/Mac/ProjectUtils_mac.cpp | 33 +++++++++++++++++++ .../Platform/Windows/ProjectUtils_windows.cpp | 5 +++ .../ProjectManager/Source/ProjectUtils.h | 4 ++- .../ProjectManager/Source/ProjectsScreen.cpp | 4 +-- 7 files changed, 73 insertions(+), 16 deletions(-) diff --git a/Code/Framework/AzFramework/Platform/Mac/AzFramework/Asset/AssetSystemComponentHelper_Mac.cpp b/Code/Framework/AzFramework/Platform/Mac/AzFramework/Asset/AssetSystemComponentHelper_Mac.cpp index c6f481b65b..a3381dabb8 100644 --- a/Code/Framework/AzFramework/Platform/Mac/AzFramework/Asset/AssetSystemComponentHelper_Mac.cpp +++ b/Code/Framework/AzFramework/Platform/Mac/AzFramework/Asset/AssetSystemComponentHelper_Mac.cpp @@ -10,6 +10,8 @@ #include #include +#include + #include #include @@ -24,14 +26,20 @@ namespace AzFramework::AssetSystem::Platform AZ::IO::FixedMaxPath assetProcessorPath{ executableDirectory }; // In Mac the Editor and game is within a bundle, so the path to the sibling app // has to go up from the Contents/MacOS folder the binary is in - assetProcessorPath /= "../../../AssetProcessor.app"; + assetProcessorPath /= "../../../AssetProcessor.app/Contents/MacOS/AssetProcessor"; assetProcessorPath = assetProcessorPath.LexicallyNormal(); if (!AZ::IO::SystemFile::Exists(assetProcessorPath.c_str())) { - // Check for existence of one under a "bin" directory, i.e. engineRoot is an SDK structure. - assetProcessorPath = - AZ::IO::FixedMaxPath{engineRoot} / "bin" / AZ_TRAIT_OS_PLATFORM_NAME / AZ_BUILD_CONFIGURATION_TYPE / "AssetProcessor.app"; + if (auto settingsRegistry = AZ::SettingsRegistry::Get(); settingsRegistry != nullptr) + { + if (AZ::IO::FixedMaxPath installedBinariesPath; + settingsRegistry->Get(installedBinariesPath.Native(), AZ::SettingsRegistryMergeUtils::FilePathKey_InstalledBinaryFolder)) + { + // Check for existence of one under a "bin" directory, i.e. engineRoot is an SDK structure. + assetProcessorPath = AZ::IO::FixedMaxPath{ engineRoot } / installedBinariesPath / "AssetProcessor.app/Contents/MacOS/AssetProcessor"; + } + } if (!AZ::IO::SystemFile::Exists(assetProcessorPath.c_str())) { @@ -39,23 +47,21 @@ namespace AzFramework::AssetSystem::Platform } } - auto fullLaunchCommand = AZ::IO::FixedMaxPathString::format(R"(open -g "%s" --args --start-hidden)", assetProcessorPath.c_str()); + AZStd::string commandLineParams; // Add the engine path to the launch command if not empty if (!engineRoot.empty()) { - fullLaunchCommand += R"( --engine-path=")"; - fullLaunchCommand += engineRoot; - fullLaunchCommand += '"'; + commandLineParams += AZStd::string::format("\"--engine-path=\"%s\"\"", engineRoot.data()); } - // Add the active project path to the launch command if not empty if (!projectPath.empty()) { - fullLaunchCommand += R"( --project-path=")"; - fullLaunchCommand += projectPath; - fullLaunchCommand += '"'; + commandLineParams += AZStd::string::format(" \"--regset=/Amazon/AzCore/Bootstrap/project_path=\"%s\"\"", projectPath.data()); } - return system(fullLaunchCommand.c_str()) == 0; + AzFramework::ProcessLauncher::ProcessLaunchInfo processLaunchInfo; + processLaunchInfo.m_processExecutableString = AZStd::move(assetProcessorPath.Native()); + processLaunchInfo.m_commandlineParameters = commandLineParams; + return AzFramework::ProcessLauncher::LaunchUnwatchedProcess(processLaunchInfo); } } diff --git a/Code/Tools/BundleLauncher/O3DE_SDK_Launcher.cpp b/Code/Tools/BundleLauncher/O3DE_SDK_Launcher.cpp index 0c362ac829..4dd7e184e9 100644 --- a/Code/Tools/BundleLauncher/O3DE_SDK_Launcher.cpp +++ b/Code/Tools/BundleLauncher/O3DE_SDK_Launcher.cpp @@ -49,6 +49,12 @@ int main(int argc, char* argv[]) AZStd::unique_ptr shellProcess(AzFramework::ProcessWatcher::LaunchProcess(shellProcessLaunch, AzFramework::ProcessCommunicationType::COMMUNICATOR_TYPE_NONE)); shellProcess->WaitForProcessToExit(120); shellProcess.reset(); + + parameters = AZStd::string::format("-c \"%s/scripts/o3de.sh register --this-engine\"", enginePath.c_str()); + shellProcessLaunch.m_commandlineParameters = parameters; + shellProcess.reset(AzFramework::ProcessWatcher::LaunchProcess(shellProcessLaunch, AzFramework::ProcessCommunicationType::COMMUNICATOR_TYPE_NONE)); + shellProcess->WaitForProcessToExit(120); + shellProcess.reset(); AZ::IO::FixedMaxPath projectManagerPath = installedBinariesFolder/"o3de.app"/"Contents"/"MacOS"/"o3de"; AzFramework::ProcessLauncher::ProcessLaunchInfo processLaunchInfo; diff --git a/Code/Tools/ProjectManager/Platform/Linux/ProjectUtils_linux.cpp b/Code/Tools/ProjectManager/Platform/Linux/ProjectUtils_linux.cpp index 0d66009d90..abb062c08a 100644 --- a/Code/Tools/ProjectManager/Platform/Linux/ProjectUtils_linux.cpp +++ b/Code/Tools/ProjectManager/Platform/Linux/ProjectUtils_linux.cpp @@ -94,5 +94,10 @@ namespace O3DE::ProjectManager QProcessEnvironment::systemEnvironment(), QObject::tr("Running get_python script...")); } + + AZ::IO::FixedMaxPath GetEditorDirectory() + { + return AZ::Utils::GetExecutableDirectory(); + } } // namespace ProjectUtils } // namespace O3DE::ProjectManager diff --git a/Code/Tools/ProjectManager/Platform/Mac/ProjectUtils_mac.cpp b/Code/Tools/ProjectManager/Platform/Mac/ProjectUtils_mac.cpp index e36f6cd0c8..b768200398 100644 --- a/Code/Tools/ProjectManager/Platform/Mac/ProjectUtils_mac.cpp +++ b/Code/Tools/ProjectManager/Platform/Mac/ProjectUtils_mac.cpp @@ -11,6 +11,9 @@ #include #include +#include +#include + namespace O3DE::ProjectManager { namespace ProjectUtils @@ -104,5 +107,35 @@ namespace O3DE::ProjectManager QProcessEnvironment::systemEnvironment(), QObject::tr("Running get_python script...")); } + + AZ::IO::FixedMaxPath GetEditorDirectory() + { + AZ::IO::FixedMaxPath executableDirectory = AZ::Utils::GetExecutableDirectory(); + AZ::IO::FixedMaxPath editorPath{ executableDirectory }; + editorPath /= "../../../Editor.app/Contents/MacOS"; + editorPath = editorPath.LexicallyNormal(); + if (!AZ::IO::SystemFile::IsDirectory(editorPath.c_str())) + { + if (auto settingsRegistry = AZ::SettingsRegistry::Get(); settingsRegistry != nullptr) + { + if (AZ::IO::FixedMaxPath installedBinariesPath; + settingsRegistry->Get(installedBinariesPath.Native(), AZ::SettingsRegistryMergeUtils::FilePathKey_InstalledBinaryFolder)) + { + if (AZ::IO::FixedMaxPath engineRootFolder; + settingsRegistry->Get(engineRootFolder.Native(), AZ::SettingsRegistryMergeUtils::FilePathKey_EngineRootFolder)) + { + editorPath = engineRootFolder / installedBinariesPath / "Editor.app/Contents/MacOS"; + } + } + } + + if (!AZ::IO::SystemFile::IsDirectory(editorPath.c_str())) + { + AZ_Error("ProjectManager", false, "Unable to find the Editor app bundle!"); + } + } + + return editorPath; + } } // namespace ProjectUtils } // namespace O3DE::ProjectManager diff --git a/Code/Tools/ProjectManager/Platform/Windows/ProjectUtils_windows.cpp b/Code/Tools/ProjectManager/Platform/Windows/ProjectUtils_windows.cpp index 831529d5e4..10344bf42f 100644 --- a/Code/Tools/ProjectManager/Platform/Windows/ProjectUtils_windows.cpp +++ b/Code/Tools/ProjectManager/Platform/Windows/ProjectUtils_windows.cpp @@ -139,5 +139,10 @@ namespace O3DE::ProjectManager QProcessEnvironment::systemEnvironment(), QObject::tr("Running get_python script...")); } + + AZ::IO::FixedMaxPath GetEditorDirectory() + { + return AZ::Utils::GetExecutableDirectory(); + } } // namespace ProjectUtils } // namespace O3DE::ProjectManager diff --git a/Code/Tools/ProjectManager/Source/ProjectUtils.h b/Code/Tools/ProjectManager/Source/ProjectUtils.h index 1fdf76913e..890d50d2de 100644 --- a/Code/Tools/ProjectManager/Source/ProjectUtils.h +++ b/Code/Tools/ProjectManager/Source/ProjectUtils.h @@ -14,6 +14,7 @@ #include #include +#include #include namespace O3DE::ProjectManager @@ -67,7 +68,8 @@ namespace O3DE::ProjectManager AZ::Outcome GetProjectBuildPath(const QString& projectPath); AZ::Outcome OpenCMakeGUI(const QString& projectPath); AZ::Outcome RunGetPythonScript(const QString& enginePath); - + + AZ::IO::FixedMaxPath GetEditorDirectory(); } // namespace ProjectUtils } // namespace O3DE::ProjectManager diff --git a/Code/Tools/ProjectManager/Source/ProjectsScreen.cpp b/Code/Tools/ProjectManager/Source/ProjectsScreen.cpp index cf42da88fa..f86a689e59 100644 --- a/Code/Tools/ProjectManager/Source/ProjectsScreen.cpp +++ b/Code/Tools/ProjectManager/Source/ProjectsScreen.cpp @@ -392,11 +392,11 @@ namespace O3DE::ProjectManager { if (!WarnIfInBuildQueue(projectPath)) { - AZ::IO::FixedMaxPath executableDirectory = AZ::Utils::GetExecutableDirectory(); + AZ::IO::FixedMaxPath executableDirectory = ProjectUtils::GetEditorDirectory(); AZStd::string executableFilename = "Editor"; AZ::IO::FixedMaxPath editorExecutablePath = executableDirectory / (executableFilename + AZ_TRAIT_OS_EXECUTABLE_EXTENSION); auto cmdPath = AZ::IO::FixedMaxPathString::format( - "%s -regset=\"/Amazon/AzCore/Bootstrap/project_path=%s\"", editorExecutablePath.c_str(), + "%s --regset=\"/Amazon/AzCore/Bootstrap/project_path=%s\"", editorExecutablePath.c_str(), projectPath.toStdString().c_str()); AzFramework::ProcessLauncher::ProcessLaunchInfo processLaunchInfo; From 41d3a38781674dbedd8ea936c00a731518cdb2c6 Mon Sep 17 00:00:00 2001 From: amzn-sj Date: Wed, 3 Nov 2021 12:42:22 -0700 Subject: [PATCH 02/34] Add missing header file Signed-off-by: amzn-sj --- Code/Tools/ProjectManager/Platform/Linux/ProjectUtils_linux.cpp | 2 ++ .../ProjectManager/Platform/Windows/ProjectUtils_windows.cpp | 2 ++ 2 files changed, 4 insertions(+) diff --git a/Code/Tools/ProjectManager/Platform/Linux/ProjectUtils_linux.cpp b/Code/Tools/ProjectManager/Platform/Linux/ProjectUtils_linux.cpp index abb062c08a..e901d807b4 100644 --- a/Code/Tools/ProjectManager/Platform/Linux/ProjectUtils_linux.cpp +++ b/Code/Tools/ProjectManager/Platform/Linux/ProjectUtils_linux.cpp @@ -10,6 +10,8 @@ #include #include +#include + namespace O3DE::ProjectManager { namespace ProjectUtils diff --git a/Code/Tools/ProjectManager/Platform/Windows/ProjectUtils_windows.cpp b/Code/Tools/ProjectManager/Platform/Windows/ProjectUtils_windows.cpp index 10344bf42f..871f8e9567 100644 --- a/Code/Tools/ProjectManager/Platform/Windows/ProjectUtils_windows.cpp +++ b/Code/Tools/ProjectManager/Platform/Windows/ProjectUtils_windows.cpp @@ -14,6 +14,8 @@ #include #include +#include + namespace O3DE::ProjectManager { namespace ProjectUtils From 5b734b9d4159c666a0a78d6d595c531e9f334367 Mon Sep 17 00:00:00 2001 From: kberg-amzn Date: Wed, 3 Nov 2021 14:17:20 -0700 Subject: [PATCH 03/34] A number of fixes to timeout and disconnect handling Signed-off-by: kberg-amzn --- .../AzCore/EBus/ScheduledEventHandle.cpp | 2 +- .../AutoGen/CorePackets.AutoPackets.xml | 4 +- .../TcpTransport/TcpConnection.cpp | 11 +- .../AzNetworking/TcpTransport/TcpConnection.h | 13 +- .../TcpTransport/TcpConnection.inl | 10 -- .../TcpTransport/TcpNetworkInterface.cpp | 47 +------ .../TcpTransport/TcpNetworkInterface.h | 11 -- .../UdpTransport/UdpConnection.cpp | 7 +- .../AzNetworking/UdpTransport/UdpConnection.h | 14 +- .../UdpTransport/UdpNetworkInterface.cpp | 68 ++++----- .../UdpTransport/UdpNetworkInterface.h | 32 ++--- .../Source/MultiplayerSystemComponent.cpp | 1 - .../Code/Source/MultiplayerSystemComponent.h | 1 - .../NetworkEntityAuthorityTracker.cpp | 132 ++++++------------ .../NetworkEntityAuthorityTracker.h | 26 +--- .../NetworkEntity/NetworkEntityManager.cpp | 4 + 16 files changed, 104 insertions(+), 279 deletions(-) diff --git a/Code/Framework/AzCore/AzCore/EBus/ScheduledEventHandle.cpp b/Code/Framework/AzCore/AzCore/EBus/ScheduledEventHandle.cpp index 37b4895b4a..08a5a7d4ad 100644 --- a/Code/Framework/AzCore/AzCore/EBus/ScheduledEventHandle.cpp +++ b/Code/Framework/AzCore/AzCore/EBus/ScheduledEventHandle.cpp @@ -50,7 +50,7 @@ namespace AZ } else { - AZLOG_WARN("ScheduledEventHandle event pointer doesn't match to the pointer of handle to the event."); + //AZLOG_WARN("ScheduledEventHandle event pointer doesn't match to the pointer of handle to the event."); } } return false; // Event has been deleted, so the handle class must be deleted after this function. diff --git a/Code/Framework/AzNetworking/AzNetworking/AutoGen/CorePackets.AutoPackets.xml b/Code/Framework/AzNetworking/AzNetworking/AutoGen/CorePackets.AutoPackets.xml index 8ce3e5ad86..ae025b67e3 100644 --- a/Code/Framework/AzNetworking/AzNetworking/AutoGen/CorePackets.AutoPackets.xml +++ b/Code/Framework/AzNetworking/AzNetworking/AutoGen/CorePackets.AutoPackets.xml @@ -13,7 +13,9 @@ - + + + diff --git a/Code/Framework/AzNetworking/AzNetworking/TcpTransport/TcpConnection.cpp b/Code/Framework/AzNetworking/AzNetworking/TcpTransport/TcpConnection.cpp index 6d7358a425..7537232a27 100644 --- a/Code/Framework/AzNetworking/AzNetworking/TcpTransport/TcpConnection.cpp +++ b/Code/Framework/AzNetworking/AzNetworking/TcpTransport/TcpConnection.cpp @@ -27,13 +27,11 @@ namespace AzNetworking ConnectionId connectionId, const IpAddress& remoteAddress, TcpNetworkInterface& networkInterface, - TcpSocket& socket, - TimeoutId timeoutId + TcpSocket& socket ) : IConnection(connectionId, remoteAddress) , m_networkInterface(networkInterface) , m_socket(socket.CloneAndTakeOwnership()) - , m_timeoutId(timeoutId) , m_state(m_socket->IsOpen() ? ConnectionState::Connecting : ConnectionState::Disconnected) , m_connectionRole(ConnectionRole::Acceptor) , m_registeredSocketFd(InvalidSocketFd) @@ -163,13 +161,6 @@ namespace AzNetworking break; } - TimeoutQueue::TimeoutItem* timeoutItem = m_networkInterface.m_connectionTimeoutQueue.RetrieveItem(GetTimeoutId()); - if (timeoutItem == nullptr) - { - return true; - } - timeoutItem->UpdateTimeoutTime(startTimeMs); - NetworkOutputSerializer serializer(buffer.GetBuffer(), static_cast(buffer.GetSize())); if (m_state == ConnectionState::Connecting) { diff --git a/Code/Framework/AzNetworking/AzNetworking/TcpTransport/TcpConnection.h b/Code/Framework/AzNetworking/AzNetworking/TcpTransport/TcpConnection.h index b769aea086..3d74f3f336 100644 --- a/Code/Framework/AzNetworking/AzNetworking/TcpTransport/TcpConnection.h +++ b/Code/Framework/AzNetworking/AzNetworking/TcpTransport/TcpConnection.h @@ -38,14 +38,12 @@ namespace AzNetworking //! @param remoteAddress IP address of the remote endpoint //! @param networkInterface TcpNetworkInterface that owns this connection instance //! @param socket TCP socket to take ownership of and use for sending and receiving data - //! @param timeoutId timeout identifier of this connection instance TcpConnection ( ConnectionId connectionId, const IpAddress& remoteAddress, TcpNetworkInterface& networkInterface, - TcpSocket& socket, - TimeoutId timeoutId + TcpSocket& socket ); //! Construct a new socket with optional encryption, used when initiating a new connection @@ -69,14 +67,6 @@ namespace AzNetworking //! @return the TcpSocket bound to this TcpConnection TcpSocket* GetTcpSocket() const; - //! Sets the timeout identifier for this TcpConnection. - //! @param timeoutId the timeout identifier to use for this TcpConnection - void SetTimeoutId(TimeoutId timeoutId); - - //! Returns the timeout identifier for this TcpConnection. - //! @return the timeout identifier for this TcpConnection - TimeoutId GetTimeoutId() const; - //! Returns true if this connection instance is in an open state, and is capable of actively sending and receiving packets. //! @return boolean true if this connection instance is in an open state bool IsOpen() const; @@ -142,7 +132,6 @@ namespace AzNetworking AZStd::unique_ptr m_socket; AZStd::unique_ptr m_compressor; - TimeoutId m_timeoutId; PacketId m_lastSentPacketId = InvalidPacketId; ConnectionState m_state = ConnectionState::Disconnected; ConnectionRole m_connectionRole = ConnectionRole::Connector; diff --git a/Code/Framework/AzNetworking/AzNetworking/TcpTransport/TcpConnection.inl b/Code/Framework/AzNetworking/AzNetworking/TcpTransport/TcpConnection.inl index ecd1e5e908..5b5f38774e 100644 --- a/Code/Framework/AzNetworking/AzNetworking/TcpTransport/TcpConnection.inl +++ b/Code/Framework/AzNetworking/AzNetworking/TcpTransport/TcpConnection.inl @@ -15,16 +15,6 @@ namespace AzNetworking return m_socket.get(); } - inline void TcpConnection::SetTimeoutId(TimeoutId timeoutId) - { - m_timeoutId = timeoutId; - } - - inline TimeoutId TcpConnection::GetTimeoutId() const - { - return m_timeoutId; - } - inline bool TcpConnection::IsOpen() const { return m_socket->IsOpen(); diff --git a/Code/Framework/AzNetworking/AzNetworking/TcpTransport/TcpNetworkInterface.cpp b/Code/Framework/AzNetworking/AzNetworking/TcpTransport/TcpNetworkInterface.cpp index 1ccff7be50..0278856ce9 100644 --- a/Code/Framework/AzNetworking/AzNetworking/TcpTransport/TcpNetworkInterface.cpp +++ b/Code/Framework/AzNetworking/AzNetworking/TcpTransport/TcpNetworkInterface.cpp @@ -21,16 +21,11 @@ namespace AzNetworking static const bool net_TcpUseEncryption = false; #endif - AZ_CVAR(bool, net_TcpTimeoutConnections, true, nullptr, AZ::ConsoleFunctorFlags::DontReplicate, "Boolean value on whether we should timeout Tcp connections"); - AZ_CVAR(AZ::TimeMs, net_TcpHeartbeatTimeMs, AZ::TimeMs{ 2 * 1000 }, nullptr, AZ::ConsoleFunctorFlags::Null, "Tcp connection heartbeat frequency"); - AZ_CVAR(AZ::TimeMs, net_TcpDefaultTimeoutMs, AZ::TimeMs{ 10 * 1000 }, nullptr, AZ::ConsoleFunctorFlags::Null, "Time in milliseconds before we timeout an idle Tcp connection"); - TcpNetworkInterface::TcpNetworkInterface(AZ::Name name, IConnectionListener& connectionListener, TrustZone trustZone, TcpListenThread& listenThread) : m_name(name) , m_trustZone(trustZone) , m_connectionListener(connectionListener) , m_listenThread(listenThread) - , m_timeoutMs(net_TcpDefaultTimeoutMs) { ; } @@ -98,8 +93,6 @@ namespace AzNetworking } AZLOG_INFO("Adding new socket %d", static_cast(tcpSocket->GetSocketFd())); - const TimeoutId newTimeoutId = m_connectionTimeoutQueue.RegisterItem(static_cast(tcpSocket->GetSocketFd()), net_TcpHeartbeatTimeMs); - connection->SetTimeoutId(newTimeoutId); connection->SendReliablePacket(CorePackets::InitiateConnectionPacket()); m_connectionListener.OnConnect(connection.get()); m_connectionSet.AddConnection(AZStd::move(connection)); @@ -110,12 +103,6 @@ namespace AzNetworking { const AZ::TimeMs startTimeMs = AZ::GetElapsedTimeMs(); - // Time out any stale connections - { - ConnectionTimeoutFunctor functor(*this); - m_connectionTimeoutQueue.UpdateTimeouts(functor); - } - AcceptNewConnections(); auto readCallback = [this, startTimeMs](SocketFd socketFd) { HandleConnectionRecv(socketFd, startTimeMs); }; @@ -258,8 +245,7 @@ namespace AzNetworking return; } AZLOG(NET_TcpTraffic, "Adding new socket %d", static_cast(tcpSocket.GetSocketFd())); - const TimeoutId timeoutId = m_connectionTimeoutQueue.RegisterItem(static_cast(tcpSocket.GetSocketFd()), m_timeoutMs); - AZStd::unique_ptr connection = AZStd::make_unique(connectionId, remoteAddress, *this, tcpSocket, timeoutId); + AZStd::unique_ptr connection = AZStd::make_unique(connectionId, remoteAddress, *this, tcpSocket); AZ_Assert(connection->GetConnectionRole() == ConnectionRole::Acceptor, "Invalid role for connection"); GetConnectionListener().OnConnect(connection.get()); m_connectionSet.AddConnection(AZStd::move(connection)); @@ -286,7 +272,6 @@ namespace AzNetworking m_pendingRemoves.resize_no_construct(0); } - TcpNetworkInterface::PendingConnection::PendingConnection(SocketFd socketFd, uint32_t remoteIpAddress, uint16_t remotePort, uint16_t listenPort) : m_socketFd(socketFd) , m_remoteIpAddress(remoteIpAddress) @@ -295,34 +280,4 @@ namespace AzNetworking { ; } - - TcpNetworkInterface::ConnectionTimeoutFunctor::ConnectionTimeoutFunctor(TcpNetworkInterface& networkInterface) - : m_networkInterface(networkInterface) - { - ; - } - - TimeoutResult TcpNetworkInterface::ConnectionTimeoutFunctor::HandleTimeout(TimeoutQueue::TimeoutItem& item) - { - const SocketFd socketFd = static_cast(item.m_userData); - TcpConnection* tcpConnection = m_networkInterface.m_connectionSet.GetConnection(socketFd); - - if (tcpConnection == nullptr) - { - // We've already deleted this connection - return TimeoutResult::Delete; - } - - if (tcpConnection->GetConnectionRole() == ConnectionRole::Connector) - { - tcpConnection->SendReliablePacket(CorePackets::HeartbeatPacket()); - } - else if (net_TcpTimeoutConnections && (m_networkInterface.GetTimeoutMs() > AZ::TimeMs{ 0 })) - { - tcpConnection->Disconnect(DisconnectReason::Timeout, TerminationEndpoint::Local); - return TimeoutResult::Delete; - } - - return TimeoutResult::Refresh; - } } diff --git a/Code/Framework/AzNetworking/AzNetworking/TcpTransport/TcpNetworkInterface.h b/Code/Framework/AzNetworking/AzNetworking/TcpTransport/TcpNetworkInterface.h index d8f5d1b62b..8d45e847a8 100644 --- a/Code/Framework/AzNetworking/AzNetworking/TcpTransport/TcpNetworkInterface.h +++ b/Code/Framework/AzNetworking/AzNetworking/TcpTransport/TcpNetworkInterface.h @@ -137,16 +137,6 @@ namespace AzNetworking AZ_DISABLE_COPY_MOVE(TcpNetworkInterface); - struct ConnectionTimeoutFunctor final - : public ITimeoutHandler - { - ConnectionTimeoutFunctor(TcpNetworkInterface& networkInterface); - TimeoutResult HandleTimeout(TimeoutQueue::TimeoutItem& item) override; - private: - AZ_DISABLE_COPY_MOVE(ConnectionTimeoutFunctor); - TcpNetworkInterface& m_networkInterface; - }; - struct PendingRemove { SocketFd m_socketFd; @@ -162,7 +152,6 @@ namespace AzNetworking TcpSocketManager m_tcpSocketManager; AZ::ThreadSafeDeque m_pendingConnections; AZStd::vector m_pendingRemoves; - TimeoutQueue m_connectionTimeoutQueue; TcpListenThread& m_listenThread; friend class TcpConnection; // For access to private RequestDisconnect() method diff --git a/Code/Framework/AzNetworking/AzNetworking/UdpTransport/UdpConnection.cpp b/Code/Framework/AzNetworking/AzNetworking/UdpTransport/UdpConnection.cpp index 7b451865c4..03d4e7bec9 100644 --- a/Code/Framework/AzNetworking/AzNetworking/UdpTransport/UdpConnection.cpp +++ b/Code/Framework/AzNetworking/AzNetworking/UdpTransport/UdpConnection.cpp @@ -79,7 +79,7 @@ namespace AzNetworking AZLOG(NET_Acks, "Unacked packet count exceeded, sending client heartbeat (curr %u : max %u)", m_unackedPacketCount, static_cast(net_UdpMaxUnackedPacketCount)); // This simply times out unreliable chunks that haven't completed within our timeout delay m_fragmentQueue.Update(); - SendUnreliablePacket(CorePackets::HeartbeatPacket()); + SendUnreliablePacket(CorePackets::HeartbeatPacket(false)); } } @@ -289,7 +289,10 @@ namespace AzNetworking { return PacketDispatchResult::Failure; } - // Do nothing, we've already processed our ack packets + if (packet.GetRequestResponse()) + { + SendUnreliablePacket(CorePackets::HeartbeatPacket(false)); + } return PacketDispatchResult::Success; } break; diff --git a/Code/Framework/AzNetworking/AzNetworking/UdpTransport/UdpConnection.h b/Code/Framework/AzNetworking/AzNetworking/UdpTransport/UdpConnection.h index 199a5a8347..c728016239 100644 --- a/Code/Framework/AzNetworking/AzNetworking/UdpTransport/UdpConnection.h +++ b/Code/Framework/AzNetworking/AzNetworking/UdpTransport/UdpConnection.h @@ -136,19 +136,19 @@ namespace AzNetworking AZ_DISABLE_COPY_MOVE(UdpConnection); UdpNetworkInterface& m_networkInterface; - UdpPacketTracker m_packetTracker; - UdpReliableQueue m_reliableQueue; - UdpFragmentQueue m_fragmentQueue; - ConnectionState m_state = ConnectionState::Disconnected; - ConnectionRole m_connectionRole = ConnectionRole::Connector; - DtlsEndpoint m_dtlsEndpoint; + UdpPacketTracker m_packetTracker; + UdpReliableQueue m_reliableQueue; + UdpFragmentQueue m_fragmentQueue; + ConnectionState m_state = ConnectionState::Disconnected; + ConnectionRole m_connectionRole = ConnectionRole::Connector; + DtlsEndpoint m_dtlsEndpoint; AZ::TimeMs m_lastSentPacketMs; uint32_t m_unackedPacketCount = 0; uint32_t m_connectionMtu = MaxUdpTransmissionUnit; TimeoutId m_timeoutId; - uint32_t m_timeoutCounter = 0; + int32_t m_timeoutCounter = 0; }; } diff --git a/Code/Framework/AzNetworking/AzNetworking/UdpTransport/UdpNetworkInterface.cpp b/Code/Framework/AzNetworking/AzNetworking/UdpTransport/UdpNetworkInterface.cpp index b0e64f93e3..67bb80a44c 100644 --- a/Code/Framework/AzNetworking/AzNetworking/UdpTransport/UdpNetworkInterface.cpp +++ b/Code/Framework/AzNetworking/AzNetworking/UdpTransport/UdpNetworkInterface.cpp @@ -31,7 +31,7 @@ namespace AzNetworking AZ_CVAR(bool, net_UdpTimeoutConnections, true, nullptr, AZ::ConsoleFunctorFlags::DontReplicate, "Boolean value on whether we should timeout Udp connections"); AZ_CVAR(AZ::TimeMs, net_UdpPacketTimeSliceMs, AZ::TimeMs{ 8 }, nullptr, AZ::ConsoleFunctorFlags::DontReplicate, "The number of milliseconds to allow for packet processing"); - AZ_CVAR(AZ::TimeMs, net_UdpHeartbeatTimeMs, AZ::TimeMs{ 2 * 1000 }, nullptr, AZ::ConsoleFunctorFlags::Null, "Udp connection heartbeat frequency"); + AZ_CVAR(int32_t, net_UdpUnackedHeartbeats, 3, nullptr, AZ::ConsoleFunctorFlags::Null, "The number of heartbeats to attempt to send to keep a connection alive before giving up"); AZ_CVAR(AZ::TimeMs, net_UdpDefaultTimeoutMs, AZ::TimeMs{ 10 * 1000 }, nullptr, AZ::ConsoleFunctorFlags::Null, "Time in milliseconds before we timeout an idle Udp connection"); AZ_CVAR(AZ::TimeMs, net_MinPacketTimeoutMs, AZ::TimeMs{ 200 }, nullptr, AZ::ConsoleFunctorFlags::DontReplicate, "Minimum time to wait before timing out an unacked packet"); AZ_CVAR(int32_t, net_MaxTimeoutsPerFrame, 1000, nullptr, AZ::ConsoleFunctorFlags::DontReplicate, "Maximum number of packet timeouts to allow to process in a single frame"); @@ -139,7 +139,8 @@ namespace AzNetworking } const ConnectionId connectionId = m_connectionSet.GetNextConnectionId(); - const TimeoutId timeoutId = m_connectionTimeoutQueue.RegisterItem(aznumeric_cast(connectionId), m_timeoutMs); + const AZ::TimeMs timeoutTimeMs = m_timeoutMs / static_cast(static_cast(net_UdpUnackedHeartbeats)); + const TimeoutId timeoutId = m_connectionTimeoutQueue.RegisterItem(aznumeric_cast(connectionId), timeoutTimeMs); AZStd::unique_ptr connection = AZStd::make_unique(connectionId, remoteAddress, *this, ConnectionRole::Connector); UdpPacketEncodingBuffer dtlsData; @@ -277,6 +278,7 @@ namespace AzNetworking } timeoutItem->UpdateTimeoutTime(startTimeMs); + connection->m_timeoutCounter = 0; PacketDispatchResult handledPacket = PacketDispatchResult::Failure; if (header.GetPacketType() < aznumeric_cast(CorePackets::PacketType::MAX)) @@ -319,16 +321,10 @@ namespace AzNetworking const AZ::TimeMs receiveTimeMs = AZ::GetElapsedTimeMs() - startTimeMs; // Time out any stale client connections - { - ConnectionTimeoutFunctor functor(*this); - m_connectionTimeoutQueue.UpdateTimeouts(functor); - } + m_connectionTimeoutQueue.UpdateTimeouts([this](TimeoutQueue::TimeoutItem& item) { return HandleConnectionTimeout(item); }); // Time out any packets that haven't been acked within our timeout window - { - PacketTimeoutFunctor functor(*this); - m_packetTimeoutQueue.UpdateTimeouts(functor, static_cast(net_MaxTimeoutsPerFrame)); - } + m_packetTimeoutQueue.UpdateTimeouts([this](TimeoutQueue::TimeoutItem& item) { return HandlePacketTimeout(item); }, static_cast(net_MaxTimeoutsPerFrame)); // Delete any connections we've disconnected for (RemovedConnection& removedConnection : m_removedConnections) @@ -709,21 +705,14 @@ namespace AzNetworking { // Packets involved in handshake are InitiateConnection, ConnectionHandshake and FragmentedPackets of ConnectionHandshake return packetType == aznumeric_cast(CorePackets::PacketType::InitiateConnectionPacket) || - packetType == aznumeric_cast(CorePackets::PacketType::ConnectionHandshakePacket) || - (packetType == aznumeric_cast(CorePackets::PacketType::FragmentedPacket) && endpoint.IsConnecting()); + packetType == aznumeric_cast(CorePackets::PacketType::ConnectionHandshakePacket) || + (packetType == aznumeric_cast(CorePackets::PacketType::FragmentedPacket) && endpoint.IsConnecting()); } - - UdpNetworkInterface::ConnectionTimeoutFunctor::ConnectionTimeoutFunctor(UdpNetworkInterface& networkInterface) - : m_networkInterface(networkInterface) - { - ; - } - - TimeoutResult UdpNetworkInterface::ConnectionTimeoutFunctor::HandleTimeout(TimeoutQueue::TimeoutItem& item) + TimeoutResult UdpNetworkInterface::HandleConnectionTimeout(TimeoutQueue::TimeoutItem& item) { const ConnectionId connectionId = ConnectionId(aznumeric_cast(item.m_userData)); - UdpConnection* udpConnection = static_cast(m_networkInterface.m_connectionSet.GetConnection(connectionId)); + UdpConnection* udpConnection = static_cast(m_connectionSet.GetConnection(connectionId)); if (udpConnection == nullptr) { @@ -731,22 +720,23 @@ namespace AzNetworking return TimeoutResult::Delete; } - if (udpConnection->GetConnectionState() == ConnectionState::Connecting) + if ((udpConnection->GetConnectionState() == ConnectionState::Connecting) + && udpConnection->GetDtlsEndpoint().IsConnecting()) { - if (udpConnection->GetDtlsEndpoint().IsConnecting()) - { - // DTLS prefers we resend data lost over the wire with fresh SSL IDs so account for that here - UdpPacketEncodingBuffer dtlsData; - udpConnection->ProcessHandshakeData(dtlsData); - return TimeoutResult::Refresh; - } + // DTLS prefers we resend data lost over the wire with fresh SSL IDs so account for that here + UdpPacketEncodingBuffer dtlsData; + udpConnection->ProcessHandshakeData(dtlsData); + return TimeoutResult::Refresh; } - if (udpConnection->GetConnectionRole() == ConnectionRole::Connector) + if ((udpConnection->GetConnectionRole() == ConnectionRole::Connector) + && (udpConnection->m_timeoutCounter < net_UdpUnackedHeartbeats)) { - udpConnection->SendUnreliablePacket(CorePackets::HeartbeatPacket()); + // Set the request response flag to true since we want a response to keep the connection alive + udpConnection->SendUnreliablePacket(CorePackets::HeartbeatPacket(true)); + ++udpConnection->m_timeoutCounter; } - else if (net_UdpTimeoutConnections && (m_networkInterface.GetTimeoutMs() > AZ::TimeMs{ 0 })) + else if (net_UdpTimeoutConnections && (GetTimeoutMs() > AZ::TimeMs{ 0 })) { udpConnection->Disconnect(DisconnectReason::Timeout, TerminationEndpoint::Local); return TimeoutResult::Delete; @@ -755,19 +745,13 @@ namespace AzNetworking return TimeoutResult::Refresh; } - UdpNetworkInterface::PacketTimeoutFunctor::PacketTimeoutFunctor(UdpNetworkInterface& networkInterface) - : m_networkInterface(networkInterface) - { - ; - } - - TimeoutResult UdpNetworkInterface::PacketTimeoutFunctor::HandleTimeout(TimeoutQueue::TimeoutItem& item) + TimeoutResult UdpNetworkInterface::HandlePacketTimeout(TimeoutQueue::TimeoutItem& item) { ConnectionId connectionId; PacketId packetId; ReliabilityType reliability; DecodeTimeoutId(item.m_userData, connectionId, packetId, reliability); - UdpConnection* connection = static_cast(m_networkInterface.m_connectionSet.GetConnection(connectionId)); + UdpConnection* connection = static_cast(m_connectionSet.GetConnection(connectionId)); if (connection == nullptr) { @@ -782,16 +766,14 @@ namespace AzNetworking case PacketTimeoutResult::Acked: // Packet was already acked, just discard this timeout entry return TimeoutResult::Delete; - case PacketTimeoutResult::Pending: // Packet timed out before we received any info about it's sequence from the remote endpoint // The connection latency may have increased, and our Rtt metrics may still be adjusting.. // Just throw it back into the timeout queue return TimeoutResult::Refresh; - case PacketTimeoutResult::Lost: // Packet timed out and was not acked, so we consider it lost - m_networkInterface.m_connectionListener.OnPacketLost(connection, packetId); + m_connectionListener.OnPacketLost(connection, packetId); break; } diff --git a/Code/Framework/AzNetworking/AzNetworking/UdpTransport/UdpNetworkInterface.h b/Code/Framework/AzNetworking/AzNetworking/UdpTransport/UdpNetworkInterface.h index 8f827c74c4..e6abeded0d 100644 --- a/Code/Framework/AzNetworking/AzNetworking/UdpTransport/UdpNetworkInterface.h +++ b/Code/Framework/AzNetworking/AzNetworking/UdpTransport/UdpNetworkInterface.h @@ -149,34 +149,24 @@ namespace AzNetworking //! @param endpoint whether the disconnection was initiated locally or remotely void RequestDisconnect(UdpConnection* connection, DisconnectReason reason, TerminationEndpoint endpoint); - //! Internal helper to check if a packet's type is for connection handshake + //! Internal helper to check if a packet's type is for connection handshake. //! @param endpoint DTLS endpoint participating in the handshake //! @param packetType type of the packet //! @return if the packet is for handshake bool IsHandshakePacket(const DtlsEndpoint& endpoint, AzNetworking::PacketType packetType) const; + //! Internal helper to manage connection timeout behaviour. + //! @param item the timeout item corresponding to the timed out connection + //! @return whether to delete or persist the timeout item + TimeoutResult HandleConnectionTimeout(TimeoutQueue::TimeoutItem& item); + + //! Internal helper to manage packet timeout behaviour. + //! @param item the timeout item corresponding to the timed out packet + //! @return whether to delete or persist the timeout item + TimeoutResult HandlePacketTimeout(TimeoutQueue::TimeoutItem& item); + AZ_DISABLE_COPY_MOVE(UdpNetworkInterface); - struct ConnectionTimeoutFunctor final - : public ITimeoutHandler - { - ConnectionTimeoutFunctor(UdpNetworkInterface& networkInterface); - TimeoutResult HandleTimeout(TimeoutQueue::TimeoutItem& item) override; - private: - AZ_DISABLE_COPY_MOVE(ConnectionTimeoutFunctor); - UdpNetworkInterface& m_networkInterface; - }; - - struct PacketTimeoutFunctor final - : public ITimeoutHandler - { - PacketTimeoutFunctor(UdpNetworkInterface& networkInterface); - TimeoutResult HandleTimeout(TimeoutQueue::TimeoutItem& item) override; - private: - AZ_DISABLE_COPY_MOVE(PacketTimeoutFunctor); - UdpNetworkInterface& m_networkInterface; - }; - AZ::Name m_name; TrustZone m_trustZone; uint16_t m_port = 0; diff --git a/Gems/Multiplayer/Code/Source/MultiplayerSystemComponent.cpp b/Gems/Multiplayer/Code/Source/MultiplayerSystemComponent.cpp index fdd7602f71..0bf5cea64b 100644 --- a/Gems/Multiplayer/Code/Source/MultiplayerSystemComponent.cpp +++ b/Gems/Multiplayer/Code/Source/MultiplayerSystemComponent.cpp @@ -1107,7 +1107,6 @@ namespace Multiplayer void MultiplayerSystemComponent::OnAutonomousEntityReplicatorCreated() { m_autonomousEntityReplicatorCreatedHandler.Disconnect(); - //m_networkEntityManager.GetNetworkEntityAuthorityTracker()->ResetTimeoutTime(AZ::TimeMs{ 2000 }); m_clientMigrationEndEvent.Signal(); } diff --git a/Gems/Multiplayer/Code/Source/MultiplayerSystemComponent.h b/Gems/Multiplayer/Code/Source/MultiplayerSystemComponent.h index 87d084d5bc..53707523bb 100644 --- a/Gems/Multiplayer/Code/Source/MultiplayerSystemComponent.h +++ b/Gems/Multiplayer/Code/Source/MultiplayerSystemComponent.h @@ -155,7 +155,6 @@ namespace Multiplayer AZ_CONSOLEFUNC(MultiplayerSystemComponent, DumpStats, AZ::ConsoleFunctorFlags::Null, "Dumps stats for the current multiplayer session"); AzNetworking::INetworkInterface* m_networkInterface = nullptr; - AzNetworking::INetworkInterface* m_networkEditorInterface = nullptr; AZ::ConsoleCommandInvokedEvent::Handler m_consoleCommandHandler; AZ::ThreadSafeDeque m_cvarCommands; diff --git a/Gems/Multiplayer/Code/Source/NetworkEntity/NetworkEntityAuthorityTracker.cpp b/Gems/Multiplayer/Code/Source/NetworkEntity/NetworkEntityAuthorityTracker.cpp index b3f87ea9ab..7b6e8476e7 100644 --- a/Gems/Multiplayer/Code/Source/NetworkEntity/NetworkEntityAuthorityTracker.cpp +++ b/Gems/Multiplayer/Code/Source/NetworkEntity/NetworkEntityAuthorityTracker.cpp @@ -11,6 +11,7 @@ #include #include #include +#include #include #include @@ -33,37 +34,21 @@ namespace Multiplayer AZLOG ( NET_AuthTracker, - "AuthTracker: Removing timeout for networkEntityId %llu from %s, new owner is %s", + "AuthTracker: Removing timeout for networkEntityId %llu, new owner is %s", aznumeric_cast(entityHandle.GetNetEntityId()), - timeoutData->second.m_previousOwner.GetString().c_str(), newOwner.GetString().c_str() ); m_timeoutDataMap.erase(timeoutData); ret = true; } - auto iter = m_entityAuthorityMap.find(entityHandle.GetNetEntityId()); - if (iter != m_entityAuthorityMap.end()) - { - AZLOG - ( - NET_AuthTracker, - "AuthTracker: Assigning networkEntityId %llu from %s to %s", - aznumeric_cast(entityHandle.GetNetEntityId()), - iter->second.back().GetString().c_str(), - newOwner.GetString().c_str() - ); - } - else - { - AZLOG - ( - NET_AuthTracker, - "AuthTracker: Assigning networkEntityId %llu to %s", - aznumeric_cast(entityHandle.GetNetEntityId()), - newOwner.GetString().c_str() - ); - } + AZLOG + ( + NET_AuthTracker, + "AuthTracker: Assigning networkEntityId %llu to %s", + aznumeric_cast(entityHandle.GetNetEntityId()), + newOwner.GetString().c_str() + ); m_entityAuthorityMap[entityHandle.GetNetEntityId()].push_back(newOwner); return ret; @@ -103,14 +88,41 @@ namespace Multiplayer { AZ_Assert ( - (m_timeoutDataMap.find(entityHandle.GetNetEntityId()) == m_timeoutDataMap.end()) || - (m_timeoutDataMap[entityHandle.GetNetEntityId()].m_previousOwner == previousOwner), + m_timeoutDataMap.find(entityHandle.GetNetEntityId()) == m_timeoutDataMap.end(), "Trying to add something twice to the timeout map, this is unexpected" ); - m_timeoutQueue.RegisterItem(aznumeric_cast(entityHandle.GetNetEntityId()), net_EntityMigrationTimeoutMs); - TimeoutData& timeoutData = m_timeoutDataMap[entityHandle.GetNetEntityId()]; - timeoutData.m_entityHandle = entityHandle; - timeoutData.m_previousOwner = previousOwner; + m_timeoutDataMap.insert(entityHandle.GetNetEntityId()); + AZ::Interface::Get()->AddCallback([this, netEntityId = entityHandle.GetNetEntityId(), previousOwner] + { + auto timeoutData = m_timeoutDataMap.find(netEntityId); + if (timeoutData != m_timeoutDataMap.end()) + { + m_timeoutDataMap.erase(timeoutData); + ConstNetworkEntityHandle entityHandle = m_networkEntityManager.GetEntity(netEntityId); + if (auto entity = entityHandle.GetEntity()) + { + NetEntityRole networkRole = NetEntityRole::InvalidRole; + NetBindComponent* netBindComponent = entityHandle.GetNetBindComponent(); + if (netBindComponent != nullptr) + { + networkRole = netBindComponent->GetNetEntityRole(); + } + if (networkRole != NetEntityRole::Authority) + { + AZLOG_ERROR + ( + "Timed out entity id %llu during migration previous owner %s, removing it", + aznumeric_cast(entityHandle.GetNetEntityId()), + previousOwner.GetString().c_str() + ); + m_networkEntityManager.MarkForRemoval(entityHandle); + } + } + } + }, + AZ::Name("Entity authority removal functor"), + net_EntityMigrationTimeoutMs + ); } else { @@ -127,18 +139,6 @@ namespace Multiplayer } HostId NetworkEntityAuthorityTracker::GetEntityAuthorityManager(ConstNetworkEntityHandle entityHandle) const - { - HostId hostId = GetEntityAuthorityManagerInternal(entityHandle); - AZ_Assert(hostId != InvalidHostId, "Unable to determine manager for entity"); - return hostId; - } - - bool NetworkEntityAuthorityTracker::DoesEntityHaveOwner(ConstNetworkEntityHandle entityHandle) const - { - return InvalidHostId != GetEntityAuthorityManagerInternal(entityHandle); - } - - HostId NetworkEntityAuthorityTracker::GetEntityAuthorityManagerInternal(ConstNetworkEntityHandle entityHandle) const { if (auto localEnt = entityHandle.GetEntity()) { @@ -167,52 +167,8 @@ namespace Multiplayer return InvalidHostId; } - NetworkEntityAuthorityTracker::TimeoutData::TimeoutData(ConstNetworkEntityHandle entityHandle, const HostId& previousOwner) - : m_entityHandle(entityHandle) - , m_previousOwner(previousOwner) + bool NetworkEntityAuthorityTracker::DoesEntityHaveOwner(ConstNetworkEntityHandle entityHandle) const { - ; - } - - NetworkEntityAuthorityTracker::NetworkEntityTimeoutFunctor::NetworkEntityTimeoutFunctor - ( - NetworkEntityAuthorityTracker& networkEntityAuthorityTracker, - INetworkEntityManager& networkEntityManager - ) - : m_networkEntityAuthorityTracker(networkEntityAuthorityTracker) - , m_networkEntityManager(networkEntityManager) - { - ; - } - - AzNetworking::TimeoutResult NetworkEntityAuthorityTracker::NetworkEntityTimeoutFunctor::HandleTimeout(AzNetworking::TimeoutQueue::TimeoutItem& item) - { - const NetEntityId netEntityId = aznumeric_cast(item.m_userData); - auto timeoutData = m_networkEntityAuthorityTracker.m_timeoutDataMap.find(netEntityId); - if (timeoutData != m_networkEntityAuthorityTracker.m_timeoutDataMap.end()) - { - m_networkEntityAuthorityTracker.m_timeoutDataMap.erase(timeoutData); - ConstNetworkEntityHandle entityHandle = m_networkEntityManager.GetEntity(netEntityId); - if (auto entity = entityHandle.GetEntity()) - { - NetEntityRole networkRole = NetEntityRole::InvalidRole; - NetBindComponent* netBindComponent = entityHandle.GetNetBindComponent(); - if (netBindComponent != nullptr) - { - networkRole = netBindComponent->GetNetEntityRole(); - } - if (networkRole != NetEntityRole::Authority) - { - AZLOG_ERROR - ( - "Timed out entity id %llu during migration previous owner %s, removing it", - aznumeric_cast(entityHandle.GetNetEntityId()), - timeoutData->second.m_previousOwner.GetString().c_str() - ); - m_networkEntityManager.MarkForRemoval(entityHandle); - } - } - } - return AzNetworking::TimeoutResult::Delete; + return InvalidHostId != GetEntityAuthorityManager(entityHandle); } } diff --git a/Gems/Multiplayer/Code/Source/NetworkEntity/NetworkEntityAuthorityTracker.h b/Gems/Multiplayer/Code/Source/NetworkEntity/NetworkEntityAuthorityTracker.h index 0f4ff5665a..c2e330ab4d 100644 --- a/Gems/Multiplayer/Code/Source/NetworkEntity/NetworkEntityAuthorityTracker.h +++ b/Gems/Multiplayer/Code/Source/NetworkEntity/NetworkEntityAuthorityTracker.h @@ -29,37 +29,13 @@ namespace Multiplayer HostId GetEntityAuthorityManager(ConstNetworkEntityHandle entityHandle) const; private: - - HostId GetEntityAuthorityManagerInternal(ConstNetworkEntityHandle entityHandle) const; - NetworkEntityAuthorityTracker& operator= (const NetworkEntityAuthorityTracker&) = delete; - struct TimeoutData final - { - TimeoutData() = default; - TimeoutData(ConstNetworkEntityHandle entityHandle, const HostId& previousOwner); - ConstNetworkEntityHandle m_entityHandle; - HostId m_previousOwner = InvalidHostId; - }; - - struct NetworkEntityTimeoutFunctor final - : public AzNetworking::ITimeoutHandler - { - NetworkEntityTimeoutFunctor(NetworkEntityAuthorityTracker& networkEntityAuthorityTracker, INetworkEntityManager& m_networkEntityManager); - AzNetworking::TimeoutResult HandleTimeout(AzNetworking::TimeoutQueue::TimeoutItem& item) override; - private: - AZ_DISABLE_COPY_MOVE(NetworkEntityTimeoutFunctor); - NetworkEntityAuthorityTracker& m_networkEntityAuthorityTracker; - INetworkEntityManager& m_networkEntityManager; - }; - - using TimeoutDataMap = AZStd::unordered_map; + using TimeoutDataMap = AZStd::unordered_set; using EntityAuthorityMap = AZStd::unordered_map>; TimeoutDataMap m_timeoutDataMap; EntityAuthorityMap m_entityAuthorityMap; INetworkEntityManager& m_networkEntityManager; - AzNetworking::TimeoutQueue m_timeoutQueue; }; } - diff --git a/Gems/Multiplayer/Code/Source/NetworkEntity/NetworkEntityManager.cpp b/Gems/Multiplayer/Code/Source/NetworkEntity/NetworkEntityManager.cpp index c7582af83f..d973f0c80a 100644 --- a/Gems/Multiplayer/Code/Source/NetworkEntity/NetworkEntityManager.cpp +++ b/Gems/Multiplayer/Code/Source/NetworkEntity/NetworkEntityManager.cpp @@ -241,6 +241,10 @@ namespace Multiplayer { AZ::Entity* entity = it->second; NetBindComponent* netBindComponent = m_networkEntityTracker.GetNetBindComponent(entity); + if (netBindComponent == nullptr) + { + continue; + } AZ::Aabb entityBounds = AZ::Interface::Get()->GetEntityWorldBoundsUnion(entity->GetId()); entityBounds.Expand(AZ::Vector3(0.01f)); if (netBindComponent->GetNetEntityRole() == NetEntityRole::Authority) From fda7a6353e758eeef016ee2c75c130aff7bf1344 Mon Sep 17 00:00:00 2001 From: kberg-amzn Date: Wed, 3 Nov 2021 14:20:56 -0700 Subject: [PATCH 04/34] Backing out some temporary debugging code Signed-off-by: kberg-amzn --- Code/Framework/AzCore/AzCore/EBus/ScheduledEventHandle.cpp | 2 +- .../Code/Source/NetworkEntity/NetworkEntityManager.cpp | 4 ---- 2 files changed, 1 insertion(+), 5 deletions(-) diff --git a/Code/Framework/AzCore/AzCore/EBus/ScheduledEventHandle.cpp b/Code/Framework/AzCore/AzCore/EBus/ScheduledEventHandle.cpp index 08a5a7d4ad..37b4895b4a 100644 --- a/Code/Framework/AzCore/AzCore/EBus/ScheduledEventHandle.cpp +++ b/Code/Framework/AzCore/AzCore/EBus/ScheduledEventHandle.cpp @@ -50,7 +50,7 @@ namespace AZ } else { - //AZLOG_WARN("ScheduledEventHandle event pointer doesn't match to the pointer of handle to the event."); + AZLOG_WARN("ScheduledEventHandle event pointer doesn't match to the pointer of handle to the event."); } } return false; // Event has been deleted, so the handle class must be deleted after this function. diff --git a/Gems/Multiplayer/Code/Source/NetworkEntity/NetworkEntityManager.cpp b/Gems/Multiplayer/Code/Source/NetworkEntity/NetworkEntityManager.cpp index d973f0c80a..c7582af83f 100644 --- a/Gems/Multiplayer/Code/Source/NetworkEntity/NetworkEntityManager.cpp +++ b/Gems/Multiplayer/Code/Source/NetworkEntity/NetworkEntityManager.cpp @@ -241,10 +241,6 @@ namespace Multiplayer { AZ::Entity* entity = it->second; NetBindComponent* netBindComponent = m_networkEntityTracker.GetNetBindComponent(entity); - if (netBindComponent == nullptr) - { - continue; - } AZ::Aabb entityBounds = AZ::Interface::Get()->GetEntityWorldBoundsUnion(entity->GetId()); entityBounds.Expand(AZ::Vector3(0.01f)); if (netBindComponent->GetNetEntityRole() == NetEntityRole::Authority) From 70a1eb65d81079b28d84e15d5ead7f53758c1801 Mon Sep 17 00:00:00 2001 From: kberg-amzn Date: Wed, 3 Nov 2021 14:26:41 -0700 Subject: [PATCH 05/34] Improving comments around heartbeat sends + bumping number of heartbeats for increased keep-alive robustness under high packet loss Signed-off-by: kberg-amzn --- .../AzNetworking/AzNetworking/UdpTransport/UdpConnection.cpp | 2 ++ .../AzNetworking/UdpTransport/UdpNetworkInterface.cpp | 2 +- 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/Code/Framework/AzNetworking/AzNetworking/UdpTransport/UdpConnection.cpp b/Code/Framework/AzNetworking/AzNetworking/UdpTransport/UdpConnection.cpp index 03d4e7bec9..452992f971 100644 --- a/Code/Framework/AzNetworking/AzNetworking/UdpTransport/UdpConnection.cpp +++ b/Code/Framework/AzNetworking/AzNetworking/UdpTransport/UdpConnection.cpp @@ -79,6 +79,7 @@ namespace AzNetworking AZLOG(NET_Acks, "Unacked packet count exceeded, sending client heartbeat (curr %u : max %u)", m_unackedPacketCount, static_cast(net_UdpMaxUnackedPacketCount)); // This simply times out unreliable chunks that haven't completed within our timeout delay m_fragmentQueue.Update(); + // This heartbeat is sent to minimize the time the remote endpoint spends waiting for ack vector replication, we don't require a response SendUnreliablePacket(CorePackets::HeartbeatPacket(false)); } } @@ -291,6 +292,7 @@ namespace AzNetworking } if (packet.GetRequestResponse()) { + // We're replying to a heartbeat request, we don't want a response SendUnreliablePacket(CorePackets::HeartbeatPacket(false)); } return PacketDispatchResult::Success; diff --git a/Code/Framework/AzNetworking/AzNetworking/UdpTransport/UdpNetworkInterface.cpp b/Code/Framework/AzNetworking/AzNetworking/UdpTransport/UdpNetworkInterface.cpp index 67bb80a44c..280b749d9e 100644 --- a/Code/Framework/AzNetworking/AzNetworking/UdpTransport/UdpNetworkInterface.cpp +++ b/Code/Framework/AzNetworking/AzNetworking/UdpTransport/UdpNetworkInterface.cpp @@ -31,7 +31,7 @@ namespace AzNetworking AZ_CVAR(bool, net_UdpTimeoutConnections, true, nullptr, AZ::ConsoleFunctorFlags::DontReplicate, "Boolean value on whether we should timeout Udp connections"); AZ_CVAR(AZ::TimeMs, net_UdpPacketTimeSliceMs, AZ::TimeMs{ 8 }, nullptr, AZ::ConsoleFunctorFlags::DontReplicate, "The number of milliseconds to allow for packet processing"); - AZ_CVAR(int32_t, net_UdpUnackedHeartbeats, 3, nullptr, AZ::ConsoleFunctorFlags::Null, "The number of heartbeats to attempt to send to keep a connection alive before giving up"); + AZ_CVAR(int32_t, net_UdpUnackedHeartbeats, 5, nullptr, AZ::ConsoleFunctorFlags::Null, "The number of heartbeats to attempt to send to keep a connection alive before giving up"); AZ_CVAR(AZ::TimeMs, net_UdpDefaultTimeoutMs, AZ::TimeMs{ 10 * 1000 }, nullptr, AZ::ConsoleFunctorFlags::Null, "Time in milliseconds before we timeout an idle Udp connection"); AZ_CVAR(AZ::TimeMs, net_MinPacketTimeoutMs, AZ::TimeMs{ 200 }, nullptr, AZ::ConsoleFunctorFlags::DontReplicate, "Minimum time to wait before timing out an unacked packet"); AZ_CVAR(int32_t, net_MaxTimeoutsPerFrame, 1000, nullptr, AZ::ConsoleFunctorFlags::DontReplicate, "Maximum number of packet timeouts to allow to process in a single frame"); From 8a3d055f8b7c4654dcb4285026f5b9d089d407d8 Mon Sep 17 00:00:00 2001 From: kberg-amzn Date: Wed, 3 Nov 2021 19:34:10 -0700 Subject: [PATCH 06/34] Some cleanup around handling of migrations to simplify interfaces and add additional hooks for functionality Signed-off-by: kberg-amzn --- .../Multiplayer/EntityDomains/IEntityDomain.h | 12 +- .../NetworkEntity/INetworkEntityManager.h | 23 ++- .../Source/Components/NetBindComponent.cpp | 4 +- .../FullOwnershipEntityDomain.cpp | 9 +- .../EntityDomains/FullOwnershipEntityDomain.h | 6 +- .../Source/EntityDomains/NullEntityDomain.cpp | 41 +++++ .../Source/EntityDomains/NullEntityDomain.h | 31 ++++ .../Source/MultiplayerSystemComponent.cpp | 10 +- .../NetworkEntityAuthorityTracker.cpp | 21 +-- .../NetworkEntityAuthorityTracker.h | 3 + .../NetworkEntity/NetworkEntityManager.cpp | 140 +++++++++--------- .../NetworkEntity/NetworkEntityManager.h | 8 +- Gems/Multiplayer/Code/multiplayer_files.cmake | 2 + 13 files changed, 194 insertions(+), 116 deletions(-) create mode 100644 Gems/Multiplayer/Code/Source/EntityDomains/NullEntityDomain.cpp create mode 100644 Gems/Multiplayer/Code/Source/EntityDomains/NullEntityDomain.h diff --git a/Gems/Multiplayer/Code/Include/Multiplayer/EntityDomains/IEntityDomain.h b/Gems/Multiplayer/Code/Include/Multiplayer/EntityDomains/IEntityDomain.h index fc41a9b4d0..4b1bbbdba8 100644 --- a/Gems/Multiplayer/Code/Include/Multiplayer/EntityDomains/IEntityDomain.h +++ b/Gems/Multiplayer/Code/Include/Multiplayer/EntityDomains/IEntityDomain.h @@ -17,8 +17,6 @@ namespace Multiplayer class IEntityDomain { public: - using EntitiesNotInDomain = AZStd::unordered_set; - virtual ~IEntityDomain() = default; //! For domains that operate on a region of space, this sets the area the domain is responsible for. @@ -34,12 +32,10 @@ namespace Multiplayer //! @return false if this entity should not belong to the entity manger, true if it could be owned by the entity manager virtual bool IsInDomain(const ConstNetworkEntityHandle& entityHandle) const = 0; - //! Enable Entity Domain Exit Tracking for entities on the host. - //! @param ownedEntitySet the set of entities to activate tracking for - virtual void ActivateTracking(const INetworkEntityManager::OwnedEntitySet& ownedEntitySet) = 0; - - //! Return the set of netbound entities not included in this domain. - virtual const EntitiesNotInDomain& RetrieveEntitiesNotInDomain() const = 0; + //! This method will be invoked whenever we unexpectedly lose the authoritative entity replicator for an entity. + //! This gives our entity domain a chance to determine whether or not it should assume authority in this instance. + //! @param entityHandle the network entity handle of the entity that has lost it's authoritative replicator + virtual void HandleLossOfAuthoritativeReplicator(const ConstNetworkEntityHandle& entityHandle) = 0; //! Debug draw to visualize host entity domains. virtual void DebugDraw() const = 0; diff --git a/Gems/Multiplayer/Code/Include/Multiplayer/NetworkEntity/INetworkEntityManager.h b/Gems/Multiplayer/Code/Include/Multiplayer/NetworkEntity/INetworkEntityManager.h index 43915127ed..8c16176736 100644 --- a/Gems/Multiplayer/Code/Include/Multiplayer/NetworkEntity/INetworkEntityManager.h +++ b/Gems/Multiplayer/Code/Include/Multiplayer/NetworkEntity/INetworkEntityManager.h @@ -26,6 +26,7 @@ namespace Multiplayer using EntityExitDomainEvent = AZ::Event; using ControllersActivatedEvent = AZ::Event; using ControllersDeactivatedEvent = AZ::Event; + using NetEntityIdSet = AZStd::unordered_set; //! @class INetworkEntityManager //! @brief The interface for managing all networked entities. @@ -34,18 +35,17 @@ namespace Multiplayer public: AZ_RTTI(INetworkEntityManager, "{109759DE-9492-439C-A0B1-AE46E6FD029C}"); - using OwnedEntitySet = AZStd::unordered_set; using EntityList = AZStd::vector; virtual ~INetworkEntityManager() = default; - //! Configures the NetworkEntityManager to operate as an authoritative host. - //! @param hostId the hostId of this NetworkEntityManager + //! Configures the NetworkEntityManager. + //! @param hostId the hostId of this NetworkEntityManager (invalid for clients) //! @param entityDomain the entity domain used to determine which entities this manager has authority over virtual void Initialize(const HostId& hostId, AZStd::unique_ptr entityDomain) = 0; - //! Returns whether or not the network entity manager has been initialized to host. - //! @return boolean true if this network entity manager has been intialized to host + //! Returns whether or not the network entity manager has been initialized. + //! @return boolean true if this network entity manager has been intialized virtual bool IsInitialized() const = 0; //! Returns the entity domain associated with this network entity manager, this will be nullptr on clients. @@ -181,6 +181,19 @@ namespace Multiplayer //! @param entityRpcMessage the local rpc message to handle virtual void HandleLocalRpcMessage(NetworkEntityRpcMessage& message) = 0; + //! Handles a set of entities transitioning between entity domains. + //! @param entitiesNotInDomain the set of entities that are no longer contained within our entity domain + virtual void HandleEntitiesExitDomain(const NetEntityIdSet& entitiesNotInDomain) = 0; + + //! Forcibly assumes authoritative control over the given entity. + //! This should only be used in the event of the unexpected loss of the previous authority, any other usage could corrupt the simulation. + //! @param entityHandle the entity to forcibly assume authoritative control over + virtual void ForceAssumeAuthority(const ConstNetworkEntityHandle& entityHandle) = 0; + + //! Overrides the default timeout time used during entity migrations. + //! @param timeoutTimeMs the timeout time to use in milliseconds + virtual void SetMigrateTimeoutTimeMs(AZ::TimeMs timeoutTimeMs) = 0; + //! Visualization of network entity manager state. virtual void DebugDraw() const = 0; }; diff --git a/Gems/Multiplayer/Code/Source/Components/NetBindComponent.cpp b/Gems/Multiplayer/Code/Source/Components/NetBindComponent.cpp index cc71000d33..ceb9412408 100644 --- a/Gems/Multiplayer/Code/Source/Components/NetBindComponent.cpp +++ b/Gems/Multiplayer/Code/Source/Components/NetBindComponent.cpp @@ -317,7 +317,7 @@ namespace Multiplayer return false; } - bool NetBindComponent::HandlePropertyChangeMessage([[maybe_unused]] AzNetworking::ISerializer& serializer, [[maybe_unused]] bool notifyChanges) + bool NetBindComponent::HandlePropertyChangeMessage(AzNetworking::ISerializer& serializer, bool notifyChanges) { const NetEntityRole netEntityRole = m_netEntityRole; ReplicationRecord replicationRecord(netEntityRole); @@ -492,7 +492,7 @@ namespace Multiplayer void NetBindComponent::FillTotalReplicationRecord(ReplicationRecord& replicationRecord) const { replicationRecord.Append(m_totalRecord); - // if we have any outstanding changes yet to be logged, grab those as well + // If we have any outstanding changes yet to be logged, grab those as well if (m_currentRecord.HasChanges()) { replicationRecord.Append(m_currentRecord); diff --git a/Gems/Multiplayer/Code/Source/EntityDomains/FullOwnershipEntityDomain.cpp b/Gems/Multiplayer/Code/Source/EntityDomains/FullOwnershipEntityDomain.cpp index 9d53990fb4..59e2afc638 100644 --- a/Gems/Multiplayer/Code/Source/EntityDomains/FullOwnershipEntityDomain.cpp +++ b/Gems/Multiplayer/Code/Source/EntityDomains/FullOwnershipEntityDomain.cpp @@ -26,14 +26,9 @@ namespace Multiplayer return true; } - void FullOwnershipEntityDomain::ActivateTracking([[maybe_unused]] const INetworkEntityManager::OwnedEntitySet& ownedEntitySet) + void FullOwnershipEntityDomain::HandleLossOfAuthoritativeReplicator([[maybe_unused]] const ConstNetworkEntityHandle& entityHandle) { - ; - } - - const IEntityDomain::EntitiesNotInDomain& FullOwnershipEntityDomain::RetrieveEntitiesNotInDomain() const - { - return m_entitiesNotInDomain; + AZ_Assert(false, "FullOwnershipEntityDomain has authoritative control over all entities, something unexpected has happened"); } void FullOwnershipEntityDomain::DebugDraw() const diff --git a/Gems/Multiplayer/Code/Source/EntityDomains/FullOwnershipEntityDomain.h b/Gems/Multiplayer/Code/Source/EntityDomains/FullOwnershipEntityDomain.h index ae80c16ab8..203d9579ea 100644 --- a/Gems/Multiplayer/Code/Source/EntityDomains/FullOwnershipEntityDomain.h +++ b/Gems/Multiplayer/Code/Source/EntityDomains/FullOwnershipEntityDomain.h @@ -24,12 +24,8 @@ namespace Multiplayer void SetAabb(const AZ::Aabb& aabb) override; const AZ::Aabb& GetAabb() const override; bool IsInDomain(const ConstNetworkEntityHandle& entityHandle) const override; - void ActivateTracking(const INetworkEntityManager::OwnedEntitySet& ownedEntitySet) override; - const EntitiesNotInDomain& RetrieveEntitiesNotInDomain() const override; + void HandleLossOfAuthoritativeReplicator(const ConstNetworkEntityHandle& entityHandle) override; void DebugDraw() const override; //! @} - - private: - EntitiesNotInDomain m_entitiesNotInDomain; }; } diff --git a/Gems/Multiplayer/Code/Source/EntityDomains/NullEntityDomain.cpp b/Gems/Multiplayer/Code/Source/EntityDomains/NullEntityDomain.cpp new file mode 100644 index 0000000000..21d6abcb5a --- /dev/null +++ b/Gems/Multiplayer/Code/Source/EntityDomains/NullEntityDomain.cpp @@ -0,0 +1,41 @@ +/* + * Copyright (c) Contributors to the Open 3D Engine Project. + * For complete copyright and license terms please see the LICENSE at the root of this distribution. + * + * SPDX-License-Identifier: Apache-2.0 OR MIT + * + */ + +#include +#include +#include + +namespace Multiplayer +{ + void NullEntityDomain::SetAabb([[maybe_unused]] const AZ::Aabb& aabb) + { + ; // Do nothing, by definition we own everything + } + + const AZ::Aabb& NullEntityDomain::GetAabb() const + { + static AZ::Aabb nullAabb = AZ::Aabb::CreateNull(); + return nullAabb; + } + + bool NullEntityDomain::IsInDomain([[maybe_unused]] const ConstNetworkEntityHandle& entityHandle) const + { + return false; + } + + void NullEntityDomain::HandleLossOfAuthoritativeReplicator([[maybe_unused]] const ConstNetworkEntityHandle& entityHandle) + { + AZLOG_ERROR("Timed out entity id %llu during migration, marking for removal", aznumeric_cast(entityHandle.GetNetEntityId())); + GetNetworkEntityManager()->MarkForRemoval(entityHandle); + } + + void NullEntityDomain::DebugDraw() const + { + ; + } +} diff --git a/Gems/Multiplayer/Code/Source/EntityDomains/NullEntityDomain.h b/Gems/Multiplayer/Code/Source/EntityDomains/NullEntityDomain.h new file mode 100644 index 0000000000..247d82b366 --- /dev/null +++ b/Gems/Multiplayer/Code/Source/EntityDomains/NullEntityDomain.h @@ -0,0 +1,31 @@ +/* + * Copyright (c) Contributors to the Open 3D Engine Project. + * For complete copyright and license terms please see the LICENSE at the root of this distribution. + * + * SPDX-License-Identifier: Apache-2.0 OR MIT + * + */ + +#pragma once + +#include + +namespace Multiplayer +{ + class NullEntityDomain + : public IEntityDomain + { + public: + NullEntityDomain() = default; + NullEntityDomain(const NullEntityDomain& rhs) = default; + + //! IEntityDomain overrides. + //! @{ + void SetAabb(const AZ::Aabb& aabb) override; + const AZ::Aabb& GetAabb() const override; + bool IsInDomain(const ConstNetworkEntityHandle& entityHandle) const override; + void HandleLossOfAuthoritativeReplicator(const ConstNetworkEntityHandle& entityHandle) override; + void DebugDraw() const override; + //! @} + }; +} diff --git a/Gems/Multiplayer/Code/Source/MultiplayerSystemComponent.cpp b/Gems/Multiplayer/Code/Source/MultiplayerSystemComponent.cpp index 0bf5cea64b..1bc3404292 100644 --- a/Gems/Multiplayer/Code/Source/MultiplayerSystemComponent.cpp +++ b/Gems/Multiplayer/Code/Source/MultiplayerSystemComponent.cpp @@ -13,6 +13,7 @@ #include #include #include +#include #include #include #include @@ -832,18 +833,21 @@ namespace Multiplayer if (multiplayerType == MultiplayerAgentType::ClientServer || multiplayerType == MultiplayerAgentType::DedicatedServer) { m_spawnNetboundEntities = true; - m_initEvent.Signal(m_networkInterface); - + m_initEvent.Signal(m_networkInterface); //< Note! This might initialize our network entity manager for us if (!m_networkEntityManager.IsInitialized()) { - // Set up a full ownership domain if we didn't construct a domain during the initialize event const AZ::CVarFixedString serverAddr = cl_serveraddr; const uint16_t serverPort = cl_serverport; const AzNetworking::ProtocolType serverProtocol = sv_protocol; const AzNetworking::IpAddress hostId = AzNetworking::IpAddress(serverAddr.c_str(), serverPort, serverProtocol); + // Set up a full ownership domain if we didn't construct a domain during the initialize event m_networkEntityManager.Initialize(hostId, AZStd::make_unique()); } } + else if (multiplayerType == MultiplayerAgentType::Client) + { + m_networkEntityManager.Initialize(AzNetworking::IpAddress(), AZStd::make_unique()); + } } m_agentType = multiplayerType; diff --git a/Gems/Multiplayer/Code/Source/NetworkEntity/NetworkEntityAuthorityTracker.cpp b/Gems/Multiplayer/Code/Source/NetworkEntity/NetworkEntityAuthorityTracker.cpp index 7b6e8476e7..9f66bbee9e 100644 --- a/Gems/Multiplayer/Code/Source/NetworkEntity/NetworkEntityAuthorityTracker.cpp +++ b/Gems/Multiplayer/Code/Source/NetworkEntity/NetworkEntityAuthorityTracker.cpp @@ -9,6 +9,7 @@ #include #include #include +#include #include #include #include @@ -17,14 +18,20 @@ namespace Multiplayer { - AZ_CVAR(AZ::TimeMs, net_EntityMigrationTimeoutMs, AZ::TimeMs{ 1000 }, nullptr, AZ::ConsoleFunctorFlags::Null, "Time to wait for a new authority to attach to an entity before we delete the entity"); + AZ_CVAR(AZ::TimeMs, net_DefaultEntityMigrationTimeoutMs, AZ::TimeMs{ 1000 }, nullptr, AZ::ConsoleFunctorFlags::Null, "Time to wait for a new authority to attach to an entity before we delete the entity"); NetworkEntityAuthorityTracker::NetworkEntityAuthorityTracker(INetworkEntityManager& networkEntityManager) : m_networkEntityManager(networkEntityManager) + , m_timeoutTimeMs(net_DefaultEntityMigrationTimeoutMs) { ; } + void NetworkEntityAuthorityTracker::SetTimeoutTimeMs(AZ::TimeMs timeoutTimeMs) + { + m_timeoutTimeMs = timeoutTimeMs; + } + bool NetworkEntityAuthorityTracker::AddEntityAuthorityManager(ConstNetworkEntityHandle entityHandle, const HostId& newOwner) { bool ret = false; @@ -92,7 +99,7 @@ namespace Multiplayer "Trying to add something twice to the timeout map, this is unexpected" ); m_timeoutDataMap.insert(entityHandle.GetNetEntityId()); - AZ::Interface::Get()->AddCallback([this, netEntityId = entityHandle.GetNetEntityId(), previousOwner] + AZ::Interface::Get()->AddCallback([this, netEntityId = entityHandle.GetNetEntityId()] { auto timeoutData = m_timeoutDataMap.find(netEntityId); if (timeoutData != m_timeoutDataMap.end()) @@ -109,19 +116,13 @@ namespace Multiplayer } if (networkRole != NetEntityRole::Authority) { - AZLOG_ERROR - ( - "Timed out entity id %llu during migration previous owner %s, removing it", - aznumeric_cast(entityHandle.GetNetEntityId()), - previousOwner.GetString().c_str() - ); - m_networkEntityManager.MarkForRemoval(entityHandle); + m_networkEntityManager.GetEntityDomain()->HandleLossOfAuthoritativeReplicator(entityHandle); } } } }, AZ::Name("Entity authority removal functor"), - net_EntityMigrationTimeoutMs + m_timeoutTimeMs ); } else diff --git a/Gems/Multiplayer/Code/Source/NetworkEntity/NetworkEntityAuthorityTracker.h b/Gems/Multiplayer/Code/Source/NetworkEntity/NetworkEntityAuthorityTracker.h index c2e330ab4d..edb1b26ca8 100644 --- a/Gems/Multiplayer/Code/Source/NetworkEntity/NetworkEntityAuthorityTracker.h +++ b/Gems/Multiplayer/Code/Source/NetworkEntity/NetworkEntityAuthorityTracker.h @@ -23,6 +23,7 @@ namespace Multiplayer public: NetworkEntityAuthorityTracker(INetworkEntityManager& networkEntityManager); + void SetTimeoutTimeMs(AZ::TimeMs timeoutTimeMs); bool DoesEntityHaveOwner(ConstNetworkEntityHandle entityHandle) const; bool AddEntityAuthorityManager(ConstNetworkEntityHandle entityHandle, const HostId& newOwner); void RemoveEntityAuthorityManager(ConstNetworkEntityHandle entityHandle, const HostId& previousOwner); @@ -37,5 +38,7 @@ namespace Multiplayer TimeoutDataMap m_timeoutDataMap; EntityAuthorityMap m_entityAuthorityMap; INetworkEntityManager& m_networkEntityManager; + + AZ::TimeMs m_timeoutTimeMs = AZ::TimeMs{ 0 }; }; } diff --git a/Gems/Multiplayer/Code/Source/NetworkEntity/NetworkEntityManager.cpp b/Gems/Multiplayer/Code/Source/NetworkEntity/NetworkEntityManager.cpp index c7582af83f..b178ebeb02 100644 --- a/Gems/Multiplayer/Code/Source/NetworkEntity/NetworkEntityManager.cpp +++ b/Gems/Multiplayer/Code/Source/NetworkEntity/NetworkEntityManager.cpp @@ -28,12 +28,10 @@ namespace Multiplayer { AZ_CVAR(bool, net_DebugCheckNetworkEntityManager, false, nullptr, AZ::ConsoleFunctorFlags::Null, "Enables extra debug checks inside the NetworkEntityManager"); - AZ_CVAR(AZ::TimeMs, net_EntityDomainUpdateMs, AZ::TimeMs{ 500 }, nullptr, AZ::ConsoleFunctorFlags::Null, "Frequency for updating the entity domain in ms"); NetworkEntityManager::NetworkEntityManager() : m_networkEntityAuthorityTracker(*this) , m_removeEntitiesEvent([this] { RemoveEntities(); }, AZ::Name("NetworkEntityManager remove entities event")) - , m_updateEntityDomainEvent([this] { UpdateEntityDomain(); }, AZ::Name("NetworkEntityManager update entity domain event")) { AZ::Interface::Register(this); AzFramework::RootSpawnableNotificationBus::Handler::BusConnect(); @@ -63,8 +61,6 @@ namespace Multiplayer } m_entityDomain = AZStd::move(entityDomain); - m_updateEntityDomainEvent.Enqueue(net_EntityDomainUpdateMs, true); - m_entityDomain->ActivateTracking(m_ownedEntities); } bool NetworkEntityManager::IsInitialized() const @@ -231,6 +227,74 @@ namespace Multiplayer m_localDeferredRpcMessages.emplace_back(AZStd::move(message)); } + void NetworkEntityManager::HandleEntitiesExitDomain(const NetEntityIdSet& entitiesNotInDomain) + { + for (NetEntityId exitingId : entitiesNotInDomain) + { + bool safeToExit = true; + NetworkEntityHandle entityHandle = m_networkEntityTracker.Get(exitingId); + + // We need special handling for the NetworkHierarchy as well, since related entities need to be migrated together + NetworkHierarchyRootComponentController* hierarchyRootController = entityHandle.FindController(); + NetworkHierarchyChildComponentController* hierarchyChildController = entityHandle.FindController(); + + // Find the root entity + AZ::Entity* hierarchyRootEntity = nullptr; + if (hierarchyRootController) + { + hierarchyRootEntity = hierarchyRootController->GetParent().GetHierarchicalRoot(); + } + else if (hierarchyChildController) + { + hierarchyRootEntity = hierarchyChildController->GetParent().GetHierarchicalRoot(); + } + + if (hierarchyRootEntity) + { + NetEntityId rootNetId = GetNetEntityIdById(hierarchyRootEntity->GetId()); + ConstNetworkEntityHandle rootEntityHandle = GetEntity(rootNetId); + + // Check if the root entity is still tracked by this authority + if (rootEntityHandle.Exists() && rootEntityHandle.GetNetBindComponent()->HasController()) + { + safeToExit = false; + } + } + + // Validate that we aren't already planning to remove this entity + if (safeToExit) + { + for (auto remoteEntityId : m_removeList) + { + if (remoteEntityId == remoteEntityId) + { + safeToExit = false; + } + } + } + + if (safeToExit) + { + // Tell all the attached replicators for this entity that it's exited the domain + m_entityExitDomainEvent.Signal(entityHandle); + } + } + } + + void NetworkEntityManager::ForceAssumeAuthority(const ConstNetworkEntityHandle& entityHandle) + { + NetBindComponent* netBindComponent = entityHandle.GetNetBindComponent(); + if (netBindComponent != nullptr) + { + netBindComponent->ConstructControllers(); + } + } + + void NetworkEntityManager::SetMigrateTimeoutTimeMs(AZ::TimeMs timeoutTimeMs) + { + m_networkEntityAuthorityTracker.SetTimeoutTimeMs(timeoutTimeMs); + } + void NetworkEntityManager::DebugDraw() const { AzFramework::DebugDisplayRequestBus::BusPtr debugDisplayBus; @@ -243,7 +307,7 @@ namespace Multiplayer NetBindComponent* netBindComponent = m_networkEntityTracker.GetNetBindComponent(entity); AZ::Aabb entityBounds = AZ::Interface::Get()->GetEntityWorldBoundsUnion(entity->GetId()); entityBounds.Expand(AZ::Vector3(0.01f)); - if (netBindComponent->GetNetEntityRole() == NetEntityRole::Authority) + if ((netBindComponent != nullptr) && netBindComponent->GetNetEntityRole() == NetEntityRole::Authority) { debugDisplay->SetColor(AZ::Colors::Black); debugDisplay->SetAlpha(0.5f); @@ -277,77 +341,11 @@ namespace Multiplayer m_localDeferredRpcMessages.clear(); } - void NetworkEntityManager::UpdateEntityDomain() - { - if (m_entityDomain == nullptr) - { - return; - } - - const IEntityDomain::EntitiesNotInDomain& entitiesNotInDomain = m_entityDomain->RetrieveEntitiesNotInDomain(); - for (NetEntityId exitingId : entitiesNotInDomain) - { - OnEntityExitDomain(exitingId); - } - } - - void NetworkEntityManager::OnEntityExitDomain(NetEntityId entityId) - { - bool safeToExit = true; - NetworkEntityHandle entityHandle = m_networkEntityTracker.Get(entityId); - - // We also need special handling for the NetworkHierarchy as well, since related entities need to be migrated together - NetworkHierarchyRootComponentController* hierarchyRootController = entityHandle.FindController(); - NetworkHierarchyChildComponentController* hierarchyChildController = entityHandle.FindController(); - - // Find the root entity - AZ::Entity* hierarchyRootEntity = nullptr; - if (hierarchyRootController) - { - hierarchyRootEntity = hierarchyRootController->GetParent().GetHierarchicalRoot(); - } - else if (hierarchyChildController) - { - hierarchyRootEntity = hierarchyChildController->GetParent().GetHierarchicalRoot(); - } - - if (hierarchyRootEntity) - { - NetEntityId rootNetId = GetNetEntityIdById(hierarchyRootEntity->GetId()); - ConstNetworkEntityHandle rootEntityHandle = GetEntity(rootNetId); - - // Check if the root entity is still tracked by this authority - if (rootEntityHandle.Exists() && rootEntityHandle.GetNetBindComponent()->HasController()) - { - safeToExit = false; - } - } - - // Validate that we aren't already planning to remove this entity - if (safeToExit) - { - for (auto remoteEntityId : m_removeList) - { - if (remoteEntityId == remoteEntityId) - { - safeToExit = false; - } - } - } - - if (safeToExit) - { - m_entityExitDomainEvent.Signal(entityHandle); - } - } - void NetworkEntityManager::Reset() { m_multiplayerComponentRegistry.Reset(); m_removeList.clear(); m_entityDomain = nullptr; - m_updateEntityDomainEvent.RemoveFromQueue(); - m_ownedEntities.clear(); m_entityExitDomainEvent.DisconnectAllHandlers(); m_onEntityMarkedDirty.DisconnectAllHandlers(); m_onEntityNotifyChanges.DisconnectAllHandlers(); diff --git a/Gems/Multiplayer/Code/Source/NetworkEntity/NetworkEntityManager.h b/Gems/Multiplayer/Code/Source/NetworkEntity/NetworkEntityManager.h index 133c35dce0..7c6fdd94f9 100644 --- a/Gems/Multiplayer/Code/Source/NetworkEntity/NetworkEntityManager.h +++ b/Gems/Multiplayer/Code/Source/NetworkEntity/NetworkEntityManager.h @@ -79,12 +79,13 @@ namespace Multiplayer void NotifyControllersActivated(const ConstNetworkEntityHandle& entityHandle, EntityIsMigrating entityIsMigrating) override; void NotifyControllersDeactivated(const ConstNetworkEntityHandle& entityHandle, EntityIsMigrating entityIsMigrating) override; void HandleLocalRpcMessage(NetworkEntityRpcMessage& message) override; + void HandleEntitiesExitDomain(const NetEntityIdSet& entitiesNotInDomain) override; + void ForceAssumeAuthority(const ConstNetworkEntityHandle& entityHandle) override; + void SetMigrateTimeoutTimeMs(AZ::TimeMs timeoutTimeMs) override; void DebugDraw() const override; //! @} void DispatchLocalDeferredRpcMessages(); - void UpdateEntityDomain(); - void OnEntityExitDomain(NetEntityId entityId); //! RootSpawnableNotificationBus //! @{ @@ -106,9 +107,6 @@ namespace Multiplayer AZ::ScheduledEvent m_removeEntitiesEvent; AZStd::vector m_removeList; AZStd::unique_ptr m_entityDomain; - AZ::ScheduledEvent m_updateEntityDomainEvent; - - OwnedEntitySet m_ownedEntities; EntityExitDomainEvent m_entityExitDomainEvent; AZ::Event<> m_onEntityMarkedDirty; diff --git a/Gems/Multiplayer/Code/multiplayer_files.cmake b/Gems/Multiplayer/Code/multiplayer_files.cmake index a799278203..d417304877 100644 --- a/Gems/Multiplayer/Code/multiplayer_files.cmake +++ b/Gems/Multiplayer/Code/multiplayer_files.cmake @@ -94,6 +94,8 @@ set(FILES Source/Editor/MultiplayerEditorConnection.h Source/EntityDomains/FullOwnershipEntityDomain.cpp Source/EntityDomains/FullOwnershipEntityDomain.h + Source/EntityDomains/NullEntityDomain.cpp + Source/EntityDomains/NullEntityDomain.h Source/MultiplayerStats.cpp Source/MultiplayerSystemComponent.cpp Source/MultiplayerSystemComponent.h From 7e65104155a539fb1999e09862928b95641f9071 Mon Sep 17 00:00:00 2001 From: kberg-amzn Date: Wed, 3 Nov 2021 19:40:20 -0700 Subject: [PATCH 07/34] Addressing PR feedback Signed-off-by: kberg-amzn --- .../AzNetworking/UdpTransport/UdpConnection.h | 2 +- .../UdpTransport/UdpNetworkInterface.cpp | 2 +- .../NetworkEntityAuthorityTracker.cpp | 16 ++++++++-------- .../NetworkEntityAuthorityTracker.h | 4 ++-- 4 files changed, 12 insertions(+), 12 deletions(-) diff --git a/Code/Framework/AzNetworking/AzNetworking/UdpTransport/UdpConnection.h b/Code/Framework/AzNetworking/AzNetworking/UdpTransport/UdpConnection.h index c728016239..ddd75c9946 100644 --- a/Code/Framework/AzNetworking/AzNetworking/UdpTransport/UdpConnection.h +++ b/Code/Framework/AzNetworking/AzNetworking/UdpTransport/UdpConnection.h @@ -148,7 +148,7 @@ namespace AzNetworking uint32_t m_connectionMtu = MaxUdpTransmissionUnit; TimeoutId m_timeoutId; - int32_t m_timeoutCounter = 0; + uint32_t m_timeoutCounter = 0; }; } diff --git a/Code/Framework/AzNetworking/AzNetworking/UdpTransport/UdpNetworkInterface.cpp b/Code/Framework/AzNetworking/AzNetworking/UdpTransport/UdpNetworkInterface.cpp index 280b749d9e..b810c7a347 100644 --- a/Code/Framework/AzNetworking/AzNetworking/UdpTransport/UdpNetworkInterface.cpp +++ b/Code/Framework/AzNetworking/AzNetworking/UdpTransport/UdpNetworkInterface.cpp @@ -31,7 +31,7 @@ namespace AzNetworking AZ_CVAR(bool, net_UdpTimeoutConnections, true, nullptr, AZ::ConsoleFunctorFlags::DontReplicate, "Boolean value on whether we should timeout Udp connections"); AZ_CVAR(AZ::TimeMs, net_UdpPacketTimeSliceMs, AZ::TimeMs{ 8 }, nullptr, AZ::ConsoleFunctorFlags::DontReplicate, "The number of milliseconds to allow for packet processing"); - AZ_CVAR(int32_t, net_UdpUnackedHeartbeats, 5, nullptr, AZ::ConsoleFunctorFlags::Null, "The number of heartbeats to attempt to send to keep a connection alive before giving up"); + AZ_CVAR(uint32_t, net_UdpUnackedHeartbeats, 5, nullptr, AZ::ConsoleFunctorFlags::Null, "The number of heartbeats to attempt to send to keep a connection alive before giving up"); AZ_CVAR(AZ::TimeMs, net_UdpDefaultTimeoutMs, AZ::TimeMs{ 10 * 1000 }, nullptr, AZ::ConsoleFunctorFlags::Null, "Time in milliseconds before we timeout an idle Udp connection"); AZ_CVAR(AZ::TimeMs, net_MinPacketTimeoutMs, AZ::TimeMs{ 200 }, nullptr, AZ::ConsoleFunctorFlags::DontReplicate, "Minimum time to wait before timing out an unacked packet"); AZ_CVAR(int32_t, net_MaxTimeoutsPerFrame, 1000, nullptr, AZ::ConsoleFunctorFlags::DontReplicate, "Maximum number of packet timeouts to allow to process in a single frame"); diff --git a/Gems/Multiplayer/Code/Source/NetworkEntity/NetworkEntityAuthorityTracker.cpp b/Gems/Multiplayer/Code/Source/NetworkEntity/NetworkEntityAuthorityTracker.cpp index 9f66bbee9e..b401b85a27 100644 --- a/Gems/Multiplayer/Code/Source/NetworkEntity/NetworkEntityAuthorityTracker.cpp +++ b/Gems/Multiplayer/Code/Source/NetworkEntity/NetworkEntityAuthorityTracker.cpp @@ -35,8 +35,8 @@ namespace Multiplayer bool NetworkEntityAuthorityTracker::AddEntityAuthorityManager(ConstNetworkEntityHandle entityHandle, const HostId& newOwner) { bool ret = false; - auto timeoutData = m_timeoutDataMap.find(entityHandle.GetNetEntityId()); - if (timeoutData != m_timeoutDataMap.end()) + auto timeoutData = m_timedOutNetEntityIds.find(entityHandle.GetNetEntityId()); + if (timeoutData != m_timedOutNetEntityIds.end()) { AZLOG ( @@ -45,7 +45,7 @@ namespace Multiplayer aznumeric_cast(entityHandle.GetNetEntityId()), newOwner.GetString().c_str() ); - m_timeoutDataMap.erase(timeoutData); + m_timedOutNetEntityIds.erase(timeoutData); ret = true; } @@ -95,16 +95,16 @@ namespace Multiplayer { AZ_Assert ( - m_timeoutDataMap.find(entityHandle.GetNetEntityId()) == m_timeoutDataMap.end(), + m_timedOutNetEntityIds.find(entityHandle.GetNetEntityId()) == m_timedOutNetEntityIds.end(), "Trying to add something twice to the timeout map, this is unexpected" ); - m_timeoutDataMap.insert(entityHandle.GetNetEntityId()); + m_timedOutNetEntityIds.insert(entityHandle.GetNetEntityId()); AZ::Interface::Get()->AddCallback([this, netEntityId = entityHandle.GetNetEntityId()] { - auto timeoutData = m_timeoutDataMap.find(netEntityId); - if (timeoutData != m_timeoutDataMap.end()) + auto timeoutData = m_timedOutNetEntityIds.find(netEntityId); + if (timeoutData != m_timedOutNetEntityIds.end()) { - m_timeoutDataMap.erase(timeoutData); + m_timedOutNetEntityIds.erase(timeoutData); ConstNetworkEntityHandle entityHandle = m_networkEntityManager.GetEntity(netEntityId); if (auto entity = entityHandle.GetEntity()) { diff --git a/Gems/Multiplayer/Code/Source/NetworkEntity/NetworkEntityAuthorityTracker.h b/Gems/Multiplayer/Code/Source/NetworkEntity/NetworkEntityAuthorityTracker.h index edb1b26ca8..ae9aac04ea 100644 --- a/Gems/Multiplayer/Code/Source/NetworkEntity/NetworkEntityAuthorityTracker.h +++ b/Gems/Multiplayer/Code/Source/NetworkEntity/NetworkEntityAuthorityTracker.h @@ -13,6 +13,7 @@ #include #include #include +#include namespace Multiplayer { @@ -32,10 +33,9 @@ namespace Multiplayer private: NetworkEntityAuthorityTracker& operator= (const NetworkEntityAuthorityTracker&) = delete; - using TimeoutDataMap = AZStd::unordered_set; using EntityAuthorityMap = AZStd::unordered_map>; - TimeoutDataMap m_timeoutDataMap; + NetEntityIdSet m_timedOutNetEntityIds; EntityAuthorityMap m_entityAuthorityMap; INetworkEntityManager& m_networkEntityManager; From be7c8c8dd33a73c8cedf6fe20e72a1dc0cc9ca26 Mon Sep 17 00:00:00 2001 From: Gene Walters Date: Thu, 4 Nov 2021 09:55:59 -0700 Subject: [PATCH 08/34] Removing flaky multiplayer tests. Will bring back once we can reproduce and correct the flakes Signed-off-by: Gene Walters --- AutomatedTesting/Gem/PythonTests/CMakeLists.txt | 3 --- 1 file changed, 3 deletions(-) diff --git a/AutomatedTesting/Gem/PythonTests/CMakeLists.txt b/AutomatedTesting/Gem/PythonTests/CMakeLists.txt index bec49185bd..fd3222ba83 100644 --- a/AutomatedTesting/Gem/PythonTests/CMakeLists.txt +++ b/AutomatedTesting/Gem/PythonTests/CMakeLists.txt @@ -59,8 +59,5 @@ add_subdirectory(smoke) ## AWS ## add_subdirectory(AWS) -## Multiplayer ## -add_subdirectory(Multiplayer) - ## Integration tests for editor testing framework ## add_subdirectory(editor_test_testing) From 781a635ef72a9942852bc7a591d141ea715f8490 Mon Sep 17 00:00:00 2001 From: Esteban Papp <81431996+amznestebanpapp@users.noreply.github.com> Date: Thu, 4 Nov 2021 11:10:29 -0700 Subject: [PATCH 09/34] Cleanup: Remove cry load dll functions (#5295) * Removes VTUNE profiler hooks from Cry Signed-off-by: Esteban Papp <81431996+amznestebanpapp@users.noreply.github.com> * Remove cry load dll functions Signed-off-by: Esteban Papp <81431996+amznestebanpapp@users.noreply.github.com> * removes unused restricted section Signed-off-by: Esteban Papp <81431996+amznestebanpapp@users.noreply.github.com> --- Code/Legacy/CryCommon/StlUtils.h | 337 --------------------------- Code/Legacy/CrySystem/System.cpp | 2 - Code/Legacy/CrySystem/System.h | 12 - Code/Legacy/CrySystem/SystemInit.cpp | 207 ---------------- 4 files changed, 558 deletions(-) diff --git a/Code/Legacy/CryCommon/StlUtils.h b/Code/Legacy/CryCommon/StlUtils.h index f5128a564f..ccbc854dc3 100644 --- a/Code/Legacy/CryCommon/StlUtils.h +++ b/Code/Legacy/CryCommon/StlUtils.h @@ -96,47 +96,6 @@ unsigned countElements (const std::vector& arrT, const T& x) */ namespace stl { - ////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// - // Compare member of class/struct. - // - // e.g. Sort Vec3s by x component - // - // std::sort(vec3s.begin(), vec3s.end(), stl::member_compare()); - // - ////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// - template > - struct member_compare - { - inline bool operator () (const OWNER_TYPE& lhs, const OWNER_TYPE& rhs) const - { - return EQUALITY()(lhs.*MEMBER_PTR, rhs.*MEMBER_PTR); - } - }; - - ////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// - // Compare member of class/struct against parameter. - // - // e.g. Find Vec3 with x component less than 1.0 - // - // std::find_if(vec3s.begin(), vec3s.end(), stl::member_compare_param(1.0f)); - // - ////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// - template > - struct member_compare_param - { - inline member_compare_param(const MEMBER_TYPE& _value) - : value(_value) - { - } - - inline bool operator () (const OWNER_TYPE& rhs) const - { - return EQUALITY()(rhs.*MEMBER_PTR, value); - } - - const MEMBER_TYPE& value; - }; - ////////////////////////////////////////////////////////////////////////// //! Searches the given entry in the map by key, and if there is none, returns the default value ////////////////////////////////////////////////////////////////////////// @@ -154,48 +113,6 @@ namespace stl } } - ////////////////////////////////////////////////////////////////////////// - //! Inserts and returns a reference to the given value in the map, or returns the current one if it's already there. - ////////////////////////////////////////////////////////////////////////// - template - inline typename Map::mapped_type& map_insert_or_get(Map& mapKeyToValue, const typename Map::key_type& key, const typename Map::mapped_type& defValue = typename Map::mapped_type()) - { - auto&& iresult = mapKeyToValue.insert(typename Map::value_type(key, defValue)); - return iresult.first->second; - } - - // searches the given entry in the map by key, and if there is none, returns the default value - // The values are taken/returned in REFERENCEs rather than values - template - inline mapped_type& find_in_map_ref(std::map& mapKeyToValue, const Key& key, mapped_type& valueDefault) - { - typedef std::map Map; - typename Map::iterator it = mapKeyToValue.find (key); - if (it == mapKeyToValue.end()) - { - return valueDefault; - } - else - { - return it->second; - } - } - - template - inline const mapped_type& find_in_map_ref(const std::map& mapKeyToValue, const Key& key, const mapped_type& valueDefault) - { - typedef std::map Map; - typename Map::const_iterator it = mapKeyToValue.find (key); - if (it == mapKeyToValue.end()) - { - return valueDefault; - } - else - { - return it->second; - } - } - ////////////////////////////////////////////////////////////////////////// //! Fills vector with contents of map. ////////////////////////////////////////////////////////////////////////// @@ -210,20 +127,6 @@ namespace stl } } - ////////////////////////////////////////////////////////////////////////// - //! Fills vector with contents of set. - ////////////////////////////////////////////////////////////////////////// - template - inline void set_to_vector(const Set& theSet, Vector& array) - { - array.resize(0); - array.reserve(theSet.size()); - for (typename Set::const_iterator it = theSet.begin(); it != theSet.end(); ++it) - { - array.push_back(*it); - } - } - ////////////////////////////////////////////////////////////////////////// //! Find and erase element from container. // @return true if item was find and erased, false if item not found. @@ -312,48 +215,6 @@ namespace stl return false; } - ////////////////////////////////////////////////////////////////////////// - //! Push back to container unique element. - // @return true if item added, false overwise. - template - inline bool push_back_unique_if(CONTAINER& container, const PREDICATE& predicate, const VALUE& value) - { - typename CONTAINER::iterator end = container.end(); - - if (AZStd::find_if(container.begin(), end, predicate) == end) - { - container.push_back(value); - - return true; - } - else - { - return false; - } - } - - ////////////////////////////////////////////////////////////////////////// - //! Push back to container contents of another container - template - inline void push_back_range(Container& container, Iter begin, Iter end) - { - for (Iter it = begin; it != end; ++it) - { - container.push_back(*it); - } - } - - ////////////////////////////////////////////////////////////////////////// - //! Push back to container contents of another container, if not already present - template - inline void push_back_range_unique(Container& container, Iter begin, Iter end) - { - for (Iter it = begin; it != end; ++it) - { - push_back_unique(container, *it); - } - } - ////////////////////////////////////////////////////////////////////////// //! Find element in container. // @return true if item found. @@ -373,107 +234,6 @@ namespace stl return (it == last || value != *it) ? last : it; } - ////////////////////////////////////////////////////////////////////////// - //! Find element in a sorted container using binary search with logarithmic efficiency. - // @return true if item was inserted. - template - inline bool binary_insert_unique(Container& container, const Value& value) - { - typename Container::iterator it = std::lower_bound(container.begin(), container.end(), value); - if (it != container.end()) - { - if (*it == value) - { - return false; - } - container.insert(it, value); - } - else - { - container.insert(container.end(), value); - } - return true; - } - ////////////////////////////////////////////////////////////////////////// - //! Find element in a sorted container using binary search with logarithmic efficiency. - // and erases if element found. - // @return true if item was erased. - template - inline bool binary_erase(Container& container, const Value& value) - { - typename Container::iterator it = std::lower_bound(container.begin(), container.end(), value); - if (it != container.end() && *it == value) - { - container.erase(it); - return true; - } - return false; - } - - template - ItT remove_from_heap(ItT begin, ItT end, ItT at, Func order) - { - using std::swap; - - --end; - if (at == end) - { - return at; - } - - size_t idx = std::distance(begin, at); - swap(*end, *at); - - size_t length = std::distance(begin, end); - size_t parent, child; - - if (idx > 0 && order(*(begin + idx / 2), *(begin + idx))) - { - do - { - parent = idx / 2; - swap(*(begin + idx), *(begin + parent)); - idx = parent; - - if (idx == 0 || order(*(begin + idx), *(begin + idx / 2))) - { - return end; - } - } - while (true); - } - else - { - do - { - child = idx * 2 + 1; - if (child >= length) - { - return end; - } - - ItT left = begin + child; - ItT right = begin + child + 1; - - if (right < end && order(*left, *right)) - { - ++child; - } - - if (order(*(begin + child), *(begin + idx))) - { - return end; - } - - swap(*(begin + child), *(begin + idx)); - idx = child; - } - while (true); - } - - return end; - } - struct container_object_deleter { template @@ -506,18 +266,6 @@ namespace stl return type.c_str(); } - ////////////////////////////////////////////////////////////////////////// - //! Case sensetive less key for any type convertable to const char*. - ////////////////////////////////////////////////////////////////////////// - template - struct less_strcmp - { - bool operator()(const Type& left, const Type& right) const - { - return strcmp(constchar_cast(left), constchar_cast(right)) < 0; - } - }; - ////////////////////////////////////////////////////////////////////////// //! Case insensetive less key for any type convertable to const char*. template @@ -690,89 +438,4 @@ namespace stl stl::free_container(container); } }; - - template - inline void for_each_array(T (&buffer)[Length], Func func) - { - std::for_each(&buffer[0], &buffer[Length], func); - } - - template - inline void for_each_array(StaticInstance(&buffer)[Length], Func func) - { - for (size_t idx = 0; idx < Length; ++idx) - { - func(*buffer[idx]); - } - } - - template - inline void destruct(T* p) - { - p->~T(); - } -} - -#define DEFINE_INTRUSIVE_LINKED_LIST(Class) \ - template<> \ - Class * stl::intrusive_linked_list_node::m_root_intrusive = nullptr; - -// define the maplikestruct, used to approximate the memory requirements for a map node -namespace stl -{ - struct MapLikeStruct - { - bool color; - void* parent; - void* left; - void* right; - }; -} -template -unsigned sizeOfMap(Map& map) -{ - unsigned size = 0; - for (typename Map::iterator it = map.begin(); it != map.end(); it++) - { - typename Map::mapped_type& T = it->second; - size += T.Size(); - } - size += map.size() * sizeof(stl::MapLikeStruct); - return size; -} -template -unsigned sizeOfMapStr(Map& map) -{ - unsigned size = 0; - for (typename Map::iterator it = map.begin(); it != map.end(); it++) - { - typename Map::mapped_type& T = it->second; - size += T.capacity(); - } - size += map.size() * sizeof(stl::MapLikeStruct); - return size; -} -template -unsigned sizeOfMapP(Map& map) -{ - unsigned size = 0; - for (typename Map::iterator it = map.begin(); it != map.end(); it++) - { - typename Map::mapped_type& T = it->second; - size += T->Size(); - } - size += map.size() * sizeof(stl::MapLikeStruct); - return size; -} -template -unsigned sizeOfMapS(Map& map) -{ - unsigned size = 0; - for (typename Map::iterator it = map.begin(); it != map.end(); it++) - { - typename Map::mapped_type& T = it->second; - size += sizeof(T); - } - size += map.size() * sizeof(stl::MapLikeStruct); - return size; } diff --git a/Code/Legacy/CrySystem/System.cpp b/Code/Legacy/CrySystem/System.cpp index eebe626918..65facba4dd 100644 --- a/Code/Legacy/CrySystem/System.cpp +++ b/Code/Legacy/CrySystem/System.cpp @@ -509,8 +509,6 @@ void CSystem::ShutDown() ShutdownFileSystem(); - ShutdownModuleLibraries(); - EBUS_EVENT(CrySystemEventBus, OnCrySystemPostShutdown); } diff --git a/Code/Legacy/CrySystem/System.h b/Code/Legacy/CrySystem/System.h index 74ce1cbbc2..a10c1f8ed8 100644 --- a/Code/Legacy/CrySystem/System.h +++ b/Code/Legacy/CrySystem/System.h @@ -303,8 +303,6 @@ public: void SetVersionInfo(const char* const szVersion); #endif - void ShutdownModuleLibraries(); - #if defined(WIN32) friend LRESULT WINAPI WndProc(HWND hWnd, UINT uMsg, WPARAM wParam, LPARAM lParam); #endif @@ -323,8 +321,6 @@ private: // Release all resources. void ShutDown(); - bool LoadEngineDLLs(); - //! @name Initialization routines //@{ bool InitConsole(); @@ -340,11 +336,8 @@ private: void CreateSystemVars(); void CreateAudioVars(); - AZStd::unique_ptr LoadDLL(const char* dllName); - void FreeLib(AZStd::unique_ptr& hLibModule); - bool UnloadDLL(const char* dllName); void QueryVersionInfo(); void LogVersion(); void LogBuildInfo(); @@ -359,8 +352,6 @@ private: void AddCVarGroupDirectory(const AZStd::string& sPath) override; - AZStd::unique_ptr LoadDynamiclibrary(const char* dllName) const; - #if defined(AZ_RESTRICTED_PLATFORM) #define AZ_RESTRICTED_SECTION SYSTEM_H_SECTION_3 #include AZ_RESTRICTED_FILE(System_h) @@ -416,9 +407,6 @@ private: // ------------------------------------------------------ bool m_bDrawConsole; //!< Set to true if OK to draw the console. bool m_bDrawUI; //!< Set to true if OK to draw UI. - - std::map > m_moduleDLLHandles; - //! current active process IProcess* m_pProcess; diff --git a/Code/Legacy/CrySystem/SystemInit.cpp b/Code/Legacy/CrySystem/SystemInit.cpp index 7ce9fcd342..d9ba3bb4c9 100644 --- a/Code/Legacy/CrySystem/SystemInit.cpp +++ b/Code/Legacy/CrySystem/SystemInit.cpp @@ -12,7 +12,6 @@ #if defined(AZ_RESTRICTED_PLATFORM) || defined(AZ_TOOLS_EXPAND_FOR_RESTRICTED_PLATFORMS) #undef AZ_RESTRICTED_SECTION -#define SYSTEMINIT_CPP_SECTION_1 1 #define SYSTEMINIT_CPP_SECTION_2 2 #define SYSTEMINIT_CPP_SECTION_3 3 #define SYSTEMINIT_CPP_SECTION_4 4 @@ -168,30 +167,6 @@ void CryEngineSignalHandler(int signal) #define LOCALIZATION_TRANSLATIONS_LIST_FILE_NAME "Libs/Localization/localization.xml" -////////////////////////////////////////////////////////////////////////// -#if defined(WIN32) || defined(LINUX) || defined(APPLE) -# define DLL_INITFUNC_RENDERER "PackageRenderConstructor" -# define DLL_INITFUNC_SOUND "CreateSoundSystem" -# define DLL_INITFUNC_FONT "CreateCryFontInterface" -# define DLL_INITFUNC_3DENGINE "CreateCry3DEngine" -# define DLL_INITFUNC_UI "CreateLyShineInterface" -#define AZ_RESTRICTED_SECTION_IMPLEMENTED -#elif defined(AZ_RESTRICTED_PLATFORM) -#define AZ_RESTRICTED_SECTION SYSTEMINIT_CPP_SECTION_1 -#include AZ_RESTRICTED_FILE(SystemInit_cpp) -#endif -#if defined(AZ_RESTRICTED_SECTION_IMPLEMENTED) -#undef AZ_RESTRICTED_SECTION_IMPLEMENTED -#else -# define DLL_INITFUNC_RENDERER (LPCSTR)1 -# define DLL_INITFUNC_RENDERER (LPCSTR)1 -# define DLL_INITFUNC_SOUND (LPCSTR)1 -# define DLL_INITFUNC_PHYSIC (LPCSTR)1 -# define DLL_INITFUNC_FONT (LPCSTR)1 -# define DLL_INITFUNC_3DENGINE (LPCSTR)1 -# define DLL_INITFUNC_UI (LPCSTR)1 -#endif - #define AZ_TRACE_SYSTEM_WINDOW AZ::Debug::Trace::GetDefaultSystemWindow() #ifdef WIN32 @@ -285,96 +260,6 @@ static void CmdCrashTest(IConsoleCmdArgs* pArgs) } AZ_POP_DISABLE_WARNING -////////////////////////////////////////////////////////////////////////// -struct SysSpecOverrideSink - : public ILoadConfigurationEntrySink -{ - virtual void OnLoadConfigurationEntry(const char* szKey, const char* szValue, const char* szGroup) - { - ICVar* pCvar = gEnv->pConsole->GetCVar(szKey); - - if (pCvar) - { - const bool wasNotInConfig = ((pCvar->GetFlags() & VF_WASINCONFIG) == 0); - bool applyCvar = wasNotInConfig; - if (applyCvar == false) - { - // Special handling for sys_spec_full - if (azstricmp(szKey, "sys_spec_full") == 0) - { - // If it is set to 0 then ignore this request to set to something else - // If it is set to 0 then the user wants to changes system spec settings in system.cfg - if (pCvar->GetIVal() != 0) - { - applyCvar = true; - } - } - else - { - // This could bypass the restricted cvar checks that exist elsewhere depending on - // the calling code so we also need check here before setting. - bool isConst = pCvar->IsConstCVar(); - bool isCheat = ((pCvar->GetFlags() & (VF_CHEAT | VF_CHEAT_NOCHECK | VF_CHEAT_ALWAYS_CHECK)) != 0); - bool isReadOnly = ((pCvar->GetFlags() & VF_READONLY) != 0); - bool isDeprecated = ((pCvar->GetFlags() & VF_DEPRECATED) != 0); - bool allowApplyCvar = true; - - if ((isConst || isCheat || isReadOnly) || isDeprecated) - { - allowApplyCvar = !isDeprecated && (gEnv->pSystem->IsDevMode()) || (gEnv->IsEditor()); - } - - if ((allowApplyCvar) || ALLOW_CONST_CVAR_MODIFICATIONS) - { - applyCvar = true; - } - } - } - - if (applyCvar) - { - pCvar->Set(szValue); - } - else - { - CryLogAlways("NOT VF_WASINCONFIG Ignoring cvar '%s' new value '%s' old value '%s' group '%s'", szKey, szValue, pCvar->GetString(), szGroup); - } - } - else - { - CryLogAlways("Can't find cvar '%s' value '%s' group '%s'", szKey, szValue, szGroup); - } - } -}; - -#if !defined(CONSOLE) -struct SysSpecOverrideSinkConsole - : public ILoadConfigurationEntrySink -{ - virtual void OnLoadConfigurationEntry(const char* szKey, const char* szValue, const char* szGroup) - { - // Ignore platform-specific cvars that should just be executed on the console - if (azstricmp(szGroup, "Platform") == 0) - { - return; - } - - ICVar* pCvar = gEnv->pConsole->GetCVar(szKey); - if (pCvar) - { - pCvar->Set(szValue); - } - else - { - // If the cvar doesn't exist, calling this function only saves the value in case it's registered later where - // at that point it will be set from the stored value. This is required because otherwise registering the - // cvar bypasses any callbacks and uses values directly from the cvar group files. - gEnv->pConsole->LoadConfigVar(szKey, szValue); - } - } -}; -#endif - static ESystemConfigPlatform GetDevicePlatform() { #if defined(AZ_PLATFORM_WINDOWS) || defined(AZ_PLATFORM_LINUX) @@ -398,98 +283,6 @@ static ESystemConfigPlatform GetDevicePlatform() #endif } -////////////////////////////////////////////////////////////////////////// -#if !defined(AZ_MONOLITHIC_BUILD) - -AZStd::unique_ptr CSystem::LoadDynamiclibrary(const char* dllName) const -{ - AZStd::unique_ptr handle = AZ::DynamicModuleHandle::Create(dllName); - - bool libraryLoaded = handle->Load(false); - // We need to inject the environment first thing so that allocators are available immediately - InjectEnvironmentFunction injectEnv = handle->GetFunction(INJECT_ENVIRONMENT_FUNCTION); - if (injectEnv) - { - auto env = AZ::Environment::GetInstance(); - injectEnv(env); - } - - if (!libraryLoaded) - { - handle.release(); - } - return handle; -} - -////////////////////////////////////////////////////////////////////////// -AZStd::unique_ptr CSystem::LoadDLL(const char* dllName) -{ - AZ_TracePrintf(AZ_TRACE_SYSTEM_WINDOW, "Loading DLL: %s", dllName); - - AZStd::unique_ptr handle = LoadDynamiclibrary(dllName); - - if (!handle) - { -#if defined(LINUX) || defined(APPLE) - AZ_Assert(false, "Error loading dylib: %s, error : %s\n", dllName, dlerror()); -#else - AZ_Assert(false, "Error loading dll: %s, error code %d", dllName, GetLastError()); -#endif - return handle; - } - - return handle; -} - -// TODO:DLL #endif //#if defined(AZ_HAS_DLL_SUPPORT) && !defined(AZ_MONOLITHIC_BUILD) -#endif //if !defined(AZ_MONOLITHIC_BUILD) -////////////////////////////////////////////////////////////////////////// -bool CSystem::LoadEngineDLLs() -{ - return true; -} - -////////////////////////////////////////////////////////////////////////// -bool CSystem::UnloadDLL(const char* dllName) -{ - bool isSuccess = false; - - AZ::Crc32 key(dllName); - AZStd::unique_ptr empty; - AZStd::unique_ptr& hModule = stl::find_in_map_ref(m_moduleDLLHandles, key, empty); - if ((hModule) && (hModule->IsLoaded())) - { - DetachEnvironmentFunction detachEnv = hModule->GetFunction(DETACH_ENVIRONMENT_FUNCTION); - if (detachEnv) - { - detachEnv(); - } - - isSuccess = hModule->Unload(); - hModule.release(); - } - - return isSuccess; -} - -////////////////////////////////////////////////////////////////////////// -void CSystem::ShutdownModuleLibraries() -{ -#if !defined(AZ_MONOLITHIC_BUILD) - for (auto iterator = m_moduleDLLHandles.begin(); iterator != m_moduleDLLHandles.end(); ++iterator) - { - if (iterator->second->IsLoaded()) - { - iterator->second->Unload(); - } - iterator->second.release(); - } - - m_moduleDLLHandles.clear(); - -#endif // !defined(AZ_MONOLITHIC_BUILD) -} - ///////////////////////////////////////////////////////////////////////////////// ///////////////////////////////////////////////////////////////////////////////// bool CSystem::InitConsole() From bc2b3b12a70d3ef3977e739ca407b6b3f5a8e8c3 Mon Sep 17 00:00:00 2001 From: jromnoa <80134229+jromnoa@users.noreply.github.com> Date: Thu, 4 Nov 2021 11:51:58 -0700 Subject: [PATCH 10/34] remove old main suite, rename optimized main suite to be the new main suite Signed-off-by: jromnoa <80134229+jromnoa@users.noreply.github.com> --- .../Gem/PythonTests/Atom/TestSuite_Main.py | 302 ++++-------------- .../Atom/TestSuite_Main_Optimized.py | 91 ------ 2 files changed, 69 insertions(+), 324 deletions(-) delete mode 100644 AutomatedTesting/Gem/PythonTests/Atom/TestSuite_Main_Optimized.py diff --git a/AutomatedTesting/Gem/PythonTests/Atom/TestSuite_Main.py b/AutomatedTesting/Gem/PythonTests/Atom/TestSuite_Main.py index 6cc48984ab..950bd44199 100644 --- a/AutomatedTesting/Gem/PythonTests/Atom/TestSuite_Main.py +++ b/AutomatedTesting/Gem/PythonTests/Atom/TestSuite_Main.py @@ -4,252 +4,88 @@ For complete copyright and license terms please see the LICENSE at the root of t SPDX-License-Identifier: Apache-2.0 OR MIT """ -import logging -import os - import pytest -import ly_test_tools.environment.file_system as file_system -import editor_python_test_tools.hydra_test_utils as hydra -from Atom.atom_utils.atom_constants import LIGHT_TYPES - -logger = logging.getLogger(__name__) -TEST_DIRECTORY = os.path.join(os.path.dirname(__file__), "tests") +from ly_test_tools.o3de.editor_test import EditorSharedTest, EditorTestSuite +@pytest.mark.xfail(reason="Optimized tests are experimental, we will enable xfail and monitor them temporarily.") @pytest.mark.parametrize("project", ["AutomatedTesting"]) @pytest.mark.parametrize("launcher_platform", ['windows_editor']) -@pytest.mark.parametrize("level", ["auto_test"]) -class TestAtomEditorComponentsMain(object): - """Holds tests for Atom components.""" +class TestAutomation(EditorTestSuite): - @pytest.mark.test_case_id("C32078118") # Decal - @pytest.mark.test_case_id("C32078119") # DepthOfField - @pytest.mark.test_case_id("C32078120") # Directional Light - @pytest.mark.test_case_id("C32078121") # Exposure Control - @pytest.mark.test_case_id("C32078115") # Global Skylight (IBL) - @pytest.mark.test_case_id("C32078125") # Physical Sky - @pytest.mark.test_case_id("C32078127") # PostFX Layer - @pytest.mark.test_case_id("C32078131") # PostFX Radius Weight Modifier - @pytest.mark.test_case_id("C32078117") # Light - @pytest.mark.test_case_id("C36525660") # Display Mapper - def test_AtomEditorComponents_AddedToEntity(self, request, editor, level, workspace, project, launcher_platform): - """ - Please review the hydra script run by this test for more specific test info. - Tests the Atom components & verifies all "expected_lines" appear in Editor.log - """ - cfg_args = [level] + @pytest.mark.test_case_id("C32078118") + class AtomEditorComponents_DecalAdded(EditorSharedTest): + from Atom.tests import hydra_AtomEditorComponents_DecalAdded as test_module - expected_lines = [ - # Decal Component - "Decal Entity successfully created", - "Decal_test: Component added to the entity: True", - "Decal_test: Component removed after UNDO: True", - "Decal_test: Component added after REDO: True", - "Decal_test: Entered game mode: True", - "Decal_test: Exit game mode: True", - "Decal Controller|Configuration|Material: SUCCESS", - "Decal_test: Entity is hidden: True", - "Decal_test: Entity is shown: True", - "Decal_test: Entity deleted: True", - "Decal_test: UNDO entity deletion works: True", - "Decal_test: REDO entity deletion works: True", - # DepthOfField Component - "DepthOfField Entity successfully created", - "DepthOfField_test: Component added to the entity: True", - "DepthOfField_test: Component removed after UNDO: True", - "DepthOfField_test: Component added after REDO: True", - "DepthOfField_test: Entered game mode: True", - "DepthOfField_test: Exit game mode: True", - "DepthOfField_test: Entity disabled initially: True", - "DepthOfField_test: Entity enabled after adding required components: True", - "DepthOfField Controller|Configuration|Camera Entity: SUCCESS", - "DepthOfField_test: Entity is hidden: True", - "DepthOfField_test: Entity is shown: True", - "DepthOfField_test: Entity deleted: True", - "DepthOfField_test: UNDO entity deletion works: True", - "DepthOfField_test: REDO entity deletion works: True", - # Directional Light Component - "Directional Light Entity successfully created", - "Directional Light_test: Component added to the entity: True", - "Directional Light_test: Component removed after UNDO: True", - "Directional Light_test: Component added after REDO: True", - "Directional Light_test: Entered game mode: True", - "Directional Light_test: Exit game mode: True", - "Directional Light_test: Entity is hidden: True", - "Directional Light_test: Entity is shown: True", - "Directional Light_test: Entity deleted: True", - "Directional Light_test: UNDO entity deletion works: True", - "Directional Light_test: REDO entity deletion works: True", - # Exposure Control Component - "Exposure Control Entity successfully created", - "Exposure Control_test: Component added to the entity: True", - "Exposure Control_test: Component removed after UNDO: True", - "Exposure Control_test: Component added after REDO: True", - "Exposure Control_test: Entered game mode: True", - "Exposure Control_test: Exit game mode: True", - "Exposure Control_test: Entity disabled initially: True", - "Exposure Control_test: Entity enabled after adding required components: True", - "Exposure Control_test: Entity is hidden: True", - "Exposure Control_test: Entity is shown: True", - "Exposure Control_test: Entity deleted: True", - "Exposure Control_test: UNDO entity deletion works: True", - "Exposure Control_test: REDO entity deletion works: True", - # Global Skylight (IBL) Component - "Global Skylight (IBL) Entity successfully created", - "Global Skylight (IBL)_test: Component added to the entity: True", - "Global Skylight (IBL)_test: Component removed after UNDO: True", - "Global Skylight (IBL)_test: Component added after REDO: True", - "Global Skylight (IBL)_test: Entered game mode: True", - "Global Skylight (IBL)_test: Exit game mode: True", - "Global Skylight (IBL) Controller|Configuration|Diffuse Image: SUCCESS", - "Global Skylight (IBL) Controller|Configuration|Specular Image: SUCCESS", - "Global Skylight (IBL)_test: Entity is hidden: True", - "Global Skylight (IBL)_test: Entity is shown: True", - "Global Skylight (IBL)_test: Entity deleted: True", - "Global Skylight (IBL)_test: UNDO entity deletion works: True", - "Global Skylight (IBL)_test: REDO entity deletion works: True", - # Physical Sky Component - "Physical Sky Entity successfully created", - "Physical Sky component was added to entity", - "Entity has a Physical Sky component", - "Physical Sky_test: Component added to the entity: True", - "Physical Sky_test: Component removed after UNDO: True", - "Physical Sky_test: Component added after REDO: True", - "Physical Sky_test: Entered game mode: True", - "Physical Sky_test: Exit game mode: True", - "Physical Sky_test: Entity is hidden: True", - "Physical Sky_test: Entity is shown: True", - "Physical Sky_test: Entity deleted: True", - "Physical Sky_test: UNDO entity deletion works: True", - "Physical Sky_test: REDO entity deletion works: True", - # PostFX Layer Component - "PostFX Layer Entity successfully created", - "PostFX Layer_test: Component added to the entity: True", - "PostFX Layer_test: Component removed after UNDO: True", - "PostFX Layer_test: Component added after REDO: True", - "PostFX Layer_test: Entered game mode: True", - "PostFX Layer_test: Exit game mode: True", - "PostFX Layer_test: Entity is hidden: True", - "PostFX Layer_test: Entity is shown: True", - "PostFX Layer_test: Entity deleted: True", - "PostFX Layer_test: UNDO entity deletion works: True", - "PostFX Layer_test: REDO entity deletion works: True", - # PostFX Radius Weight Modifier Component - "PostFX Radius Weight Modifier Entity successfully created", - "PostFX Radius Weight Modifier_test: Component added to the entity: True", - "PostFX Radius Weight Modifier_test: Component removed after UNDO: True", - "PostFX Radius Weight Modifier_test: Component added after REDO: True", - "PostFX Radius Weight Modifier_test: Entered game mode: True", - "PostFX Radius Weight Modifier_test: Exit game mode: True", - "PostFX Radius Weight Modifier_test: Entity is hidden: True", - "PostFX Radius Weight Modifier_test: Entity is shown: True", - "PostFX Radius Weight Modifier_test: Entity deleted: True", - "PostFX Radius Weight Modifier_test: UNDO entity deletion works: True", - "PostFX Radius Weight Modifier_test: REDO entity deletion works: True", - # Light Component - "Light Entity successfully created", - "Light_test: Component added to the entity: True", - "Light_test: Component removed after UNDO: True", - "Light_test: Component added after REDO: True", - "Light_test: Entered game mode: True", - "Light_test: Exit game mode: True", - "Light_test: Entity is hidden: True", - "Light_test: Entity is shown: True", - "Light_test: Entity deleted: True", - "Light_test: UNDO entity deletion works: True", - "Light_test: REDO entity deletion works: True", - # Display Mapper Component - "Display Mapper Entity successfully created", - "Display Mapper_test: Component added to the entity: True", - "Display Mapper_test: Component removed after UNDO: True", - "Display Mapper_test: Component added after REDO: True", - "Display Mapper_test: Entered game mode: True", - "Display Mapper_test: Exit game mode: True", - "Display Mapper_test: Entity is hidden: True", - "Display Mapper_test: Entity is shown: True", - "Display Mapper_test: Entity deleted: True", - "Display Mapper_test: UNDO entity deletion works: True", - "Display Mapper_test: REDO entity deletion works: True", - ] + @pytest.mark.test_case_id("C32078119") + class AtomEditorComponents_DepthOfFieldAdded(EditorSharedTest): + from Atom.tests import hydra_AtomEditorComponents_DepthOfFieldAdded as test_module - unexpected_lines = [ - "Trace::Assert", - "Trace::Error", - "Traceback (most recent call last):", - ] + @pytest.mark.test_case_id("C32078120") + class AtomEditorComponents_DirectionalLightAdded(EditorSharedTest): + from Atom.tests import hydra_AtomEditorComponents_DirectionalLightAdded as test_module - hydra.launch_and_validate_results( - request, - TEST_DIRECTORY, - editor, - "hydra_AtomEditorComponents_AddedToEntity.py", - timeout=120, - expected_lines=expected_lines, - unexpected_lines=unexpected_lines, - halt_on_unexpected=True, - null_renderer=True, - cfg_args=cfg_args, - ) + @pytest.mark.test_case_id("C36525660") + class AtomEditorComponents_DisplayMapperAdded(EditorSharedTest): + from Atom.tests import hydra_AtomEditorComponents_DisplayMapperAdded as test_module - @pytest.mark.test_case_id("C34525095") - def test_AtomEditorComponents_LightComponent( - self, request, editor, workspace, project, launcher_platform, level): - """ - Please review the hydra script run by this test for more specific test info. - Tests that the Light component has the expected property options available to it. - """ - cfg_args = [level] + @pytest.mark.test_case_id("C32078121") + class AtomEditorComponents_ExposureControlAdded(EditorSharedTest): + from Atom.tests import hydra_AtomEditorComponents_ExposureControlAdded as test_module - expected_lines = [ - "light_entity Entity successfully created", - "Entity has a Light component", - "light_entity_test: Component added to the entity: True", - f"light_entity_test: Property value is {LIGHT_TYPES['sphere']} which matches {LIGHT_TYPES['sphere']}", - "Controller|Configuration|Shadows|Enable shadow set to True", - "light_entity Controller|Configuration|Shadows|Shadowmap size: SUCCESS", - "Controller|Configuration|Shadows|Shadow filter method set to 1", # PCF - "Controller|Configuration|Shadows|Filtering sample count set to 4", - "Controller|Configuration|Shadows|Filtering sample count set to 64", - "Controller|Configuration|Shadows|Shadow filter method set to 2", # ESM - "Controller|Configuration|Shadows|ESM exponent set to 50.0", - "Controller|Configuration|Shadows|ESM exponent set to 5000.0", - "Controller|Configuration|Shadows|Shadow filter method set to 3", # ESM+PCF - f"light_entity_test: Property value is {LIGHT_TYPES['spot_disk']} which matches {LIGHT_TYPES['spot_disk']}", - f"light_entity_test: Property value is {LIGHT_TYPES['capsule']} which matches {LIGHT_TYPES['capsule']}", - f"light_entity_test: Property value is {LIGHT_TYPES['quad']} which matches {LIGHT_TYPES['quad']}", - "light_entity Controller|Configuration|Fast approximation: SUCCESS", - "light_entity Controller|Configuration|Both directions: SUCCESS", - f"light_entity_test: Property value is {LIGHT_TYPES['polygon']} which matches {LIGHT_TYPES['polygon']}", - f"light_entity_test: Property value is {LIGHT_TYPES['simple_point']} " - f"which matches {LIGHT_TYPES['simple_point']}", - "Controller|Configuration|Attenuation radius|Mode set to 0", - "Controller|Configuration|Attenuation radius|Radius set to 100.0", - f"light_entity_test: Property value is {LIGHT_TYPES['simple_spot']} " - f"which matches {LIGHT_TYPES['simple_spot']}", - "Controller|Configuration|Shutters|Outer angle set to 45.0", - "Controller|Configuration|Shutters|Outer angle set to 90.0", - "light_entity_test: Component added to the entity: True", - "Light component test (non-GPU) completed.", - ] + @pytest.mark.test_case_id("C32078115") + class AtomEditorComponents_GlobalSkylightIBLAdded(EditorSharedTest): + from Atom.tests import hydra_AtomEditorComponents_GlobalSkylightIBLAdded as test_module + + @pytest.mark.test_case_id("C32078122") + class AtomEditorComponents_GridAdded(EditorSharedTest): + from Atom.tests import hydra_AtomEditorComponents_GridAdded as test_module - unexpected_lines = [ - "Trace::Assert", - "Trace::Error", - "Traceback (most recent call last):", - ] + @pytest.mark.test_case_id("C36525671") + class AtomEditorComponents_HDRColorGradingAdded(EditorSharedTest): + from Atom.tests import hydra_AtomEditorComponents_HDRColorGradingAdded as test_module - hydra.launch_and_validate_results( - request, - TEST_DIRECTORY, - editor, - "hydra_AtomEditorComponents_LightComponent.py", - timeout=120, - expected_lines=expected_lines, - unexpected_lines=unexpected_lines, - halt_on_unexpected=True, - null_renderer=True, - cfg_args=cfg_args, - ) + @pytest.mark.test_case_id("C32078117") + class AtomEditorComponents_LightAdded(EditorSharedTest): + from Atom.tests import hydra_AtomEditorComponents_LightAdded as test_module + @pytest.mark.test_case_id("C32078123") + class AtomEditorComponents_MaterialAdded(EditorSharedTest): + from Atom.tests import hydra_AtomEditorComponents_MaterialAdded as test_module + @pytest.mark.test_case_id("C32078124") + class AtomEditorComponents_MeshAdded(EditorSharedTest): + from Atom.tests import hydra_AtomEditorComponents_MeshAdded as test_module + + @pytest.mark.test_case_id("C36525663") + class AtomEditorComponents_OcclusionCullingPlaneAdded(EditorSharedTest): + from Atom.tests import hydra_AtomEditorComponents_OcclusionCullingPlaneAdded as test_module + + @pytest.mark.test_case_id("C32078125") + class AtomEditorComponents_PhysicalSkyAdded(EditorSharedTest): + from Atom.tests import hydra_AtomEditorComponents_PhysicalSkyAdded as test_module + + @pytest.mark.test_case_id("C36525664") + class AtomEditorComponents_PostFXGradientWeightModifierAdded(EditorSharedTest): + from Atom.tests import hydra_AtomEditorComponents_PostFXGradientWeightModifierAdded as test_module + + @pytest.mark.test_case_id("C32078127") + class AtomEditorComponents_PostFXLayerAdded(EditorSharedTest): + from Atom.tests import hydra_AtomEditorComponents_PostFXLayerAdded as test_module + + @pytest.mark.test_case_id("C32078131") + class AtomEditorComponents_PostFXRadiusWeightModifierAdded(EditorSharedTest): + from Atom.tests import ( + hydra_AtomEditorComponents_PostFXRadiusWeightModifierAdded as test_module) + + @pytest.mark.test_case_id("C36525665") + class AtomEditorComponents_PostFXShapeWeightModifierAdded(EditorSharedTest): + from Atom.tests import hydra_AtomEditorComponents_PostFxShapeWeightModifierAdded as test_module + + @pytest.mark.test_case_id("C32078128") + class AtomEditorComponents_ReflectionProbeAdded(EditorSharedTest): + from Atom.tests import hydra_AtomEditorComponents_ReflectionProbeAdded as test_module + + class ShaderAssetBuilder_RecompilesShaderAsChainOfDependenciesChanges(EditorSharedTest): + from Atom.tests import hydra_ShaderAssetBuilder_RecompilesShaderAsChainOfDependenciesChanges as test_module diff --git a/AutomatedTesting/Gem/PythonTests/Atom/TestSuite_Main_Optimized.py b/AutomatedTesting/Gem/PythonTests/Atom/TestSuite_Main_Optimized.py deleted file mode 100644 index 950bd44199..0000000000 --- a/AutomatedTesting/Gem/PythonTests/Atom/TestSuite_Main_Optimized.py +++ /dev/null @@ -1,91 +0,0 @@ -""" -Copyright (c) Contributors to the Open 3D Engine Project. -For complete copyright and license terms please see the LICENSE at the root of this distribution. - -SPDX-License-Identifier: Apache-2.0 OR MIT -""" -import pytest - -from ly_test_tools.o3de.editor_test import EditorSharedTest, EditorTestSuite - - -@pytest.mark.xfail(reason="Optimized tests are experimental, we will enable xfail and monitor them temporarily.") -@pytest.mark.parametrize("project", ["AutomatedTesting"]) -@pytest.mark.parametrize("launcher_platform", ['windows_editor']) -class TestAutomation(EditorTestSuite): - - @pytest.mark.test_case_id("C32078118") - class AtomEditorComponents_DecalAdded(EditorSharedTest): - from Atom.tests import hydra_AtomEditorComponents_DecalAdded as test_module - - @pytest.mark.test_case_id("C32078119") - class AtomEditorComponents_DepthOfFieldAdded(EditorSharedTest): - from Atom.tests import hydra_AtomEditorComponents_DepthOfFieldAdded as test_module - - @pytest.mark.test_case_id("C32078120") - class AtomEditorComponents_DirectionalLightAdded(EditorSharedTest): - from Atom.tests import hydra_AtomEditorComponents_DirectionalLightAdded as test_module - - @pytest.mark.test_case_id("C36525660") - class AtomEditorComponents_DisplayMapperAdded(EditorSharedTest): - from Atom.tests import hydra_AtomEditorComponents_DisplayMapperAdded as test_module - - @pytest.mark.test_case_id("C32078121") - class AtomEditorComponents_ExposureControlAdded(EditorSharedTest): - from Atom.tests import hydra_AtomEditorComponents_ExposureControlAdded as test_module - - @pytest.mark.test_case_id("C32078115") - class AtomEditorComponents_GlobalSkylightIBLAdded(EditorSharedTest): - from Atom.tests import hydra_AtomEditorComponents_GlobalSkylightIBLAdded as test_module - - @pytest.mark.test_case_id("C32078122") - class AtomEditorComponents_GridAdded(EditorSharedTest): - from Atom.tests import hydra_AtomEditorComponents_GridAdded as test_module - - @pytest.mark.test_case_id("C36525671") - class AtomEditorComponents_HDRColorGradingAdded(EditorSharedTest): - from Atom.tests import hydra_AtomEditorComponents_HDRColorGradingAdded as test_module - - @pytest.mark.test_case_id("C32078117") - class AtomEditorComponents_LightAdded(EditorSharedTest): - from Atom.tests import hydra_AtomEditorComponents_LightAdded as test_module - - @pytest.mark.test_case_id("C32078123") - class AtomEditorComponents_MaterialAdded(EditorSharedTest): - from Atom.tests import hydra_AtomEditorComponents_MaterialAdded as test_module - - @pytest.mark.test_case_id("C32078124") - class AtomEditorComponents_MeshAdded(EditorSharedTest): - from Atom.tests import hydra_AtomEditorComponents_MeshAdded as test_module - - @pytest.mark.test_case_id("C36525663") - class AtomEditorComponents_OcclusionCullingPlaneAdded(EditorSharedTest): - from Atom.tests import hydra_AtomEditorComponents_OcclusionCullingPlaneAdded as test_module - - @pytest.mark.test_case_id("C32078125") - class AtomEditorComponents_PhysicalSkyAdded(EditorSharedTest): - from Atom.tests import hydra_AtomEditorComponents_PhysicalSkyAdded as test_module - - @pytest.mark.test_case_id("C36525664") - class AtomEditorComponents_PostFXGradientWeightModifierAdded(EditorSharedTest): - from Atom.tests import hydra_AtomEditorComponents_PostFXGradientWeightModifierAdded as test_module - - @pytest.mark.test_case_id("C32078127") - class AtomEditorComponents_PostFXLayerAdded(EditorSharedTest): - from Atom.tests import hydra_AtomEditorComponents_PostFXLayerAdded as test_module - - @pytest.mark.test_case_id("C32078131") - class AtomEditorComponents_PostFXRadiusWeightModifierAdded(EditorSharedTest): - from Atom.tests import ( - hydra_AtomEditorComponents_PostFXRadiusWeightModifierAdded as test_module) - - @pytest.mark.test_case_id("C36525665") - class AtomEditorComponents_PostFXShapeWeightModifierAdded(EditorSharedTest): - from Atom.tests import hydra_AtomEditorComponents_PostFxShapeWeightModifierAdded as test_module - - @pytest.mark.test_case_id("C32078128") - class AtomEditorComponents_ReflectionProbeAdded(EditorSharedTest): - from Atom.tests import hydra_AtomEditorComponents_ReflectionProbeAdded as test_module - - class ShaderAssetBuilder_RecompilesShaderAsChainOfDependenciesChanges(EditorSharedTest): - from Atom.tests import hydra_ShaderAssetBuilder_RecompilesShaderAsChainOfDependenciesChanges as test_module From 16526f58b7f0a4c02a90d3c8ca2ddec84bdc47f7 Mon Sep 17 00:00:00 2001 From: jromnoa <80134229+jromnoa@users.noreply.github.com> Date: Thu, 4 Nov 2021 11:52:28 -0700 Subject: [PATCH 11/34] remove xfail marker Signed-off-by: jromnoa <80134229+jromnoa@users.noreply.github.com> --- AutomatedTesting/Gem/PythonTests/Atom/TestSuite_Main.py | 1 - 1 file changed, 1 deletion(-) diff --git a/AutomatedTesting/Gem/PythonTests/Atom/TestSuite_Main.py b/AutomatedTesting/Gem/PythonTests/Atom/TestSuite_Main.py index 950bd44199..62a402d9c4 100644 --- a/AutomatedTesting/Gem/PythonTests/Atom/TestSuite_Main.py +++ b/AutomatedTesting/Gem/PythonTests/Atom/TestSuite_Main.py @@ -9,7 +9,6 @@ import pytest from ly_test_tools.o3de.editor_test import EditorSharedTest, EditorTestSuite -@pytest.mark.xfail(reason="Optimized tests are experimental, we will enable xfail and monitor them temporarily.") @pytest.mark.parametrize("project", ["AutomatedTesting"]) @pytest.mark.parametrize("launcher_platform", ['windows_editor']) class TestAutomation(EditorTestSuite): From b82ea09a67bec33a19e657d60b834da8e4dcb25c Mon Sep 17 00:00:00 2001 From: jromnoa <80134229+jromnoa@users.noreply.github.com> Date: Thu, 4 Nov 2021 11:54:21 -0700 Subject: [PATCH 12/34] remove redundant sandbox test, remove unsued/old hydra script for old log lines test approach Signed-off-by: jromnoa <80134229+jromnoa@users.noreply.github.com> --- ...ydra_AtomEditorComponents_AddedToEntity.py | 238 ------------------ 1 file changed, 238 deletions(-) delete mode 100644 AutomatedTesting/Gem/PythonTests/Atom/tests/hydra_AtomEditorComponents_AddedToEntity.py diff --git a/AutomatedTesting/Gem/PythonTests/Atom/tests/hydra_AtomEditorComponents_AddedToEntity.py b/AutomatedTesting/Gem/PythonTests/Atom/tests/hydra_AtomEditorComponents_AddedToEntity.py deleted file mode 100644 index bbc8463152..0000000000 --- a/AutomatedTesting/Gem/PythonTests/Atom/tests/hydra_AtomEditorComponents_AddedToEntity.py +++ /dev/null @@ -1,238 +0,0 @@ -""" -Copyright (c) Contributors to the Open 3D Engine Project. -For complete copyright and license terms please see the LICENSE at the root of this distribution. - -SPDX-License-Identifier: Apache-2.0 OR MIT -""" - -import os -import sys - -import azlmbr.math as math -import azlmbr.bus as bus -import azlmbr.paths -import azlmbr.asset as asset -import azlmbr.entity as entity -import azlmbr.legacy.general as general -import azlmbr.editor as editor -import azlmbr.render as render - -sys.path.append(os.path.join(azlmbr.paths.projectroot, "Gem", "PythonTests")) - -import editor_python_test_tools.hydra_editor_utils as hydra -from editor_python_test_tools.utils import TestHelper - - -def run(): - """ - Summary: - The below common tests are done for each of the components. - 1) Addition of component to the entity - 2) UNDO/REDO of addition of component - 3) Enter/Exit game mode - 4) Hide/Show entity containing component - 5) Deletion of component - 6) UNDO/REDO of deletion of component - Some additional tests for specific components include - 1) Assigning value to some properties of each component - 2) Verifying if the component is activated only when the required components are added - - Expected Result: - 1) Component can be added to an entity. - 2) The addition of component can be undone and redone. - 3) Game mode can be entered/exited without issue. - 4) Entity with component can be hidden/shown. - 5) Component can be deleted. - 6) The deletion of component can be undone and redone. - 7) Component is activated only when the required components are added - 8) Values can be assigned to the properties of the component - - :return: None - """ - - def create_entity_undo_redo_component_addition(component_name): - new_entity = hydra.Entity(f"{component_name}") - new_entity.create_entity(math.Vector3(512.0, 512.0, 34.0), [component_name]) - general.log(f"{component_name}_test: Component added to the entity: " - f"{hydra.has_components(new_entity.id, [component_name])}") - - # undo component addition - general.undo() - TestHelper.wait_for_condition(lambda: not hydra.has_components(new_entity.id, [component_name]), 2.0) - general.log(f"{component_name}_test: Component removed after UNDO: " - f"{not hydra.has_components(new_entity.id, [component_name])}") - - # redo component addition - general.redo() - TestHelper.wait_for_condition(lambda: hydra.has_components(new_entity.id, [component_name]), 2.0) - general.log(f"{component_name}_test: Component added after REDO: " - f"{hydra.has_components(new_entity.id, [component_name])}") - - return new_entity - - def verify_enter_exit_game_mode(component_name): - general.enter_game_mode() - TestHelper.wait_for_condition(lambda: general.is_in_game_mode(), 2.0) - general.log(f"{component_name}_test: Entered game mode: {general.is_in_game_mode()}") - general.exit_game_mode() - TestHelper.wait_for_condition(lambda: not general.is_in_game_mode(), 2.0) - general.log(f"{component_name}_test: Exit game mode: {not general.is_in_game_mode()}") - - def verify_hide_unhide_entity(component_name, entity_obj): - - def is_entity_hidden(entity_id): - return editor.EditorEntityInfoRequestBus(bus.Event, "IsHidden", entity_id) - - editor.EditorEntityAPIBus(bus.Event, "SetVisibilityState", entity_obj.id, False) - general.idle_wait_frames(1) - general.log(f"{component_name}_test: Entity is hidden: {is_entity_hidden(entity_obj.id)}") - editor.EditorEntityAPIBus(bus.Event, "SetVisibilityState", entity_obj.id, True) - general.idle_wait_frames(1) - general.log(f"{component_name}_test: Entity is shown: {not is_entity_hidden(entity_obj.id)}") - - def verify_deletion_undo_redo(component_name, entity_obj): - editor.ToolsApplicationRequestBus(bus.Broadcast, "DeleteEntityById", entity_obj.id) - TestHelper.wait_for_condition(lambda: not hydra.find_entity_by_name(entity_obj.name), 2.0) - general.log(f"{component_name}_test: Entity deleted: {not hydra.find_entity_by_name(entity_obj.name)}") - - general.undo() - TestHelper.wait_for_condition(lambda: hydra.find_entity_by_name(entity_obj.name) is not None, 2.0) - general.log(f"{component_name}_test: UNDO entity deletion works: " - f"{hydra.find_entity_by_name(entity_obj.name) is not None}") - - general.redo() - TestHelper.wait_for_condition(lambda: not hydra.find_entity_by_name(entity_obj.name), 2.0) - general.log(f"{component_name}_test: REDO entity deletion works: " - f"{not hydra.find_entity_by_name(entity_obj.name)}") - - def verify_required_component_addition(entity_obj, components_to_add, component_name): - - def is_component_enabled(entity_componentid_pair): - return editor.EditorComponentAPIBus(bus.Broadcast, "IsComponentEnabled", entity_componentid_pair) - - general.log( - f"{component_name}_test: Entity disabled initially: " - f"{not is_component_enabled(entity_obj.components[0])}") - for component in components_to_add: - entity_obj.add_component(component) - TestHelper.wait_for_condition(lambda: is_component_enabled(entity_obj.components[0]), 2.0) - general.log( - f"{component_name}_test: Entity enabled after adding " - f"required components: {is_component_enabled(entity_obj.components[0])}" - ) - - def verify_set_property(entity_obj, path, value): - entity_obj.get_set_test(0, path, value) - - # Verify cubemap generation - def verify_cubemap_generation(component_name, entity_obj): - # Initially Check if the component has Reflection Probe component - if not hydra.has_components(entity_obj.id, ["Reflection Probe"]): - raise ValueError(f"Given entity {entity_obj.name} has no Reflection Probe component") - render.EditorReflectionProbeBus(azlmbr.bus.Event, "BakeReflectionProbe", entity_obj.id) - - def get_value(): - hydra.get_component_property_value(entity_obj.components[0], "Cubemap|Baked Cubemap Path") - - TestHelper.wait_for_condition(lambda: get_value() != "", 20.0) - general.log(f"{component_name}_test: Cubemap is generated: {get_value() != ''}") - - # Wait for Editor idle loop before executing Python hydra scripts. - TestHelper.init_idle() - - # Delete all existing entities initially - search_filter = azlmbr.entity.SearchFilter() - all_entities = entity.SearchBus(azlmbr.bus.Broadcast, "SearchEntities", search_filter) - editor.ToolsApplicationRequestBus(bus.Broadcast, "DeleteEntities", all_entities) - - class ComponentTests: - """Test launcher for each component.""" - def __init__(self, component_name, *additional_tests): - self.component_name = component_name - self.additional_tests = additional_tests - self.run_component_tests() - - def run_component_tests(self): - # Run common and additional tests - entity_obj = create_entity_undo_redo_component_addition(self.component_name) - - # Enter/Exit game mode test - verify_enter_exit_game_mode(self.component_name) - - # Any additional tests are executed here - for test in self.additional_tests: - test(entity_obj) - - # Hide/Unhide entity test - verify_hide_unhide_entity(self.component_name, entity_obj) - - # Deletion/Undo/Redo test - verify_deletion_undo_redo(self.component_name, entity_obj) - - # DepthOfField Component - camera_entity = hydra.Entity("camera_entity") - camera_entity.create_entity(math.Vector3(512.0, 512.0, 34.0), ["Camera"]) - depth_of_field = "DepthOfField" - ComponentTests( - depth_of_field, - lambda entity_obj: verify_required_component_addition(entity_obj, ["PostFX Layer"], depth_of_field), - lambda entity_obj: verify_set_property( - entity_obj, "Controller|Configuration|Camera Entity", camera_entity.id)) - - # Decal Component - material_asset_path = os.path.join("AutomatedTesting", "Materials", "basic_grey.material") - material_asset = asset.AssetCatalogRequestBus( - bus.Broadcast, "GetAssetIdByPath", material_asset_path, math.Uuid(), False) - ComponentTests( - "Decal", lambda entity_obj: verify_set_property( - entity_obj, "Controller|Configuration|Material", material_asset)) - - # Directional Light Component - ComponentTests( - "Directional Light", - lambda entity_obj: verify_set_property( - entity_obj, "Controller|Configuration|Shadow|Camera", camera_entity.id)) - - # Exposure Control Component - ComponentTests( - "Exposure Control", lambda entity_obj: verify_required_component_addition( - entity_obj, ["PostFX Layer"], "Exposure Control")) - - # Global Skylight (IBL) Component - diffuse_image_path = os.path.join("LightingPresets", "greenwich_park_02_4k_iblskyboxcm.exr.streamingimage") - diffuse_image_asset = asset.AssetCatalogRequestBus( - bus.Broadcast, "GetAssetIdByPath", diffuse_image_path, math.Uuid(), False) - specular_image_path = os.path.join("LightingPresets", "greenwich_park_02_4k_iblskyboxcm.exr.streamingimage") - specular_image_asset = asset.AssetCatalogRequestBus( - bus.Broadcast, "GetAssetIdByPath", specular_image_path, math.Uuid(), False) - ComponentTests( - "Global Skylight (IBL)", - lambda entity_obj: verify_set_property( - entity_obj, "Controller|Configuration|Diffuse Image", diffuse_image_asset), - lambda entity_obj: verify_set_property( - entity_obj, "Controller|Configuration|Specular Image", specular_image_asset)) - - # Physical Sky Component - ComponentTests("Physical Sky") - - # PostFX Layer Component - ComponentTests("PostFX Layer") - - # PostFX Radius Weight Modifier Component - ComponentTests("PostFX Radius Weight Modifier") - - # Light Component - ComponentTests("Light") - - # Display Mapper Component - ComponentTests("Display Mapper") - - # Reflection Probe Component - reflection_probe = "Reflection Probe" - ComponentTests( - reflection_probe, - lambda entity_obj: verify_required_component_addition(entity_obj, ["Box Shape"], reflection_probe), - lambda entity_obj: verify_cubemap_generation(reflection_probe, entity_obj),) - -if __name__ == "__main__": - run() From ae72463581d7bcf59ee3f8f9816ab81c3a89b3e6 Mon Sep 17 00:00:00 2001 From: jromnoa <80134229+jromnoa@users.noreply.github.com> Date: Thu, 4 Nov 2021 11:54:49 -0700 Subject: [PATCH 13/34] sandbox portion Signed-off-by: jromnoa <80134229+jromnoa@users.noreply.github.com> --- .../Gem/PythonTests/Atom/TestSuite_Sandbox.py | 50 +------------------ 1 file changed, 2 insertions(+), 48 deletions(-) diff --git a/AutomatedTesting/Gem/PythonTests/Atom/TestSuite_Sandbox.py b/AutomatedTesting/Gem/PythonTests/Atom/TestSuite_Sandbox.py index ad45e51080..55b1540b7e 100644 --- a/AutomatedTesting/Gem/PythonTests/Atom/TestSuite_Sandbox.py +++ b/AutomatedTesting/Gem/PythonTests/Atom/TestSuite_Sandbox.py @@ -9,63 +9,19 @@ import os import pytest +import ly_test_tools.environment.file_system as file_system import editor_python_test_tools.hydra_test_utils as hydra logger = logging.getLogger(__name__) TEST_DIRECTORY = os.path.join(os.path.dirname(__file__), "tests") + class TestAtomEditorComponentsSandbox(object): # It requires at least one test def test_Dummy(self, request, editor, level, workspace, project, launcher_platform): pass - @pytest.mark.parametrize("project", ["AutomatedTesting"]) - @pytest.mark.parametrize("launcher_platform", ['windows_editor']) - @pytest.mark.parametrize("level", ["auto_test"]) - class TestAtomEditorComponentsMain(object): - """Holds tests for Atom components.""" - - @pytest.mark.test_case_id("C32078128") - def test_AtomEditorComponents_ReflectionProbeAddedToEntity( - self, request, editor, level, workspace, project, launcher_platform): - """ - Please review the hydra script run by this test for more specific test info. - Tests the following Atom components and verifies all "expected_lines" appear in Editor.log: - 1. Reflection Probe - """ - cfg_args = [level] - - expected_lines = [ - # Reflection Probe Component - "Reflection Probe Entity successfully created", - "Reflection Probe_test: Component added to the entity: True", - "Reflection Probe_test: Component removed after UNDO: True", - "Reflection Probe_test: Component added after REDO: True", - "Reflection Probe_test: Entered game mode: True", - "Reflection Probe_test: Exit game mode: True", - "Reflection Probe_test: Entity disabled initially: True", - "Reflection Probe_test: Entity enabled after adding required components: True", - "Reflection Probe_test: Cubemap is generated: True", - "Reflection Probe_test: Entity is hidden: True", - "Reflection Probe_test: Entity is shown: True", - "Reflection Probe_test: Entity deleted: True", - "Reflection Probe_test: UNDO entity deletion works: True", - "Reflection Probe_test: REDO entity deletion works: True", - ] - - hydra.launch_and_validate_results( - request, - TEST_DIRECTORY, - editor, - "hydra_AtomEditorComponents_AddedToEntity.py", - timeout=120, - expected_lines=expected_lines, - unexpected_lines=[], - halt_on_unexpected=True, - null_renderer=True, - cfg_args=cfg_args, - ) @pytest.mark.parametrize("project", ["AutomatedTesting"]) @pytest.mark.parametrize("launcher_platform", ['windows_generic']) @@ -119,8 +75,6 @@ class TestMaterialEditorBasicTests(object): "Save All worked as expected: True", ] unexpected_lines = [ - # "Trace::Assert", - # "Trace::Error", "Traceback (most recent call last):" ] From 6f294457da88290ef3a542dfb5c1b8ac943bc82f Mon Sep 17 00:00:00 2001 From: Vishal Das Date: Fri, 5 Nov 2021 00:41:31 +0530 Subject: [PATCH 14/34] fix issue #5172 (#5198) Signed-off-by: Vishal Das --- .../ComponentEntityEditorPlugin/UI/Outliner/EntityOutliner.qss | 1 - .../AzToolsFramework/UI/Outliner/EntityOutliner.qss | 1 - 2 files changed, 2 deletions(-) diff --git a/Code/Editor/Plugins/ComponentEntityEditorPlugin/UI/Outliner/EntityOutliner.qss b/Code/Editor/Plugins/ComponentEntityEditorPlugin/UI/Outliner/EntityOutliner.qss index de35f91678..a7d3949527 100644 --- a/Code/Editor/Plugins/ComponentEntityEditorPlugin/UI/Outliner/EntityOutliner.qss +++ b/Code/Editor/Plugins/ComponentEntityEditorPlugin/UI/Outliner/EntityOutliner.qss @@ -11,7 +11,6 @@ OutlinerWidget #m_display_options { qproperty-icon: url(:/Menu/menu.svg); qproperty-iconSize: 16px 16px; - qproperty-flat: true; } OutlinerWidget QWidget[PulseHighlight="true"] diff --git a/Code/Framework/AzToolsFramework/AzToolsFramework/UI/Outliner/EntityOutliner.qss b/Code/Framework/AzToolsFramework/AzToolsFramework/UI/Outliner/EntityOutliner.qss index b3c5882334..93460f8816 100644 --- a/Code/Framework/AzToolsFramework/AzToolsFramework/UI/Outliner/EntityOutliner.qss +++ b/Code/Framework/AzToolsFramework/AzToolsFramework/UI/Outliner/EntityOutliner.qss @@ -10,7 +10,6 @@ AzToolsFramework--EntityOutlinerWidget #m_display_options { qproperty-icon: url(:/stylesheet/img/UI20/menu-centered.svg); qproperty-iconSize: 16px 16px; - qproperty-flat: true; } AzToolsFramework--EntityOutlinerWidget QTreeView From f26e272a8dc39e1e94946f4e8e32367f48963072 Mon Sep 17 00:00:00 2001 From: jromnoa <80134229+jromnoa@users.noreply.github.com> Date: Thu, 4 Nov 2021 12:20:38 -0700 Subject: [PATCH 15/34] move light component non optimized test to sandbox until it gets optimized (ticket cut for this) Signed-off-by: jromnoa <80134229+jromnoa@users.noreply.github.com> --- .../Gem/PythonTests/Atom/TestSuite_Sandbox.py | 65 +++++++++++++++++++ 1 file changed, 65 insertions(+) diff --git a/AutomatedTesting/Gem/PythonTests/Atom/TestSuite_Sandbox.py b/AutomatedTesting/Gem/PythonTests/Atom/TestSuite_Sandbox.py index 55b1540b7e..9ceb3c951f 100644 --- a/AutomatedTesting/Gem/PythonTests/Atom/TestSuite_Sandbox.py +++ b/AutomatedTesting/Gem/PythonTests/Atom/TestSuite_Sandbox.py @@ -12,6 +12,8 @@ import pytest import ly_test_tools.environment.file_system as file_system import editor_python_test_tools.hydra_test_utils as hydra +from Atom.atom_utils.atom_constants import LIGHT_TYPES + logger = logging.getLogger(__name__) TEST_DIRECTORY = os.path.join(os.path.dirname(__file__), "tests") @@ -23,6 +25,69 @@ class TestAtomEditorComponentsSandbox(object): pass +@pytest.mark.parametrize("project", ["AutomatedTesting"]) +@pytest.mark.parametrize("launcher_platform", ['windows_editor']) +@pytest.mark.parametrize("level", ["auto_test"]) +class TestAtomEditorComponentsMain(object): + """Holds tests for Atom components.""" + + @pytest.mark.test_case_id("C34525095") + def test_AtomEditorComponents_LightComponent( + self, request, editor, workspace, project, launcher_platform, level): + """ + Please review the hydra script run by this test for more specific test info. + Tests that the Light component has the expected property options available to it. + """ + cfg_args = [level] + + expected_lines = [ + "light_entity Entity successfully created", + "Entity has a Light component", + "light_entity_test: Component added to the entity: True", + f"light_entity_test: Property value is {LIGHT_TYPES['sphere']} which matches {LIGHT_TYPES['sphere']}", + "Controller|Configuration|Shadows|Enable shadow set to True", + "light_entity Controller|Configuration|Shadows|Shadowmap size: SUCCESS", + "Controller|Configuration|Shadows|Shadow filter method set to 1", # PCF + "Controller|Configuration|Shadows|Filtering sample count set to 4", + "Controller|Configuration|Shadows|Filtering sample count set to 64", + "Controller|Configuration|Shadows|Shadow filter method set to 2", # ESM + "Controller|Configuration|Shadows|ESM exponent set to 50.0", + "Controller|Configuration|Shadows|ESM exponent set to 5000.0", + "Controller|Configuration|Shadows|Shadow filter method set to 3", # ESM+PCF + f"light_entity_test: Property value is {LIGHT_TYPES['spot_disk']} which matches {LIGHT_TYPES['spot_disk']}", + f"light_entity_test: Property value is {LIGHT_TYPES['capsule']} which matches {LIGHT_TYPES['capsule']}", + f"light_entity_test: Property value is {LIGHT_TYPES['quad']} which matches {LIGHT_TYPES['quad']}", + "light_entity Controller|Configuration|Fast approximation: SUCCESS", + "light_entity Controller|Configuration|Both directions: SUCCESS", + f"light_entity_test: Property value is {LIGHT_TYPES['polygon']} which matches {LIGHT_TYPES['polygon']}", + f"light_entity_test: Property value is {LIGHT_TYPES['simple_point']} " + f"which matches {LIGHT_TYPES['simple_point']}", + "Controller|Configuration|Attenuation radius|Mode set to 0", + "Controller|Configuration|Attenuation radius|Radius set to 100.0", + f"light_entity_test: Property value is {LIGHT_TYPES['simple_spot']} " + f"which matches {LIGHT_TYPES['simple_spot']}", + "Controller|Configuration|Shutters|Outer angle set to 45.0", + "Controller|Configuration|Shutters|Outer angle set to 90.0", + "light_entity_test: Component added to the entity: True", + "Light component test (non-GPU) completed.", + ] + + unexpected_lines = ["Traceback (most recent call last):"] + + hydra.launch_and_validate_results( + request, + TEST_DIRECTORY, + editor, + "hydra_AtomEditorComponents_LightComponent.py", + timeout=120, + expected_lines=expected_lines, + unexpected_lines=unexpected_lines, + halt_on_unexpected=True, + null_renderer=True, + cfg_args=cfg_args, + ) + + @pytest.mark.parametrize("project", ["AutomatedTesting"]) @pytest.mark.parametrize("launcher_platform", ['windows_generic']) @pytest.mark.system From 5a3cd96eab655260f494daca914e91e8bc492725 Mon Sep 17 00:00:00 2001 From: Esteban Papp <81431996+amznestebanpapp@users.noreply.github.com> Date: Thu, 4 Nov 2021 12:23:41 -0700 Subject: [PATCH 16/34] Fixes for CMake 3.22rc (#5314) Signed-off-by: Esteban Papp <81431996+amznestebanpapp@users.noreply.github.com> --- .../Platform/Common/MSVC/Configurations_msvc.cmake | 13 +++++++++++-- 1 file changed, 11 insertions(+), 2 deletions(-) diff --git a/cmake/Platform/Common/MSVC/Configurations_msvc.cmake b/cmake/Platform/Common/MSVC/Configurations_msvc.cmake index a4d8533626..66a5b7b01f 100644 --- a/cmake/Platform/Common/MSVC/Configurations_msvc.cmake +++ b/cmake/Platform/Common/MSVC/Configurations_msvc.cmake @@ -139,11 +139,20 @@ endif() # Configure system includes ly_set(LY_CXX_SYSTEM_INCLUDE_CONFIGURATION_FLAG - /experimental:external # Turns on "external" headers feature for MSVC compilers + /experimental:external # Turns on "external" headers feature for MSVC compilers, required for MSVC < 16.10 /external:W0 # Set warning level in external headers to 0. This is used to suppress warnings 3rdParty libraries which uses the "system_includes" option in their json configuration ) + +# CMake 3.22rc added a definition for CMAKE_INCLUDE_SYSTEM_FLAG_CXX. However, its defined as "-external:I ", that space causes +# issues when trying to use in TargetIncludeSystemDirectories_unsupported.cmake. +# CMake 3.22rc has also not added support for external directories in MSVC through target_include_directories(... SYSTEM +# So we will just fix the flag that was added by 3.22rc so it works with our TargetIncludeSystemDirectories_unsupported.cmake +# Once target_include_directories(... SYSTEM is supported, we can branch and use TargetIncludeSystemDirectories_supported.cmake +# Reported this here: https://gitlab.kitware.com/cmake/cmake/-/issues/17904#note_1078281 if(NOT CMAKE_INCLUDE_SYSTEM_FLAG_CXX) - ly_set(CMAKE_INCLUDE_SYSTEM_FLAG_CXX /external:I) + ly_set(CMAKE_INCLUDE_SYSTEM_FLAG_CXX "/external:I") +else() + string(STRIP ${CMAKE_INCLUDE_SYSTEM_FLAG_CXX} CMAKE_INCLUDE_SYSTEM_FLAG_CXX) endif() include(cmake/Platform/Common/TargetIncludeSystemDirectories_unsupported.cmake) From 64ab8d5956c7024280826dae7d63a482f60b67ee Mon Sep 17 00:00:00 2001 From: Sean Masterson Date: Thu, 4 Nov 2021 12:13:25 -0700 Subject: [PATCH 17/34] Add P0 Bloom Test Signed-off-by: Sean Masterson --- .../Atom/TestSuite_Main_Optimized.py | 4 + .../Atom/atom_utils/atom_constants.py | 3 + .../hydra_AtomEditorComponents_BloomAdded.py | 188 ++++++++++++++++++ 3 files changed, 195 insertions(+) create mode 100644 AutomatedTesting/Gem/PythonTests/Atom/tests/hydra_AtomEditorComponents_BloomAdded.py diff --git a/AutomatedTesting/Gem/PythonTests/Atom/TestSuite_Main_Optimized.py b/AutomatedTesting/Gem/PythonTests/Atom/TestSuite_Main_Optimized.py index 950bd44199..18c77391dc 100644 --- a/AutomatedTesting/Gem/PythonTests/Atom/TestSuite_Main_Optimized.py +++ b/AutomatedTesting/Gem/PythonTests/Atom/TestSuite_Main_Optimized.py @@ -14,6 +14,10 @@ from ly_test_tools.o3de.editor_test import EditorSharedTest, EditorTestSuite @pytest.mark.parametrize("launcher_platform", ['windows_editor']) class TestAutomation(EditorTestSuite): + @pytest.mark.test_case_id("C36525657") + class AtomEditorComponents_BloomAdded(EditorSharedTest): + from Atom.tests import hydra_AtomEditorComponents_BloomAdded as test_module + @pytest.mark.test_case_id("C32078118") class AtomEditorComponents_DecalAdded(EditorSharedTest): from Atom.tests import hydra_AtomEditorComponents_DecalAdded as test_module diff --git a/AutomatedTesting/Gem/PythonTests/Atom/atom_utils/atom_constants.py b/AutomatedTesting/Gem/PythonTests/Atom/atom_utils/atom_constants.py index c365999335..41ffaa0ce1 100644 --- a/AutomatedTesting/Gem/PythonTests/Atom/atom_utils/atom_constants.py +++ b/AutomatedTesting/Gem/PythonTests/Atom/atom_utils/atom_constants.py @@ -42,12 +42,14 @@ class AtomComponentProperties: Bloom component properties. Requires PostFX Layer component. - 'requires' a list of component names as strings required by this component. Use editor_entity_utils EditorEntity.add_components(list) to add this list of requirements.\n + - 'Enable Bloom' Toggle active state of the component True/False :param property: From the last element of the property tree path. Default 'name' for component name string. :return: Full property path OR component name if no property specified. """ properties = { 'name': 'Bloom', 'requires': [AtomComponentProperties.postfx_layer()], + 'Enable Bloom': 'Controller|Configuration|Enable Bloom', } return properties[property] @@ -212,6 +214,7 @@ class AtomComponentProperties: HDR Color Grading component properties. Requires PostFX Layer component. - 'requires' a list of component names as strings required by this component. Use editor_entity_utils EditorEntity.add_components(list) to add this list of requirements.\n + - 'Enable HDR color grading' Toggle active state of the component True/False :param property: From the last element of the property tree path. Default 'name' for component name string. :return: Full property path OR component name if no property specified. """ diff --git a/AutomatedTesting/Gem/PythonTests/Atom/tests/hydra_AtomEditorComponents_BloomAdded.py b/AutomatedTesting/Gem/PythonTests/Atom/tests/hydra_AtomEditorComponents_BloomAdded.py new file mode 100644 index 0000000000..bce4b86a35 --- /dev/null +++ b/AutomatedTesting/Gem/PythonTests/Atom/tests/hydra_AtomEditorComponents_BloomAdded.py @@ -0,0 +1,188 @@ +""" +Copyright (c) Contributors to the Open 3D Engine Project. +For complete copyright and license terms please see the LICENSE at the root of this distribution. + +SPDX-License-Identifier: Apache-2.0 OR MIT +""" + +class Tests: + creation_undo = ( + "UNDO Entity creation success", + "UNDO Entity creation failed") + creation_redo = ( + "REDO Entity creation success", + "REDO Entity creation failed") + bloom_creation = ( + "Bloom Entity successfully created", + "Bloom Entity failed to be created") + bloom_component = ( + "Entity has a Bloom component", + "Entity failed to find Bloom component") + bloom_disabled = ( + "Bloom component disabled", + "Bloom component was not disabled") + postfx_layer_component = ( + "Entity has a PostFX Layer component", + "Entity did not have an PostFX Layer component") + bloom_enabled = ( + "Bloom component enabled", + "Bloom component was not enabled") + enable_bloom_parameter_enabled = ( + "Enable Bloom parameter enabled", + "Enable Bloom parameter was not enabled") + enter_game_mode = ( + "Entered game mode", + "Failed to enter game mode") + exit_game_mode = ( + "Exited game mode", + "Couldn't exit game mode") + is_visible = ( + "Entity is visible", + "Entity was not visible") + is_hidden = ( + "Entity is hidden", + "Entity was not hidden") + entity_deleted = ( + "Entity deleted", + "Entity was not deleted") + deletion_undo = ( + "UNDO deletion success", + "UNDO deletion failed") + deletion_redo = ( + "REDO deletion success", + "REDO deletion failed") + + +def AtomEditorComponents_Bloom_AddedToEntity(): + """ + Summary: + Tests the Bloom component can be added to an entity and has the expected functionality. + + Test setup: + - Wait for Editor idle loop. + - Open the "Base" level. + + Expected Behavior: + The component can be added, used in game mode, hidden/shown, deleted, and has accurate required components. + Creation and deletion undo/redo should also work. + + Test Steps: + 1) Create an Bloom entity with no components. + 2) Add Bloom component to Bloom entity. + 3) UNDO the entity creation and component addition. + 4) REDO the entity creation and component addition. + 5) Verify Bloom component not enabled. + 6) Add PostFX Layer component since it is required by the Bloom component. + 7) Verify Bloom component is enabled. + 8) Enable the "Enable Bloom" parameter. + 9) Enter/Exit game mode. + 10) Test IsHidden. + 11) Test IsVisible. + 12) Delete Bloom entity. + 13) UNDO deletion. + 14) REDO deletion. + 15) Look for errors. + + :return: None + """ + + import azlmbr.legacy.general as general + + from editor_python_test_tools.editor_entity_utils import EditorEntity + from editor_python_test_tools.utils import Report, Tracer, TestHelper + from Atom.atom_utils.atom_constants import AtomComponentProperties + + with Tracer() as error_tracer: + # Test setup begins. + # Setup: Wait for Editor idle loop before executing Python hydra scripts then open "Base" level. + TestHelper.init_idle() + TestHelper.open_level("", "Base") + + # Test steps begin. + # 1. Create an Bloom entity with no components. + bloom_entity = EditorEntity.create_editor_entity(AtomComponentProperties.bloom()) + Report.critical_result(Tests.bloom_creation, bloom_entity.exists()) + + # 2. Add Bloom component to Bloom entity. + bloom_component = bloom_entity.add_component(AtomComponentProperties.bloom()) + Report.critical_result(Tests.bloom_component, bloom_entity.has_component(AtomComponentProperties.bloom())) + + # 3. UNDO the entity creation and component addition. + # -> UNDO component addition. + general.undo() + # -> UNDO naming entity. + general.undo() + # -> UNDO selecting entity. + general.undo() + # -> UNDO entity creation. + general.undo() + general.idle_wait_frames(1) + Report.result(Tests.creation_undo, not bloom_entity.exists()) + + # 4. REDO the entity creation and component addition. + # -> REDO entity creation. + general.redo() + # -> REDO selecting entity. + general.redo() + # -> REDO naming entity. + general.redo() + # -> REDO component addition. + general.redo() + general.idle_wait_frames(1) + Report.result(Tests.creation_redo, bloom_entity.exists()) + + # 5. Verify Bloom component not enabled. + Report.result(Tests.bloom_disabled, not bloom_component.is_enabled()) + + # 6. Add PostFX Layer component since it is required by the Bloom component. + bloom_entity.add_component(AtomComponentProperties.postfx_layer()) + Report.result( + Tests.postfx_layer_component, + bloom_entity.has_component(AtomComponentProperties.postfx_layer())) + + # 7. Verify Bloom component is enabled. + Report.result(Tests.bloom_enabled, bloom_component.is_enabled()) + + # 8. Enable the "Enable Bloom" parameter. + bloom_component.set_component_property_value(AtomComponentProperties.bloom('Enable Bloom'), True) + Report.result( + Tests.enable_bloom_parameter_enabled, + bloom_component.get_component_property_value(AtomComponentProperties.bloom('Enable Bloom')) is True) + + # 9. Enter/Exit game mode. + TestHelper.enter_game_mode(Tests.enter_game_mode) + general.idle_wait_frames(1) + TestHelper.exit_game_mode(Tests.exit_game_mode) + + # 10. Test IsHidden. + bloom_entity.set_visibility_state(False) + Report.result(Tests.is_hidden, bloom_entity.is_hidden() is True) + + # 11. Test IsVisible. + bloom_entity.set_visibility_state(True) + general.idle_wait_frames(1) + Report.result(Tests.is_visible, bloom_entity.is_visible() is True) + + # 12. Delete Bloom entity. + bloom_entity.delete() + Report.result(Tests.entity_deleted, not bloom_entity.exists()) + + # 13. UNDO deletion. + general.undo() + Report.result(Tests.deletion_undo, bloom_entity.exists()) + + # 14. REDO deletion. + general.redo() + Report.result(Tests.deletion_redo, not bloom_entity.exists()) + + # 15. Look for errors and asserts. + TestHelper.wait_for_condition(lambda: error_tracer.has_errors or error_tracer.has_asserts, 1.0) + for error_info in error_tracer.errors: + Report.info(f"Error: {error_info.filename} {error_info.function} | {error_info.message}") + for assert_info in error_tracer.asserts: + Report.info(f"Assert: {assert_info.filename} {assert_info.function} | {assert_info.message}") + + +if __name__ == "__main__": + from editor_python_test_tools.utils import Report + Report.start_test(AtomEditorComponents_Bloom_AddedToEntity) From 1f4cb58c5ebdfa1c1a68e200551737aea358c943 Mon Sep 17 00:00:00 2001 From: Gene Walters Date: Thu, 4 Nov 2021 13:49:33 -0700 Subject: [PATCH 18/34] Moving flaky multiplayer tests into sandbox so they will still be ran nightly, but we can still fix and create new tests Signed-off-by: Gene Walters --- .../Gem/PythonTests/CMakeLists.txt | 3 ++ .../PythonTests/Multiplayer/CMakeLists.txt | 13 +++++++ .../PythonTests/Multiplayer/TestSuite_Main.py | 4 --- .../Multiplayer/TestSuite_Sandbox.py | 35 +++++++++++++++++++ 4 files changed, 51 insertions(+), 4 deletions(-) create mode 100644 AutomatedTesting/Gem/PythonTests/Multiplayer/TestSuite_Sandbox.py diff --git a/AutomatedTesting/Gem/PythonTests/CMakeLists.txt b/AutomatedTesting/Gem/PythonTests/CMakeLists.txt index fd3222ba83..bec49185bd 100644 --- a/AutomatedTesting/Gem/PythonTests/CMakeLists.txt +++ b/AutomatedTesting/Gem/PythonTests/CMakeLists.txt @@ -59,5 +59,8 @@ add_subdirectory(smoke) ## AWS ## add_subdirectory(AWS) +## Multiplayer ## +add_subdirectory(Multiplayer) + ## Integration tests for editor testing framework ## add_subdirectory(editor_test_testing) diff --git a/AutomatedTesting/Gem/PythonTests/Multiplayer/CMakeLists.txt b/AutomatedTesting/Gem/PythonTests/Multiplayer/CMakeLists.txt index 5e74d1e93b..506d15c323 100644 --- a/AutomatedTesting/Gem/PythonTests/Multiplayer/CMakeLists.txt +++ b/AutomatedTesting/Gem/PythonTests/Multiplayer/CMakeLists.txt @@ -20,4 +20,17 @@ if(PAL_TRAIT_BUILD_TESTS_SUPPORTED AND PAL_TRAIT_BUILD_HOST_TOOLS) COMPONENT Multiplayer ) + ly_add_pytest( + NAME AutomatedTesting::MultiplayerTests_Sandbox + TEST_SUITE sandbox + TEST_SERIAL + PATH ${CMAKE_CURRENT_LIST_DIR}/TestSuite_Sandbox.py + RUNTIME_DEPENDENCIES + Legacy::Editor + AZ::AssetProcessor + AutomatedTesting.Assets + AutomatedTesting.ServerLauncher + COMPONENT + Multiplayer + ) endif() diff --git a/AutomatedTesting/Gem/PythonTests/Multiplayer/TestSuite_Main.py b/AutomatedTesting/Gem/PythonTests/Multiplayer/TestSuite_Main.py index 9cecaa7fe8..df1eb62943 100644 --- a/AutomatedTesting/Gem/PythonTests/Multiplayer/TestSuite_Main.py +++ b/AutomatedTesting/Gem/PythonTests/Multiplayer/TestSuite_Main.py @@ -27,7 +27,3 @@ class TestAutomation(TestAutomationBase): batch_mode=batch_mode, autotest_mode=autotest_mode) - def test_Multiplayer_AutoComponent_NetworkInput(self, request, workspace, editor, launcher_platform): - from .tests import Multiplayer_AutoComponent_NetworkInput as test_module - self._run_prefab_test(request, workspace, editor, test_module) - diff --git a/AutomatedTesting/Gem/PythonTests/Multiplayer/TestSuite_Sandbox.py b/AutomatedTesting/Gem/PythonTests/Multiplayer/TestSuite_Sandbox.py new file mode 100644 index 0000000000..52ac19b26e --- /dev/null +++ b/AutomatedTesting/Gem/PythonTests/Multiplayer/TestSuite_Sandbox.py @@ -0,0 +1,35 @@ +""" +Copyright (c) Contributors to the Open 3D Engine Project. +For complete copyright and license terms please see the LICENSE at the root of this distribution. + +SPDX-License-Identifier: Apache-2.0 OR MIT + +""" + +# This suite consists of all test cases that are under development and have not been verified yet. +# Once they are verified, please move them to TestSuite_Active.py + +import pytest +import os +import sys + + +sys.path.append(os.path.dirname(os.path.abspath(__file__)) + '/../automatedtesting_shared') + +from base import TestAutomationBase + +@pytest.mark.SUITE_sandbox +@pytest.mark.parametrize("project", ["AutomatedTesting"]) +@pytest.mark.parametrize("launcher_platform", ['windows_editor']) +class TestAutomation(TestAutomationBase): + def _run_prefab_test(self, request, workspace, editor, test_module, batch_mode=True, autotest_mode=True): + self._run_test(request, workspace, editor, test_module, + extra_cmdline_args=["--regset=/Amazon/Preferences/EnablePrefabSystem=true"], + batch_mode=batch_mode, + autotest_mode=autotest_mode) + + ## Seems to be flaky, need to investigate + def test_Multiplayer_AutoComponent_NetworkInput(self, request, workspace, editor, launcher_platform): + from .tests import Multiplayer_AutoComponent_NetworkInput as test_module + self._run_prefab_test(request, workspace, editor, test_module) + From bbeefe43b836a67ccd3eca0ca4775600c9684edd Mon Sep 17 00:00:00 2001 From: Sean Masterson Date: Thu, 4 Nov 2021 13:52:54 -0700 Subject: [PATCH 19/34] Added extra line above class Tests Signed-off-by: Sean Masterson --- .../Atom/tests/hydra_AtomEditorComponents_BloomAdded.py | 1 + 1 file changed, 1 insertion(+) diff --git a/AutomatedTesting/Gem/PythonTests/Atom/tests/hydra_AtomEditorComponents_BloomAdded.py b/AutomatedTesting/Gem/PythonTests/Atom/tests/hydra_AtomEditorComponents_BloomAdded.py index bce4b86a35..56b22eb20d 100644 --- a/AutomatedTesting/Gem/PythonTests/Atom/tests/hydra_AtomEditorComponents_BloomAdded.py +++ b/AutomatedTesting/Gem/PythonTests/Atom/tests/hydra_AtomEditorComponents_BloomAdded.py @@ -5,6 +5,7 @@ For complete copyright and license terms please see the LICENSE at the root of t SPDX-License-Identifier: Apache-2.0 OR MIT """ + class Tests: creation_undo = ( "UNDO Entity creation success", From 0c97c879c108073b58ae6e9276212b2796170ef6 Mon Sep 17 00:00:00 2001 From: jromnoa <80134229+jromnoa@users.noreply.github.com> Date: Thu, 4 Nov 2021 13:58:30 -0700 Subject: [PATCH 20/34] remove the main suite optimized jobs from CMakeLists.txt since its causing a false build failure on AR Signed-off-by: jromnoa <80134229+jromnoa@users.noreply.github.com> --- .../Gem/PythonTests/Atom/CMakeLists.txt | 13 ------------- 1 file changed, 13 deletions(-) diff --git a/AutomatedTesting/Gem/PythonTests/Atom/CMakeLists.txt b/AutomatedTesting/Gem/PythonTests/Atom/CMakeLists.txt index ff3cd5c465..87a66c880e 100644 --- a/AutomatedTesting/Gem/PythonTests/Atom/CMakeLists.txt +++ b/AutomatedTesting/Gem/PythonTests/Atom/CMakeLists.txt @@ -20,19 +20,6 @@ if(PAL_TRAIT_BUILD_HOST_TOOLS AND PAL_TRAIT_BUILD_TESTS_SUPPORTED) COMPONENT Atom ) - ly_add_pytest( - NAME AutomatedTesting::Atom_TestSuite_Main_Optimized - TEST_SUITE main - PATH ${CMAKE_CURRENT_LIST_DIR}/TestSuite_Main_Optimized.py - TEST_SERIAL - TIMEOUT 600 - RUNTIME_DEPENDENCIES - AssetProcessor - AutomatedTesting.Assets - Editor - COMPONENT - Atom - ) ly_add_pytest( NAME AutomatedTesting::Atom_TestSuite_Sandbox TEST_SUITE sandbox From 6bae0bca7e5a0947615a1814da4b1c397660e095 Mon Sep 17 00:00:00 2001 From: jromnoa <80134229+jromnoa@users.noreply.github.com> Date: Thu, 4 Nov 2021 15:08:02 -0700 Subject: [PATCH 21/34] add new bloom test to updated Main Suite test file since optimized main suite test file is removed Signed-off-by: jromnoa <80134229+jromnoa@users.noreply.github.com> --- AutomatedTesting/Gem/PythonTests/Atom/TestSuite_Main.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/AutomatedTesting/Gem/PythonTests/Atom/TestSuite_Main.py b/AutomatedTesting/Gem/PythonTests/Atom/TestSuite_Main.py index 62a402d9c4..cce9a27da6 100644 --- a/AutomatedTesting/Gem/PythonTests/Atom/TestSuite_Main.py +++ b/AutomatedTesting/Gem/PythonTests/Atom/TestSuite_Main.py @@ -13,6 +13,10 @@ from ly_test_tools.o3de.editor_test import EditorSharedTest, EditorTestSuite @pytest.mark.parametrize("launcher_platform", ['windows_editor']) class TestAutomation(EditorTestSuite): + @pytest.mark.test_case_id("C36525657") + class AtomEditorComponents_BloomAdded(EditorSharedTest): + from Atom.tests import hydra_AtomEditorComponents_BloomAdded as test_module + @pytest.mark.test_case_id("C32078118") class AtomEditorComponents_DecalAdded(EditorSharedTest): from Atom.tests import hydra_AtomEditorComponents_DecalAdded as test_module From 97e10d82107684a329c1a07cf3033f5de4259dd6 Mon Sep 17 00:00:00 2001 From: kberg-amzn Date: Thu, 4 Nov 2021 16:27:05 -0700 Subject: [PATCH 22/34] Fix mock and benchmark interfaces Signed-off-by: kberg-amzn --- Gems/Multiplayer/Code/Tests/CommonBenchmarkSetup.h | 3 +++ Gems/Multiplayer/Code/Tests/MockInterfaces.h | 3 +++ 2 files changed, 6 insertions(+) diff --git a/Gems/Multiplayer/Code/Tests/CommonBenchmarkSetup.h b/Gems/Multiplayer/Code/Tests/CommonBenchmarkSetup.h index 3c3d77e011..9ffdddfd20 100644 --- a/Gems/Multiplayer/Code/Tests/CommonBenchmarkSetup.h +++ b/Gems/Multiplayer/Code/Tests/CommonBenchmarkSetup.h @@ -270,6 +270,9 @@ namespace Multiplayer [[maybe_unused]] EntityIsMigrating entityIsMigrating) override {} void HandleLocalRpcMessage( [[maybe_unused]] NetworkEntityRpcMessage& message) override {} + void HandleEntitiesExitDomain(const NetEntityIdSet&) override {} + void ForceAssumeAuthority(const ConstNetworkEntityHandle&) override {} + void SetMigrateTimeoutTimeMs(AZ::TimeMs) override {} mutable AZStd::map m_networkEntityMap; diff --git a/Gems/Multiplayer/Code/Tests/MockInterfaces.h b/Gems/Multiplayer/Code/Tests/MockInterfaces.h index 8cebf280b9..f5207d94c8 100644 --- a/Gems/Multiplayer/Code/Tests/MockInterfaces.h +++ b/Gems/Multiplayer/Code/Tests/MockInterfaces.h @@ -87,6 +87,9 @@ namespace UnitTest MOCK_METHOD2(NotifyControllersActivated, void(const Multiplayer::ConstNetworkEntityHandle&, Multiplayer::EntityIsMigrating)); MOCK_METHOD2(NotifyControllersDeactivated, void(const Multiplayer::ConstNetworkEntityHandle&, Multiplayer::EntityIsMigrating)); MOCK_METHOD1(HandleLocalRpcMessage, void(Multiplayer::NetworkEntityRpcMessage&)); + MOCK_METHOD1(HandleEntitiesExitDomain, void(const Multiplayer::NetEntityIdSet&)); + MOCK_METHOD1(ForceAssumeAuthority, void(const Multiplayer::ConstNetworkEntityHandle&)); + MOCK_METHOD1(SetMigrateTimeoutTimeMs, void(AZ::TimeMs)); MOCK_CONST_METHOD0(DebugDraw, void()); }; From dedb367e9affd4e7bd3665955b0e38595fa7b680 Mon Sep 17 00:00:00 2001 From: kberg-amzn Date: Thu, 4 Nov 2021 16:29:41 -0700 Subject: [PATCH 23/34] Remove lots of mock interface code duplication Signed-off-by: kberg-amzn --- .../Code/Tests/CommonBenchmarkSetup.h | 66 +------------------ 1 file changed, 1 insertion(+), 65 deletions(-) diff --git a/Gems/Multiplayer/Code/Tests/CommonBenchmarkSetup.h b/Gems/Multiplayer/Code/Tests/CommonBenchmarkSetup.h index 9ffdddfd20..3e4a33290a 100644 --- a/Gems/Multiplayer/Code/Tests/CommonBenchmarkSetup.h +++ b/Gems/Multiplayer/Code/Tests/CommonBenchmarkSetup.h @@ -212,7 +212,7 @@ namespace Multiplayer } }; - class BenchmarkNetworkEntityManager : public Multiplayer::INetworkEntityManager + class BenchmarkNetworkEntityManager : public MockNetworkEntityManager { public: BenchmarkNetworkEntityManager() : m_authorityTracker(*this) {} @@ -221,58 +221,6 @@ namespace Multiplayer NetworkEntityAuthorityTracker* GetNetworkEntityAuthorityTracker() override { return &m_authorityTracker; } MultiplayerComponentRegistry* GetMultiplayerComponentRegistry() override { return &m_multiplayerComponentRegistry; } const HostId& GetHostId() const override { return m_hostId; } - EntityList CreateEntitiesImmediate( - [[maybe_unused]] const PrefabEntityId& prefabEntryId, - [[maybe_unused]] NetEntityRole netEntityRole, - [[maybe_unused]] const AZ::Transform& transform, - [[maybe_unused]] AutoActivate autoActivate) override { - return {}; - } - EntityList CreateEntitiesImmediate( - [[maybe_unused]] const PrefabEntityId& prefabEntryId, - [[maybe_unused]] NetEntityId netEntityId, - [[maybe_unused]] NetEntityRole netEntityRole, - [[maybe_unused]] AutoActivate autoActivate, - [[maybe_unused]] const AZ::Transform& transform) override { - return {}; - } - void SetupNetEntity( - [[maybe_unused]] AZ::Entity* netEntity, - [[maybe_unused]] PrefabEntityId prefabEntityId, - [[maybe_unused]] NetEntityRole netEntityRole) override {} - uint32_t GetEntityCount() const override { return {}; } - void MarkForRemoval( - [[maybe_unused]] const ConstNetworkEntityHandle& entityHandle) override {} - bool IsMarkedForRemoval( - [[maybe_unused]] const ConstNetworkEntityHandle& entityHandle) const override { - return {}; - } - void ClearEntityFromRemovalList( - [[maybe_unused]] const ConstNetworkEntityHandle& entityHandle) override {} - void ClearAllEntities() override {} - void AddEntityMarkedDirtyHandler( - [[maybe_unused]] AZ::Event<>::Handler& entityMarkedDirtyHandle) override {} - void AddEntityNotifyChangesHandler( - [[maybe_unused]] AZ::Event<>::Handler& entityNotifyChangesHandle) override {} - void AddEntityExitDomainHandler( - [[maybe_unused]] EntityExitDomainEvent::Handler& entityExitDomainHandler) override {} - void AddControllersActivatedHandler( - [[maybe_unused]] ControllersActivatedEvent::Handler& controllersActivatedHandler) override {} - void AddControllersDeactivatedHandler( - [[maybe_unused]] ControllersDeactivatedEvent::Handler& controllersDeactivatedHandler) override {} - void NotifyEntitiesDirtied() override {} - void NotifyEntitiesChanged() override {} - void NotifyControllersActivated( - [[maybe_unused]] const ConstNetworkEntityHandle& entityHandle, - [[maybe_unused]] EntityIsMigrating entityIsMigrating) override {} - void NotifyControllersDeactivated( - [[maybe_unused]] const ConstNetworkEntityHandle& entityHandle, - [[maybe_unused]] EntityIsMigrating entityIsMigrating) override {} - void HandleLocalRpcMessage( - [[maybe_unused]] NetworkEntityRpcMessage& message) override {} - void HandleEntitiesExitDomain(const NetEntityIdSet&) override {} - void ForceAssumeAuthority(const ConstNetworkEntityHandle&) override {} - void SetMigrateTimeoutTimeMs(AZ::TimeMs) override {} mutable AZStd::map m_networkEntityMap; @@ -301,18 +249,6 @@ namespace Multiplayer return InvalidNetEntityId; } - [[nodiscard]] AZStd::unique_ptr RequestNetSpawnableInstantiation( - [[maybe_unused]] const AZ::Data::Asset& netSpawnable, - [[maybe_unused]] const AZ::Transform& transform) override - { - return {}; - } - - void Initialize([[maybe_unused]] const HostId& hostId, [[maybe_unused]] AZStd::unique_ptr entityDomain) override {} - bool IsInitialized() const override { return true; } - IEntityDomain* GetEntityDomain() const override { return nullptr; } - void DebugDraw() const override {} - NetworkEntityTracker m_tracker; NetworkEntityAuthorityTracker m_authorityTracker; MultiplayerComponentRegistry m_multiplayerComponentRegistry; From ed06ef7ed24dcffede00e3c2e1ccfcb49b5e989b Mon Sep 17 00:00:00 2001 From: kberg-amzn Date: Thu, 4 Nov 2021 16:48:39 -0700 Subject: [PATCH 24/34] Removing ITimeoutHandler to simplify timeout queue interface, removes some unneeded code Signed-off-by: kberg-amzn --- .../DataStructures/TimeoutQueue.cpp | 6 ---- .../DataStructures/TimeoutQueue.h | 20 ------------- .../UdpTransport/UdpFragmentQueue.cpp | 16 +++++----- .../UdpTransport/UdpFragmentQueue.h | 6 ---- .../EntityReplicationManager.h | 2 -- .../EntityReplicationManager.cpp | 30 +++++++++---------- 6 files changed, 21 insertions(+), 59 deletions(-) diff --git a/Code/Framework/AzNetworking/AzNetworking/DataStructures/TimeoutQueue.cpp b/Code/Framework/AzNetworking/AzNetworking/DataStructures/TimeoutQueue.cpp index b0f316cf50..eb07efe80f 100644 --- a/Code/Framework/AzNetworking/AzNetworking/DataStructures/TimeoutQueue.cpp +++ b/Code/Framework/AzNetworking/AzNetworking/DataStructures/TimeoutQueue.cpp @@ -122,10 +122,4 @@ namespace AzNetworking m_timeoutItemMap.erase(itemTimeoutId); } } - - void TimeoutQueue::UpdateTimeouts(ITimeoutHandler& timeoutHandler, int32_t maxTimeouts) - { - TimeoutHandler handler([&timeoutHandler](TimeoutQueue::TimeoutItem& item) { return timeoutHandler.HandleTimeout(item); }); - UpdateTimeouts(handler, maxTimeouts); - } } diff --git a/Code/Framework/AzNetworking/AzNetworking/DataStructures/TimeoutQueue.h b/Code/Framework/AzNetworking/AzNetworking/DataStructures/TimeoutQueue.h index 63417ea36f..097e45c960 100644 --- a/Code/Framework/AzNetworking/AzNetworking/DataStructures/TimeoutQueue.h +++ b/Code/Framework/AzNetworking/AzNetworking/DataStructures/TimeoutQueue.h @@ -23,8 +23,6 @@ namespace AzNetworking Delete }; - class ITimeoutHandler; - //! @class TimeoutQueue //! @brief class for managing timeout items. class TimeoutQueue @@ -70,11 +68,6 @@ namespace AzNetworking using TimeoutHandler = AZStd::function; void UpdateTimeouts(const TimeoutHandler& timeoutHandler, int32_t maxTimeouts = -1); - //! Updates timeouts for all items, invokes timeout handlers if required. - //! @param timeoutHandler listener instance to call back on for timeouts - //! @param maxTimeouts the maximum number of timeouts to process before breaking iteration - void UpdateTimeouts(ITimeoutHandler& timeoutHandler, int32_t maxTimeouts = -1); - private: struct TimeoutQueueItem @@ -94,19 +87,6 @@ namespace AzNetworking TimeoutItemMap m_timeoutItemMap; TimeoutItemQueue m_timeoutItemQueue; }; - - //! @class ITimeoutHandler - //! @brief interface class for managing timeout items. - class ITimeoutHandler - { - public: - virtual ~ITimeoutHandler() = default; - - //! Handler callback for timed out items. - //! @param item containing registered timeout details - //! @return ETimeoutResult for whether to re-register or discard the timeout params - virtual TimeoutResult HandleTimeout(TimeoutQueue::TimeoutItem& item) = 0; - }; } #include diff --git a/Code/Framework/AzNetworking/AzNetworking/UdpTransport/UdpFragmentQueue.cpp b/Code/Framework/AzNetworking/AzNetworking/UdpTransport/UdpFragmentQueue.cpp index fa4ee78a92..0c710f5a14 100644 --- a/Code/Framework/AzNetworking/AzNetworking/UdpTransport/UdpFragmentQueue.cpp +++ b/Code/Framework/AzNetworking/AzNetworking/UdpTransport/UdpFragmentQueue.cpp @@ -20,7 +20,13 @@ namespace AzNetworking void UdpFragmentQueue::Update() { - m_timeoutQueue.UpdateTimeouts(*this); + m_timeoutQueue.UpdateTimeouts([this](TimeoutQueue::TimeoutItem& item) + { + const SequenceId fragmentSequence = static_cast(item.m_userData & 0xFF); + AZLOG(NET_FragmentQueue, "Timing out unreliable fragmented packet %u", static_cast(fragmentSequence)); + m_packetFragments.erase(fragmentSequence); + return TimeoutResult::Delete; + }); } void UdpFragmentQueue::Reset() @@ -163,12 +169,4 @@ namespace AzNetworking return handledPacket; } - - TimeoutResult UdpFragmentQueue::HandleTimeout(TimeoutQueue::TimeoutItem& item) - { - const SequenceId fragmentSequence = static_cast(item.m_userData & 0xFF); - AZLOG(NET_FragmentQueue, "Timing out unreliable fragmented packet %u", static_cast(fragmentSequence)); - m_packetFragments.erase(fragmentSequence); - return TimeoutResult::Delete; - } } diff --git a/Code/Framework/AzNetworking/AzNetworking/UdpTransport/UdpFragmentQueue.h b/Code/Framework/AzNetworking/AzNetworking/UdpTransport/UdpFragmentQueue.h index 9c929d63e8..5efa767283 100644 --- a/Code/Framework/AzNetworking/AzNetworking/UdpTransport/UdpFragmentQueue.h +++ b/Code/Framework/AzNetworking/AzNetworking/UdpTransport/UdpFragmentQueue.h @@ -26,7 +26,6 @@ namespace AzNetworking //! @class UdpFragmentQueue //! @brief Class for reconstructing packet chunks into the original unsegmented packet. class UdpFragmentQueue - : public ITimeoutHandler { public: @@ -51,11 +50,6 @@ namespace AzNetworking private: - //! Handler callback for timed out items. - //! @param item containing registered timeout details - //! @return ETimeoutResult for whether to re-register or discard the timeout params - TimeoutResult HandleTimeout(TimeoutQueue::TimeoutItem& item) override; - TimeoutQueue m_timeoutQueue; SequenceGenerator m_sequenceGenerator; diff --git a/Gems/Multiplayer/Code/Include/Multiplayer/NetworkEntity/EntityReplication/EntityReplicationManager.h b/Gems/Multiplayer/Code/Include/Multiplayer/NetworkEntity/EntityReplication/EntityReplicationManager.h index 10346ad777..74935d5746 100644 --- a/Gems/Multiplayer/Code/Include/Multiplayer/NetworkEntity/EntityReplication/EntityReplicationManager.h +++ b/Gems/Multiplayer/Code/Include/Multiplayer/NetworkEntity/EntityReplication/EntityReplicationManager.h @@ -150,7 +150,6 @@ namespace Multiplayer void ClearRemovedReplicators(); class OrphanedEntityRpcs - : public AzNetworking::ITimeoutHandler { public: OrphanedEntityRpcs(EntityReplicationManager& replicationManager); @@ -159,7 +158,6 @@ namespace Multiplayer void AddOrphanedRpc(NetEntityId entityId, NetworkEntityRpcMessage& entityRpcMessage); AZStd::size_t Size() const { return m_entityRpcMap.size(); } private: - AzNetworking::TimeoutResult HandleTimeout(AzNetworking::TimeoutQueue::TimeoutItem& item) override; struct OrphanedRpcs { OrphanedRpcs() = default; diff --git a/Gems/Multiplayer/Code/Source/NetworkEntity/EntityReplication/EntityReplicationManager.cpp b/Gems/Multiplayer/Code/Source/NetworkEntity/EntityReplication/EntityReplicationManager.cpp index fa099842c5..583671d1f3 100644 --- a/Gems/Multiplayer/Code/Source/NetworkEntity/EntityReplication/EntityReplicationManager.cpp +++ b/Gems/Multiplayer/Code/Source/NetworkEntity/EntityReplication/EntityReplicationManager.cpp @@ -912,24 +912,22 @@ namespace Multiplayer ; } - AzNetworking::TimeoutResult EntityReplicationManager::OrphanedEntityRpcs::HandleTimeout(AzNetworking::TimeoutQueue::TimeoutItem& item) - { - NetEntityId timedOutEntityId = aznumeric_cast(item.m_userData); - auto entityRpcsIter = m_entityRpcMap.find(timedOutEntityId); - if (entityRpcsIter != m_entityRpcMap.end()) - { - for (NetworkEntityRpcMessage& rpcMessage : entityRpcsIter->second.m_rpcMessages) - { - m_replicationManager.DispatchOrphanedRpc(rpcMessage, nullptr); - } - m_entityRpcMap.erase(entityRpcsIter); - } - return AzNetworking::TimeoutResult::Delete; - } - void EntityReplicationManager::OrphanedEntityRpcs::Update() { - m_timeoutQueue.UpdateTimeouts(*this); + m_timeoutQueue.UpdateTimeouts([this](AzNetworking::TimeoutQueue::TimeoutItem& item) + { + NetEntityId timedOutEntityId = aznumeric_cast(item.m_userData); + auto entityRpcsIter = m_entityRpcMap.find(timedOutEntityId); + if (entityRpcsIter != m_entityRpcMap.end()) + { + for (NetworkEntityRpcMessage& rpcMessage : entityRpcsIter->second.m_rpcMessages) + { + m_replicationManager.DispatchOrphanedRpc(rpcMessage, nullptr); + } + m_entityRpcMap.erase(entityRpcsIter); + } + return AzNetworking::TimeoutResult::Delete; + }); } bool EntityReplicationManager::OrphanedEntityRpcs::DispatchOrphanedRpcs(EntityReplicator& entityReplicator) From 708582731dc58f99fe71290863878600141ca432 Mon Sep 17 00:00:00 2001 From: Sean Masterson Date: Thu, 4 Nov 2021 16:56:46 -0700 Subject: [PATCH 25/34] Add P0 Deferred Fog Test Signed-off-by: Sean Masterson --- .../Gem/PythonTests/Atom/TestSuite_Main.py | 4 + .../Atom/atom_utils/atom_constants.py | 2 + ...a_AtomEditorComponents_DeferredFogAdded.py | 193 ++++++++++++++++++ 3 files changed, 199 insertions(+) create mode 100644 AutomatedTesting/Gem/PythonTests/Atom/tests/hydra_AtomEditorComponents_DeferredFogAdded.py diff --git a/AutomatedTesting/Gem/PythonTests/Atom/TestSuite_Main.py b/AutomatedTesting/Gem/PythonTests/Atom/TestSuite_Main.py index cce9a27da6..52e7ec993e 100644 --- a/AutomatedTesting/Gem/PythonTests/Atom/TestSuite_Main.py +++ b/AutomatedTesting/Gem/PythonTests/Atom/TestSuite_Main.py @@ -21,6 +21,10 @@ class TestAutomation(EditorTestSuite): class AtomEditorComponents_DecalAdded(EditorSharedTest): from Atom.tests import hydra_AtomEditorComponents_DecalAdded as test_module + @pytest.mark.test_case_id("C36525658") + class AtomEditorComponents_DeferredFogAdded(EditorSharedTest): + from Atom.tests import hydra_AtomEditorComponents_DeferredFogAdded as test_module + @pytest.mark.test_case_id("C32078119") class AtomEditorComponents_DepthOfFieldAdded(EditorSharedTest): from Atom.tests import hydra_AtomEditorComponents_DepthOfFieldAdded as test_module diff --git a/AutomatedTesting/Gem/PythonTests/Atom/atom_utils/atom_constants.py b/AutomatedTesting/Gem/PythonTests/Atom/atom_utils/atom_constants.py index 41ffaa0ce1..0c3410ff33 100644 --- a/AutomatedTesting/Gem/PythonTests/Atom/atom_utils/atom_constants.py +++ b/AutomatedTesting/Gem/PythonTests/Atom/atom_utils/atom_constants.py @@ -86,11 +86,13 @@ class AtomComponentProperties: - 'requires' a list of component names as strings required by this component. Use editor_entity_utils EditorEntity.add_components(list) to add this list of requirements.\n :param property: From the last element of the property tree path. Default 'name' for component name string. + - 'Enable Deferred Fog' Toggle active state of the component True/False :return: Full property path OR component name if no property specified. """ properties = { 'name': 'Deferred Fog', 'requires': [AtomComponentProperties.postfx_layer()], + 'Enable Deferred Fog': 'Controller|Configuration|Enable Deferred Fog', } return properties[property] diff --git a/AutomatedTesting/Gem/PythonTests/Atom/tests/hydra_AtomEditorComponents_DeferredFogAdded.py b/AutomatedTesting/Gem/PythonTests/Atom/tests/hydra_AtomEditorComponents_DeferredFogAdded.py new file mode 100644 index 0000000000..71163ece94 --- /dev/null +++ b/AutomatedTesting/Gem/PythonTests/Atom/tests/hydra_AtomEditorComponents_DeferredFogAdded.py @@ -0,0 +1,193 @@ +""" +Copyright (c) Contributors to the Open 3D Engine Project. +For complete copyright and license terms please see the LICENSE at the root of this distribution. + +SPDX-License-Identifier: Apache-2.0 OR MIT +""" + + +class Tests: + creation_undo = ( + "UNDO Entity creation success", + "UNDO Entity creation failed") + creation_redo = ( + "REDO Entity creation success", + "REDO Entity creation failed") + deferred_fog_creation = ( + "Deferred Fog Entity successfully created", + "Deferred Fog Entity failed to be created") + deferred_fog_component = ( + "Entity has a Deferred Fog component", + "Entity failed to find Deferred Fog component") + deferred_fog_disabled = ( + "Deferred Fog component disabled", + "Deferred Fog component was not disabled") + postfx_layer_component = ( + "Entity has a PostFX Layer component", + "Entity did not have an PostFX Layer component") + deferred_fog_enabled = ( + "Deferred Fog component enabled", + "Deferred Fog component was not enabled") + enable_deferred_fog_parameter_enabled = ( + "Enable Deferred Fog parameter enabled", + "Enable Deferred Fog parameter was not enabled") + enter_game_mode = ( + "Entered game mode", + "Failed to enter game mode") + exit_game_mode = ( + "Exited game mode", + "Couldn't exit game mode") + is_visible = ( + "Entity is visible", + "Entity was not visible") + is_hidden = ( + "Entity is hidden", + "Entity was not hidden") + entity_deleted = ( + "Entity deleted", + "Entity was not deleted") + deletion_undo = ( + "UNDO deletion success", + "UNDO deletion failed") + deletion_redo = ( + "REDO deletion success", + "REDO deletion failed") + + +def AtomEditorComponents_DeferredFog_AddedToEntity(): + """ + Summary: + Tests the Deferred Fog component can be added to an entity and has the expected functionality. + + Test setup: + - Wait for Editor idle loop. + - Open the "Base" level. + + Expected Behavior: + The component can be added, used in game mode, hidden/shown, deleted, and has accurate required components. + Creation and deletion undo/redo should also work. + + Test Steps: + 1) Create an Deferred Fog entity with no components. + 2) Add Deferred Fog component to Deferred Fog entity. + 3) UNDO the entity creation and component addition. + 4) REDO the entity creation and component addition. + 5) Verify Deferred Fog component not enabled. + 6) Add PostFX Layer component since it is required by the Deferred Fog component. + 7) Verify Deferred Fog component is enabled. + 8) Enable the "Enable Deferred Fog" parameter. + 9) Enter/Exit game mode. + 10) Test IsHidden. + 11) Test IsVisible. + 12) Delete Deferred Fog entity. + 13) UNDO deletion. + 14) REDO deletion. + 15) Look for errors. + + :return: None + """ + + import azlmbr.legacy.general as general + + from editor_python_test_tools.editor_entity_utils import EditorEntity + from editor_python_test_tools.utils import Report, Tracer, TestHelper + from Atom.atom_utils.atom_constants import AtomComponentProperties + + with Tracer() as error_tracer: + # Test setup begins. + # Setup: Wait for Editor idle loop before executing Python hydra scripts then open "Base" level. + TestHelper.init_idle() + TestHelper.open_level("", "Base") + + # Test steps begin. + # 1. Create an Deferred Fog entity with no components. + deferred_fog_entity = EditorEntity.create_editor_entity(AtomComponentProperties.deferred_fog()) + Report.critical_result(Tests.deferred_fog_creation, deferred_fog_entity.exists()) + + # 2. Add Deferred Fog component to Deferred Fog entity. + deferred_fog_component = deferred_fog_entity.add_component( + AtomComponentProperties.deferred_fog()) + Report.critical_result( + Tests.deferred_fog_component, + deferred_fog_entity.has_component(AtomComponentProperties.deferred_fog())) + + # 3. UNDO the entity creation and component addition. + # -> UNDO component addition. + general.undo() + # -> UNDO naming entity. + general.undo() + # -> UNDO selecting entity. + general.undo() + # -> UNDO entity creation. + general.undo() + general.idle_wait_frames(1) + Report.result(Tests.creation_undo, not deferred_fog_entity.exists()) + + # 4. REDO the entity creation and component addition. + # -> REDO entity creation. + general.redo() + # -> REDO selecting entity. + general.redo() + # -> REDO naming entity. + general.redo() + # -> REDO component addition. + general.redo() + general.idle_wait_frames(1) + Report.result(Tests.creation_redo, deferred_fog_entity.exists()) + + # 5. Verify Deferred Fog component not enabled. + Report.result(Tests.deferred_fog_disabled, not deferred_fog_component.is_enabled()) + + # 6. Add PostFX Layer component since it is required by the Deferred Fog component. + deferred_fog_entity.add_component(AtomComponentProperties.postfx_layer()) + Report.result( + Tests.postfx_layer_component, + deferred_fog_entity.has_component(AtomComponentProperties.postfx_layer())) + + # 7. Verify Deferred Fog component is enabled. + Report.result(Tests.deferred_fog_enabled, deferred_fog_component.is_enabled()) + + # 8. Enable the "Enable Deferred Fog" parameter. + deferred_fog_component.set_component_property_value( + AtomComponentProperties.deferred_fog('Enable Deferred Fog'), True) + Report.result(Tests.enable_deferred_fog_parameter_enabled, + deferred_fog_component.get_component_property_value( + AtomComponentProperties.deferred_fog('Enable Deferred Fog')) is True) + + # 9. Enter/Exit game mode. + TestHelper.enter_game_mode(Tests.enter_game_mode) + general.idle_wait_frames(1) + TestHelper.exit_game_mode(Tests.exit_game_mode) + + # 10. Test IsHidden. + deferred_fog_entity.set_visibility_state(False) + Report.result(Tests.is_hidden, deferred_fog_entity.is_hidden() is True) + + # 11. Test IsVisible. + deferred_fog_entity.set_visibility_state(True) + general.idle_wait_frames(1) + Report.result(Tests.is_visible, deferred_fog_entity.is_visible() is True) + + # 12. Delete Deferred Fog entity. + deferred_fog_entity.delete() + Report.result(Tests.entity_deleted, not deferred_fog_entity.exists()) + + # 13. UNDO deletion. + general.undo() + Report.result(Tests.deletion_undo, deferred_fog_entity.exists()) + + # 14. REDO deletion. + general.redo() + Report.result(Tests.deletion_redo, not deferred_fog_entity.exists()) + + # 15. Look for errors and asserts. + TestHelper.wait_for_condition(lambda: error_tracer.has_errors or error_tracer.has_asserts, 1.0) + for error_info in error_tracer.errors: + Report.info(f"Error: {error_info.filename} {error_info.function} | {error_info.message}") + for assert_info in error_tracer.asserts: + Report.info(f"Assert: {assert_info.filename} {assert_info.function} | {assert_info.message}") + + +if __name__ == "__main__": + from editor_python_test_tools.utils import Report + Report.start_test(AtomEditorComponents_DeferredFog_AddedToEntity) From 84d38a4f9cc613972513639fa7b5158738cc87e6 Mon Sep 17 00:00:00 2001 From: Gene Walters Date: Thu, 4 Nov 2021 22:23:00 -0700 Subject: [PATCH 26/34] Removing Multiplayer MainSuite tests, because AR considers no-test to be a failure Signed-off-by: Gene Walters --- .../Gem/PythonTests/Multiplayer/CMakeLists.txt | 13 ------------- 1 file changed, 13 deletions(-) diff --git a/AutomatedTesting/Gem/PythonTests/Multiplayer/CMakeLists.txt b/AutomatedTesting/Gem/PythonTests/Multiplayer/CMakeLists.txt index 506d15c323..367de4da9a 100644 --- a/AutomatedTesting/Gem/PythonTests/Multiplayer/CMakeLists.txt +++ b/AutomatedTesting/Gem/PythonTests/Multiplayer/CMakeLists.txt @@ -7,19 +7,6 @@ # if(PAL_TRAIT_BUILD_TESTS_SUPPORTED AND PAL_TRAIT_BUILD_HOST_TOOLS) - ly_add_pytest( - NAME AutomatedTesting::MultiplayerTests_Main - TEST_SUITE main - TEST_SERIAL - PATH ${CMAKE_CURRENT_LIST_DIR}/TestSuite_Main.py - RUNTIME_DEPENDENCIES - Legacy::Editor - AZ::AssetProcessor - AutomatedTesting.Assets - AutomatedTesting.ServerLauncher - COMPONENT - Multiplayer - ) ly_add_pytest( NAME AutomatedTesting::MultiplayerTests_Sandbox TEST_SUITE sandbox From c0ece7d32d8e7ba68e3efadcd47a2f3024a01c78 Mon Sep 17 00:00:00 2001 From: Sean Masterson Date: Fri, 5 Nov 2021 09:49:59 -0700 Subject: [PATCH 27/34] Update to atom_constants.py docstring Signed-off-by: Sean Masterson --- .../Gem/PythonTests/Atom/atom_utils/atom_constants.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/AutomatedTesting/Gem/PythonTests/Atom/atom_utils/atom_constants.py b/AutomatedTesting/Gem/PythonTests/Atom/atom_utils/atom_constants.py index 0c3410ff33..70156b9375 100644 --- a/AutomatedTesting/Gem/PythonTests/Atom/atom_utils/atom_constants.py +++ b/AutomatedTesting/Gem/PythonTests/Atom/atom_utils/atom_constants.py @@ -85,8 +85,8 @@ class AtomComponentProperties: Deferred Fog component properties. Requires PostFX Layer component. - 'requires' a list of component names as strings required by this component. Use editor_entity_utils EditorEntity.add_components(list) to add this list of requirements.\n + - 'Enable Deferred Fog' Toggle active state of the component True/False :param property: From the last element of the property tree path. Default 'name' for component name string. - - 'Enable Deferred Fog' Toggle active state of the component True/False :return: Full property path OR component name if no property specified. """ properties = { From 08115fc41fe8ae94933ac73c2fbb33a15eef1836 Mon Sep 17 00:00:00 2001 From: Shirang Jia Date: Fri, 5 Nov 2021 10:36:14 -0700 Subject: [PATCH 28/34] Add platform name to AP log path on S3 (#5316) * Add platform name to AP log path on S3 Signed-off-by: shiranj * Add platform name to AP log path on S3 Signed-off-by: shiranj --- scripts/build/Jenkins/Jenkinsfile | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/scripts/build/Jenkins/Jenkinsfile b/scripts/build/Jenkins/Jenkinsfile index ff04902861..6c3d697d2a 100644 --- a/scripts/build/Jenkins/Jenkinsfile +++ b/scripts/build/Jenkins/Jenkinsfile @@ -271,7 +271,7 @@ def CheckoutRepo(boolean disableSubmodules = false) { palRm('commitdate') } -def HandleDriveMount(String snapshot, String repositoryName, String projectName, String pipeline, String branchName, String platform, String buildType, String workspace, boolean recreateVolume = false) { +def HandleDriveMount(String snapshot, String repositoryName, String projectName, String pipeline, String branchName, String platform, String buildType, String workspace, boolean recreateVolume = false) { unstash name: 'incremental_build_script' def pythonCmd = '' @@ -435,7 +435,7 @@ def ExportTestScreenshots(Map options, String branchName, String platformName, S } } -def UploadAPLogs(Map options, String branchName, String jobName, String workspace, Map params) { +def UploadAPLogs(Map options, String branchName, String platformName, String jobName, String workspace, Map params) { dir("${workspace}/${ENGINE_REPOSITORY_NAME}") { projects = params.CMAKE_LY_PROJECTS.split(",") projects.each{ project -> @@ -449,7 +449,7 @@ def UploadAPLogs(Map options, String branchName, String jobName, String workspac } def command = "${pythonPath} -u ${s3UploadScriptPath} --base_dir ${apLogsPath} " + "--file_regex \".*\" --bucket ${env.AP_LOGS_S3_BUCKET} " + - "--search_subdirectories True --key_prefix ${env.JENKINS_JOB_NAME}/${branchName}/${env.BUILD_NUMBER}/${jobName} " + + "--search_subdirectories True --key_prefix ${env.JENKINS_JOB_NAME}/${branchName}/${env.BUILD_NUMBER}/${platformName}/${jobName} " + '--extra_args {\\"ACL\\":\\"bucket-owner-full-control\\"}' palSh(command, "Uploading AP logs for job ${jobName} for branch ${branchName}", false) } @@ -519,10 +519,10 @@ def CreateExportTestScreenshotsStage(Map pipelineConfig, String branchName, Stri } } -def CreateUploadAPLogsStage(Map pipelineConfig, String branchName, String jobName, String workspace, Map params) { +def CreateUploadAPLogsStage(Map pipelineConfig, String branchName, String platformName, String jobName, String workspace, Map params) { return { stage("${jobName}_upload_ap_logs") { - UploadAPLogs(pipelineConfig, branchName, jobName, workspace, params) + UploadAPLogs(pipelineConfig, branchName, platformName, jobName, workspace, params) } } } @@ -577,7 +577,7 @@ def CreateSingleNode(Map pipelineConfig, def platform, def build_job, Map envVar } } if (IsAPLogUpload(branchName, build_job_name)) { - CreateUploadAPLogsStage(pipelineConfig, branchName, build_job_name, envVars['WORKSPACE'], platform.value.build_types[build_job_name].PARAMETERS).call() + CreateUploadAPLogsStage(pipelineConfig, branchName, platform.key, build_job_name, envVars['WORKSPACE'], platform.value.build_types[build_job_name].PARAMETERS).call() } // All other errors will be raised outside the retry block currentResult = envVars['ON_FAILURE_MARK'] ?: 'FAILURE' From 988561920adeec83b4b3e6f597e8386807ec2ed8 Mon Sep 17 00:00:00 2001 From: kberg-amzn Date: Fri, 5 Nov 2021 12:26:10 -0700 Subject: [PATCH 29/34] Sets up the event scheduler system component for hierarchy tests Signed-off-by: kberg-amzn --- Gems/Multiplayer/Code/Tests/CommonHierarchySetup.h | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/Gems/Multiplayer/Code/Tests/CommonHierarchySetup.h b/Gems/Multiplayer/Code/Tests/CommonHierarchySetup.h index 249837b484..1b64efc5e2 100644 --- a/Gems/Multiplayer/Code/Tests/CommonHierarchySetup.h +++ b/Gems/Multiplayer/Code/Tests/CommonHierarchySetup.h @@ -10,6 +10,7 @@ #include #include #include +#include #include #include #include @@ -119,6 +120,8 @@ namespace Multiplayer m_mockTime = AZStd::make_unique>(); AZ::Interface::Register(m_mockTime.get()); + m_eventScheduler = AZStd::make_unique(); + m_mockNetworkTime = AZStd::make_unique>(); AZ::Interface::Register(m_mockNetworkTime.get()); @@ -170,6 +173,7 @@ namespace Multiplayer AZ::Interface::Unregister(m_mockMultiplayer.get()); AZ::Interface::Unregister(m_mockComponentApplicationRequests.get()); + m_eventScheduler.reset(); m_mockTime.reset(); m_mockNetworkEntityManager.reset(); @@ -204,6 +208,7 @@ namespace Multiplayer AZStd::unique_ptr> m_mockMultiplayer; AZStd::unique_ptr m_mockNetworkEntityManager; AZStd::unique_ptr> m_mockTime; + AZStd::unique_ptr m_eventScheduler; AZStd::unique_ptr> m_mockNetworkTime; AZStd::unique_ptr> m_mockConnection; From 989952e106cc0326c1566832b36094f23e9214bd Mon Sep 17 00:00:00 2001 From: kberg-amzn Date: Fri, 5 Nov 2021 12:28:06 -0700 Subject: [PATCH 30/34] Fix comment Signed-off-by: kberg-amzn --- .../Code/Include/Multiplayer/EntityDomains/IEntityDomain.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Gems/Multiplayer/Code/Include/Multiplayer/EntityDomains/IEntityDomain.h b/Gems/Multiplayer/Code/Include/Multiplayer/EntityDomains/IEntityDomain.h index 4b1bbbdba8..b650f9e1d9 100644 --- a/Gems/Multiplayer/Code/Include/Multiplayer/EntityDomains/IEntityDomain.h +++ b/Gems/Multiplayer/Code/Include/Multiplayer/EntityDomains/IEntityDomain.h @@ -34,7 +34,7 @@ namespace Multiplayer //! This method will be invoked whenever we unexpectedly lose the authoritative entity replicator for an entity. //! This gives our entity domain a chance to determine whether or not it should assume authority in this instance. - //! @param entityHandle the network entity handle of the entity that has lost it's authoritative replicator + //! @param entityHandle the network entity handle of the entity that has lost its authoritative replicator virtual void HandleLossOfAuthoritativeReplicator(const ConstNetworkEntityHandle& entityHandle) = 0; //! Debug draw to visualize host entity domains. From eecf6ab920ea3688e2430e7ac885380685610492 Mon Sep 17 00:00:00 2001 From: Esteban Papp <81431996+amznestebanpapp@users.noreply.github.com> Date: Fri, 5 Nov 2021 12:30:19 -0700 Subject: [PATCH 31/34] Create a nightly job that validates project-centric/engine-prebuilt (#5287) * adds a test_install_profile_vs2019_pipe job to validate a project can build from the SDK Signed-off-by: Esteban Papp <81431996+amznestebanpapp@users.noreply.github.com> * missed escaping these variables and breaks runtime dependencines in the install layout Signed-off-by: Esteban Papp <81431996+amznestebanpapp@users.noreply.github.com> * Changes to PIPELINE_ENV_OVERRIDE Signed-off-by: Esteban Papp <81431996+amznestebanpapp@users.noreply.github.com> * Tries to propagate ENV variables from pipeline jobs to jobs under it Signed-off-by: Esteban Papp <81431996+amznestebanpapp@users.noreply.github.com> * Fixes typo Signed-off-by: Esteban Papp <81431996+amznestebanpapp@users.noreply.github.com> * echoing a var to understand why is not going to the right path Signed-off-by: Esteban Papp <81431996+amznestebanpapp@users.noreply.github.com> * put the COMMAND_CWD in the wrong job Signed-off-by: Esteban Papp <81431996+amznestebanpapp@users.noreply.github.com> * adding similar jobs for linux/mac Signed-off-by: Esteban Papp <81431996+amznestebanpapp@users.noreply.github.com> * dont pass an empty LY_PROJECTS Signed-off-by: Esteban Papp <81431996+amznestebanpapp@users.noreply.github.com> * cmd -> sh, copy-paste mistake Signed-off-by: Esteban Papp <81431996+amznestebanpapp@users.noreply.github.com> * inverting check in linux/mac Signed-off-by: Esteban Papp <81431996+amznestebanpapp@users.noreply.github.com> * more fixes for linux/mac Signed-off-by: Esteban Papp <81431996+amznestebanpapp@users.noreply.github.com> * fixing script paths for macos Signed-off-by: Esteban Papp <81431996+amznestebanpapp@users.noreply.github.com> * more fixes for linux/mac Signed-off-by: Esteban Papp <81431996+amznestebanpapp@users.noreply.github.com> * Test use of %% instead of !! for windows builds Signed-off-by: Esteban Papp <81431996+amznestebanpapp@users.noreply.github.com> * fixes typo Signed-off-by: Esteban Papp <81431996+amznestebanpapp@users.noreply.github.com> --- scripts/build/Jenkins/Jenkinsfile | 4 +- .../build/Platform/Linux/build_config.json | 43 +++++++++++++++++ scripts/build/Platform/Linux/build_linux.sh | 5 +- scripts/build/Platform/Linux/env_linux.sh | 5 ++ scripts/build/Platform/Mac/build_config.json | 45 +++++++++++++++++ scripts/build/Platform/Mac/build_mac.sh | 5 +- scripts/build/Platform/Mac/env_mac.sh | 5 ++ .../build/Platform/Windows/build_config.json | 48 ++++++++++++------- .../build/Platform/Windows/build_windows.cmd | 19 ++++---- .../build/Platform/Windows/env_windows.cmd | 5 ++ 10 files changed, 157 insertions(+), 27 deletions(-) diff --git a/scripts/build/Jenkins/Jenkinsfile b/scripts/build/Jenkins/Jenkinsfile index 6c3d697d2a..8eddb61c33 100644 --- a/scripts/build/Jenkins/Jenkinsfile +++ b/scripts/build/Jenkins/Jenkinsfile @@ -551,9 +551,11 @@ def CreateSingleNode(Map pipelineConfig, def platform, def build_job, Map envVar CreateSetupStage(pipelineConfig, snapshot, repositoryName, projectName, pipelineName, branchName, platform.key, build_job.key, envVars, onlyMountEBSVolume).call() if(build_job.value.steps) { //this is a pipe with many steps so create all the build stages + pipelineEnvVars = GetBuildEnvVars(platform.value.PIPELINE_ENV ?: EMPTY_JSON, build_job.value.PIPELINE_ENV ?: EMPTY_JSON, pipelineName) build_job.value.steps.each { build_step -> build_job_name = build_step - envVars = GetBuildEnvVars(platform.value.PIPELINE_ENV ?: EMPTY_JSON, platform.value.build_types[build_step].PIPELINE_ENV ?: EMPTY_JSON, pipelineName) + // This addition of maps makes it that the right operand will override entries if they overlap with the left operand + envVars = pipelineEnvVars + GetBuildEnvVars(platform.value.PIPELINE_ENV ?: EMPTY_JSON, platform.value.build_types[build_step].PIPELINE_ENV ?: EMPTY_JSON, pipelineName) try { CreateBuildStage(pipelineConfig, platform.key, build_step, envVars).call() } diff --git a/scripts/build/Platform/Linux/build_config.json b/scripts/build/Platform/Linux/build_config.json index c0307de23d..8551c3dcb3 100644 --- a/scripts/build/Platform/Linux/build_config.json +++ b/scripts/build/Platform/Linux/build_config.json @@ -214,5 +214,48 @@ "CMAKE_LY_PROJECTS": "AutomatedTesting", "CMAKE_TARGET": "all" } + }, + "install_profile": { + "TAGS": [], + "COMMAND": "build_linux.sh", + "PARAMETERS": { + "CONFIGURATION": "profile", + "OUTPUT_DIRECTORY": "build/linux", + "CMAKE_OPTIONS": "-G 'Ninja Multi-Config' -DCMAKE_C_COMPILER=clang-6.0 -DCMAKE_CXX_COMPILER=clang++-6.0 -DLY_PARALLEL_LINK_JOBS=4 -DLY_DISABLE_TEST_MODULES=TRUE", + "CMAKE_TARGET": "install" + } + }, + "install_profile_pipe": { + "TAGS": [ + "nightly-incremental", + "nightly-clean" + ], + "PIPELINE_ENV": { + "PROJECT_REPOSITORY_NAME": "TestProject" + }, + "steps": [ + "install_profile", + "project_generate", + "project_engineinstall_profile" + ] + }, + "project_generate": { + "TAGS": [], + "COMMAND": "python_linux.sh", + "PARAMETERS": { + "SCRIPT_PATH": "install/scripts/o3de.py", + "SCRIPT_PARAMETERS": "create-project -pp ${WORKSPACE}/${PROJECT_REPOSITORY_NAME} --force" + } + }, + "project_engineinstall_profile": { + "TAGS": [], + "COMMAND": "build_linux.sh", + "PARAMETERS": { + "COMMAND_CWD": "${WORKSPACE}/${PROJECT_REPOSITORY_NAME}", + "CONFIGURATION": "profile", + "OUTPUT_DIRECTORY": "build/linux", + "CMAKE_OPTIONS": "-G 'Ninja Multi-Config' -DCMAKE_C_COMPILER=clang-6.0 -DCMAKE_CXX_COMPILER=clang++-6.0 -DLY_PARALLEL_LINK_JOBS=4 -DCMAKE_MODULE_PATH=${WORKSPACE}/o3de/install/cmake", + "CMAKE_TARGET": "all" + } } } diff --git a/scripts/build/Platform/Linux/build_linux.sh b/scripts/build/Platform/Linux/build_linux.sh index fd73e17a12..ab51913550 100755 --- a/scripts/build/Platform/Linux/build_linux.sh +++ b/scripts/build/Platform/Linux/build_linux.sh @@ -17,7 +17,10 @@ SOURCE_DIRECTORY=${PWD} pushd $OUTPUT_DIRECTORY LAST_CONFIGURE_CMD_FILE=ci_last_configure_cmd.txt -CONFIGURE_CMD="cmake ${SOURCE_DIRECTORY} ${CMAKE_OPTIONS} ${EXTRA_CMAKE_OPTIONS} -DLY_3RDPARTY_PATH=${LY_3RDPARTY_PATH} -DLY_PROJECTS='${CMAKE_LY_PROJECTS}'" +CONFIGURE_CMD="cmake ${SOURCE_DIRECTORY} ${CMAKE_OPTIONS} ${EXTRA_CMAKE_OPTIONS} -DLY_3RDPARTY_PATH=${LY_3RDPARTY_PATH}" +if [[ -n "$CMAKE_LY_PROJECTS" ]]; then + CONFIGURE_CMD="${CONFIGURE_CMD} -DLY_PROJECTS='${CMAKE_LY_PROJECTS}'" +fi if [[ ! -e "CMakeCache.txt" ]]; then echo [ci_build] First run, generating RUN_CONFIGURE=1 diff --git a/scripts/build/Platform/Linux/env_linux.sh b/scripts/build/Platform/Linux/env_linux.sh index a03b9642fb..059bb119ff 100755 --- a/scripts/build/Platform/Linux/env_linux.sh +++ b/scripts/build/Platform/Linux/env_linux.sh @@ -18,3 +18,8 @@ if ! command -v ninja &> /dev/null; then echo "[ci_build] Ninja not found" exit 1 fi + +if [[ -n "${COMMAND_CWD}" ]]; then + echo $(eval echo [ci_build] Changing CWD to $COMMAND_CWD) + cd $(eval echo ${COMMAND_CWD}) +fi diff --git a/scripts/build/Platform/Mac/build_config.json b/scripts/build/Platform/Mac/build_config.json index b57cc522bc..34b02a1aec 100644 --- a/scripts/build/Platform/Mac/build_config.json +++ b/scripts/build/Platform/Mac/build_config.json @@ -162,5 +162,50 @@ "SCRIPT_PATH": "scripts/build/package/package.py", "SCRIPT_PARAMETERS": "--platform Mac --type all" } + }, + "install_profile": { + "TAGS": [], + "COMMAND": "build_mac.sh", + "PARAMETERS": { + "CONFIGURATION": "profile", + "OUTPUT_DIRECTORY": "build/mac", + "CMAKE_OPTIONS": "-G Xcode -DLY_DISABLE_TEST_MODULES=TRUE", + "CMAKE_LY_PROJECTS": "", + "CMAKE_TARGET": "install" + } + }, + "install_profile_pipe": { + "TAGS": [ + "nightly-incremental", + "nightly-clean" + ], + "PIPELINE_ENV": { + "PROJECT_REPOSITORY_NAME": "TestProject" + }, + "steps": [ + "install_profile", + "project_generate", + "project_engineinstall_profile" + ] + }, + "project_generate": { + "TAGS": [], + "COMMAND": "python_mac.sh", + "PARAMETERS": { + "SCRIPT_PATH": "install/O3DE_SDK.app/Contents/Engine/scripts/o3de.py", + "SCRIPT_PARAMETERS": "create-project -pp ${WORKSPACE}/${PROJECT_REPOSITORY_NAME} --force" + } + }, + "project_engineinstall_profile": { + "TAGS": [], + "COMMAND": "build_mac.sh", + "PARAMETERS": { + "COMMAND_CWD": "${WORKSPACE}/${PROJECT_REPOSITORY_NAME}", + "CONFIGURATION": "profile", + "OUTPUT_DIRECTORY": "build/mac", + "CMAKE_OPTIONS": "-G Xcode -DCMAKE_MODULE_PATH=${WORKSPACE}/o3de/install/O3DE_SDK.app/Contents/Engine/cmake", + "CMAKE_LY_PROJECTS": "", + "CMAKE_TARGET": "ALL_BUILD" + } } } diff --git a/scripts/build/Platform/Mac/build_mac.sh b/scripts/build/Platform/Mac/build_mac.sh index cb271212d6..169e91c24f 100755 --- a/scripts/build/Platform/Mac/build_mac.sh +++ b/scripts/build/Platform/Mac/build_mac.sh @@ -17,7 +17,10 @@ SOURCE_DIRECTORY=${PWD} pushd $OUTPUT_DIRECTORY LAST_CONFIGURE_CMD_FILE=ci_last_configure_cmd.txt -CONFIGURE_CMD="cmake ${SOURCE_DIRECTORY} ${CMAKE_OPTIONS} ${EXTRA_CMAKE_OPTIONS} -DLY_3RDPARTY_PATH=${LY_3RDPARTY_PATH} -DLY_PROJECTS='${CMAKE_LY_PROJECTS}'" +CONFIGURE_CMD="cmake ${SOURCE_DIRECTORY} ${CMAKE_OPTIONS} ${EXTRA_CMAKE_OPTIONS} -DLY_3RDPARTY_PATH=${LY_3RDPARTY_PATH}" +if [[ -n "$CMAKE_LY_PROJECTS" ]]; then + CONFIGURE_CMD="${CONFIGURE_CMD} -DLY_PROJECTS='${CMAKE_LY_PROJECTS}'" +fi if [[ ! -e "CMakeCache.txt" ]]; then echo [ci_build] First run, generating RUN_CONFIGURE=1 diff --git a/scripts/build/Platform/Mac/env_mac.sh b/scripts/build/Platform/Mac/env_mac.sh index 2c974a1efe..f5fd9f5773 100755 --- a/scripts/build/Platform/Mac/env_mac.sh +++ b/scripts/build/Platform/Mac/env_mac.sh @@ -13,3 +13,8 @@ if ! command -v cmake &> /dev/null; then echo "[ci_build] CMake not found" exit 1 fi + +if [[ -n "${COMMAND_CWD}" ]]; then + echo $(eval echo [ci_build] Changing CWD to $COMMAND_CWD) + cd $(eval echo ${COMMAND_CWD}) +fi diff --git a/scripts/build/Platform/Windows/build_config.json b/scripts/build/Platform/Windows/build_config.json index b185a845ec..b0bb79b5dd 100644 --- a/scripts/build/Platform/Windows/build_config.json +++ b/scripts/build/Platform/Windows/build_config.json @@ -56,7 +56,7 @@ "COMMAND": "python_windows.cmd", "PARAMETERS": { "SCRIPT_PATH": "scripts/build/ci_build_metrics.py", - "SCRIPT_PARAMETERS": "--platform=Windows --repository=!REPOSITORY_NAME! --jobname=!JOB_NAME! --jobnumber=!BUILD_NUMBER! --jobnode=!NODE_LABEL! --changelist=!CHANGE_ID!" + "SCRIPT_PARAMETERS": "--platform=Windows --repository=%REPOSITORY_NAME% --jobname=%JOB_NAME% --jobnumber=%BUILD_NUMBER% --jobnode=%NODE_LABEL% --changelist=%CHANGE_ID%" } }, "windows_packaging_all": { @@ -88,7 +88,7 @@ "CONFIGURATION": "profile", "SCRIPT_PATH": "scripts/build/TestImpactAnalysis/tiaf_driver.py", "SCRIPT_PARAMETERS": - "--config=\"!OUTPUT_DIRECTORY!/bin/TestImpactFramework/profile/Persistent/tiaf.json\" --src-branch=!BRANCH_NAME! --dst-branch=!CHANGE_TARGET! --commit=!CHANGE_ID! --s3-bucket=!TEST_IMPACT_S3_BUCKET! --mars-index-prefix=jonawals --s3-top-level-dir=!REPOSITORY_NAME! --build-number=!BUILD_NUMBER! --suite=main --test-failure-policy=continue" + "--config=\"%OUTPUT_DIRECTORY%/bin/TestImpactFramework/profile/Persistent/tiaf.json\" --src-branch=%BRANCH_NAME% --dst-branch=%CHANGE_TARGET% --commit=%CHANGE_ID% --s3-bucket=%TEST_IMPACT_S3_BUCKET% --mars-index-prefix=jonawals --s3-top-level-dir=%REPOSITORY_NAME% --build-number=%BUILD_NUMBER% --suite=main --test-failure-policy=continue" } }, "debug_vs2019": { @@ -131,7 +131,7 @@ "PARAMETERS": { "CONFIGURATION": "profile", "OUTPUT_DIRECTORY": "build\\windows_vs2019", - "CMAKE_OPTIONS": "-G \"Visual Studio 16 2019\" -DCMAKE_SYSTEM_VERSION=10.0 -DLY_TEST_IMPACT_INSTRUMENTATION_BIN=!TEST_IMPACT_WIN_BINARY!", + "CMAKE_OPTIONS": "-G \"Visual Studio 16 2019\" -DCMAKE_SYSTEM_VERSION=10.0 -DLY_TEST_IMPACT_INSTRUMENTATION_BIN=%TEST_IMPACT_WIN_BINARY%", "CMAKE_LY_PROJECTS": "AutomatedTesting", "CMAKE_TARGET": "ALL_BUILD", "CMAKE_NATIVE_BUILD_ARGS": "/m /nologo" @@ -337,16 +337,12 @@ } }, "install_profile_vs2019": { - "TAGS": [ - "nightly-incremental", - "nightly-clean" - ], + "TAGS": [], "COMMAND": "build_windows.cmd", "PARAMETERS": { "CONFIGURATION": "profile", "OUTPUT_DIRECTORY": "build\\windows_vs2019", "CMAKE_OPTIONS": "-G \"Visual Studio 16 2019\" -DCMAKE_SYSTEM_VERSION=10.0 -DLY_DISABLE_TEST_MODULES=TRUE", - "CMAKE_LY_PROJECTS": "", "CMAKE_TARGET": "INSTALL", "CMAKE_NATIVE_BUILD_ARGS": "/m /nologo" } @@ -363,14 +359,35 @@ "PARAMETERS": { "CONFIGURATION": "profile", "OUTPUT_DIRECTORY": "build\\windows_vs2019", - "CMAKE_OPTIONS": "-G \"Visual Studio 16 2019\" -DCMAKE_SYSTEM_VERSION=10.0 -DLY_DISABLE_TEST_MODULES=TRUE -DLY_VERSION_ENGINE_NAME=o3de-sdk -DLY_INSTALLER_WIX_ROOT=\"!WIX! \"", - "EXTRA_CMAKE_OPTIONS": "-DLY_INSTALLER_AUTO_GEN_TAG=ON -DLY_INSTALLER_DOWNLOAD_URL=!INSTALLER_DOWNLOAD_URL! -DLY_INSTALLER_LICENSE_URL=!INSTALLER_DOWNLOAD_URL!/license", - "CPACK_BUCKET": "!INSTALLER_BUCKET!", - "CMAKE_LY_PROJECTS": "", + "CMAKE_OPTIONS": "-G \"Visual Studio 16 2019\" -DCMAKE_SYSTEM_VERSION=10.0 -DLY_DISABLE_TEST_MODULES=TRUE -DLY_VERSION_ENGINE_NAME=o3de-sdk -DLY_INSTALLER_WIX_ROOT=\"%WIX% \"", + "EXTRA_CMAKE_OPTIONS": "-DLY_INSTALLER_AUTO_GEN_TAG=ON -DLY_INSTALLER_DOWNLOAD_URL=%INSTALLER_DOWNLOAD_URL% -DLY_INSTALLER_LICENSE_URL=%INSTALLER_DOWNLOAD_URL%/license", + "CPACK_BUCKET": "%INSTALLER_BUCKET%", "CMAKE_TARGET": "ALL_BUILD", "CMAKE_NATIVE_BUILD_ARGS": "/m /nologo" } }, + "install_profile_vs2019_pipe": { + "TAGS": [ + "nightly-incremental", + "nightly-clean" + ], + "PIPELINE_ENV": { + "PROJECT_REPOSITORY_NAME": "TestProject" + }, + "steps": [ + "install_profile_vs2019", + "project_generate", + "project_engineinstall_profile_vs2019" + ] + }, + "project_generate": { + "TAGS": [], + "COMMAND": "python_windows.cmd", + "PARAMETERS": { + "SCRIPT_PATH": "install\\scripts\\o3de.py", + "SCRIPT_PARAMETERS": "create-project -pp %WORKSPACE%\\%PROJECT_REPOSITORY_NAME% --force" + } + }, "project_enginesource_profile_vs2019": { "TAGS": [ "project" @@ -382,8 +399,7 @@ "PARAMETERS": { "CONFIGURATION": "profile", "OUTPUT_DIRECTORY": "build\\windows_vs2019", - "CMAKE_OPTIONS": "-G \"Visual Studio 16 2019\" -DCMAKE_SYSTEM_VERSION=10.0 -DCMAKE_MODULE_PATH=!WORKSPACE!/o3de/cmake", - "CMAKE_LY_PROJECTS": "", + "CMAKE_OPTIONS": "-G \"Visual Studio 16 2019\" -DCMAKE_SYSTEM_VERSION=10.0 -DCMAKE_MODULE_PATH=%WORKSPACE%/o3de/cmake", "CMAKE_TARGET": "ALL_BUILD", "CMAKE_NATIVE_BUILD_ARGS": "/m /nologo" } @@ -395,10 +411,10 @@ }, "COMMAND": "build_windows.cmd", "PARAMETERS": { + "COMMAND_CWD": "%WORKSPACE%\\%PROJECT_REPOSITORY_NAME%", "CONFIGURATION": "profile", "OUTPUT_DIRECTORY": "build\\windows_vs2019", - "CMAKE_OPTIONS": "-G \"Visual Studio 16 2019\" -DCMAKE_SYSTEM_VERSION=10.0 -DCMAKE_MODULE_PATH=!WORKSPACE!/o3de/install/cmake", - "CMAKE_LY_PROJECTS": "", + "CMAKE_OPTIONS": "-G \"Visual Studio 16 2019\" -DCMAKE_SYSTEM_VERSION=10.0 -DCMAKE_MODULE_PATH=%WORKSPACE%/o3de/install/cmake", "CMAKE_TARGET": "ALL_BUILD", "CMAKE_NATIVE_BUILD_ARGS": "/m /nologo" } diff --git a/scripts/build/Platform/Windows/build_windows.cmd b/scripts/build/Platform/Windows/build_windows.cmd index b5a0245d1a..b9e862e04f 100644 --- a/scripts/build/Platform/Windows/build_windows.cmd +++ b/scripts/build/Platform/Windows/build_windows.cmd @@ -9,6 +9,13 @@ REM SETLOCAL EnableDelayedExpansion +REM Jenkins reports MSB8029 when TMP/TEMP is not defined, define a dummy folder +SET TMP=%cd%/temp +SET TEMP=%cd%/temp +IF NOT EXIST %TMP% ( + MKDIR temp +) + CALL %~dp0env_windows.cmd IF NOT EXIST "%OUTPUT_DIRECTORY%" ( @@ -25,18 +32,14 @@ IF ERRORLEVEL 1 ( exit /b 1 ) -REM Jenkins reports MSB8029 when TMP/TEMP is not defined, define a dummy folder -SET TMP=%cd%/temp -SET TEMP=%cd%/temp -IF NOT EXIST %TMP% ( - MKDIR temp -) - REM Compute half the amount of processors so some jobs can run SET /a HALF_PROCESSORS = NUMBER_OF_PROCESSORS / 2 SET LAST_CONFIGURE_CMD_FILE=ci_last_configure_cmd.txt -SET CONFIGURE_CMD=cmake %SOURCE_DIRECTORY% %CMAKE_OPTIONS% %EXTRA_CMAKE_OPTIONS% -DLY_3RDPARTY_PATH="%LY_3RDPARTY_PATH%" -DLY_PROJECTS=%CMAKE_LY_PROJECTS% +SET CONFIGURE_CMD=cmake %SOURCE_DIRECTORY% %CMAKE_OPTIONS% %EXTRA_CMAKE_OPTIONS% -DLY_3RDPARTY_PATH="%LY_3RDPARTY_PATH%" +IF NOT "%CMAKE_LY_PROJECTS%"=="" ( + SET CONFIGURE_CMD=!CONFIGURE_CMD! -DLY_PROJECTS="%CMAKE_LY_PROJECTS%" +) IF NOT EXIST CMakeCache.txt ( ECHO [ci_build] First run, generating SET RUN_CONFIGURE=1 diff --git a/scripts/build/Platform/Windows/env_windows.cmd b/scripts/build/Platform/Windows/env_windows.cmd index 1c54e36bfc..f11d394519 100644 --- a/scripts/build/Platform/Windows/env_windows.cmd +++ b/scripts/build/Platform/Windows/env_windows.cmd @@ -13,6 +13,11 @@ IF NOT %ERRORLEVEL%==0 ( GOTO :error ) +IF NOT "%COMMAND_CWD%"=="" ( + ECHO [ci_build] Changing CWD to %COMMAND_CWD% + CD %COMMAND_CWD% +) + EXIT /b 0 :error From 2206d2d8f10c5a2b7b9e18425c52e42af86e40e6 Mon Sep 17 00:00:00 2001 From: Brian Herrera Date: Fri, 5 Nov 2021 12:57:03 -0700 Subject: [PATCH 32/34] Add restricted folder to gitignore Signed-off-by: brianherrera --- .gitignore | 1 + 1 file changed, 1 insertion(+) diff --git a/.gitignore b/.gitignore index 3a9b8f6f8e..5f6172cd76 100644 --- a/.gitignore +++ b/.gitignore @@ -12,6 +12,7 @@ Editor/EditorEventLog.xml Editor/EditorLayout.xml **/*egg-info/** **/*egg-link +**/[Rr]estricted UserSettings.xml [Uu]ser/ FrameCapture/** From 46f4935ee44693b17e884237810d0428bb4825a6 Mon Sep 17 00:00:00 2001 From: Mike Chang Date: Fri, 5 Nov 2021 14:04:05 -0700 Subject: [PATCH 33/34] Update to licenses/notices script (#5214) This updates the licenses script to pull in all PackageInfo.json files with a specific argument, then follows each license file defined and writes the contents to a file. In this mode, if a packageinfo file is found, it will only grab the license file path defined within. Also has the following features: * Generalizes the function and variable names for non-license specific references * Sorts os.walk to maintain consistent ordering * Uses an ordered dict for the output, also to maintain ordering if using Python below 3.7 * Adds an additional json config file to have specific exclusion rules for 3p packages * Adds a package creation function and config file entry * Allow multipath scans, optional use of gitignore, merged license file scan Signed-off-by: Mike Chang --- scripts/license_scanner/license_scanner.py | 140 ++++++++++++++------ scripts/license_scanner/scanner_config.json | 3 + 2 files changed, 103 insertions(+), 40 deletions(-) diff --git a/scripts/license_scanner/license_scanner.py b/scripts/license_scanner/license_scanner.py index c0e3c1f1ba..2936e343f9 100644 --- a/scripts/license_scanner/license_scanner.py +++ b/scripts/license_scanner/license_scanner.py @@ -6,6 +6,7 @@ # import argparse +from collections import OrderedDict import fnmatch import json import os @@ -24,15 +25,19 @@ class LicenseScanner: """ DEFAULT_CONFIG_FILE = 'scanner_config.json' + DEFAULT_EXCLUDE_FILE = '.gitignore' + DEFAULT_PACKAGE_INFO_FILE = 'PackageInfo.json' def __init__(self, config_file=None): self.config_file = config_file self.config_data = self._load_config() - self.license_regex = self._load_license_regex() + self.file_regex = self._load_file_regex(self.config_data['license_patterns']) + self.package_info = self._load_file_regex(self.config_data['package_patterns']) + self.excluded_directories = self._load_file_regex(self.config_data['excluded_directories']) def _load_config(self): """Load config from the provided file. Sets default file if one is not provided.""" - if self.config_file is None: + if not self.config_file: script_directory = os.path.dirname(os.path.abspath(__file__)) # Default file expected in same dir as script self.config_file = os.path.join(script_directory, self.DEFAULT_CONFIG_FILE) @@ -43,45 +48,68 @@ class LicenseScanner: print('Config file cannot be found') raise - def _load_license_regex(self): + def _load_file_regex(self, patterns): """Returns regex object with case-insensitive matching from the list of filename patterns.""" regex_patterns = [] - for pattern in self.config_data['license_patterns']: + for pattern in patterns: regex_patterns.append(fnmatch.translate(pattern)) + + if not regex_patterns: + print(f'Warning: No patterns from {patterns} found') + return None + return re.compile('|'.join(regex_patterns), re.IGNORECASE) - def scan(self, path=os.curdir): - """Scan directory tree for filenames matching license_regex. + def scan(self, paths=os.curdir): + """Scan directory tree for filenames matching file_regex, package info, and exclusion files. - :param path: Path of the directory to run scanner - :return: Package paths and their corresponding license file contents - :rtype: dict + :param paths: Paths of the directory to run scanner + :return: Package paths and their corresponding file contents + :rtype: Ordered dict """ - licenses = 0 - license_files = {} + files = 0 + matching_files = OrderedDict() + excluded_directories = None - for dirpath, dirnames, filenames in os.walk(path): - for file in filenames: - if self.license_regex.match(file): - license_file_content = self._get_license_file_contents(os.path.join(dirpath, file)) - rel_dirpath = os.path.relpath(dirpath, path) # Limit path inside scanned directory - license_files[rel_dirpath] = license_file_content - licenses += 1 - print(f'License file: {os.path.join(dirpath, file)}') + if not self.package_info: + self.package_info = self.DEFAULT_PACKAGE_INFO_FILE - # Remove directories that should not be scanned - for dir in self.config_data['excluded_directories']: - if dir in dirnames: - dirnames.remove(dir) - print(f'{licenses} license files found.') - return license_files + if not self.excluded_directories: + print(f'No excluded directory in config, looking for {self.DEFAULT_EXCLUDE_FILE} instead') - def _get_license_file_contents(self, filepath): + for path in paths: + for dirpath, dirnames, filenames in os.walk(path, topdown=True): + dirnames.sort(key=str.casefold) # Ensure that results are sorted + for file in filenames: + if self.file_regex.match(file) or self.package_info.match(file): + file_path = os.path.join(dirpath, file) + matching_file_content = self._get_file_contents(file_path) + matching_files[file_path] = matching_file_content + files += 1 + print(f'Matching file: {file_path}') + if self.package_info.match(file): + dirnames[:] = [] # Stop scanning subdirectories if package info file found + if self.DEFAULT_EXCLUDE_FILE in file and not self.excluded_directories: + ignore_list = self._get_file_contents(os.path.join(dirpath, file)).splitlines() + ignore_list.append('.git') # .gitignore doesn't usually have .git in its exclusions + excluded_directories = self._load_file_regex(ignore_list) + + # Remove directories that should not be scanned + if self.excluded_directories: + excluded_directories = self.excluded_directories + for dir in dirnames: + if excluded_directories.match(dir): + dirnames.remove(dir) + + print(f'{files} files found.') + return matching_files + + def _get_file_contents(self, filepath): try: with open(filepath, encoding='utf8') as f: return f.read() except UnicodeDecodeError: - print(f'Unable to read license file: {filepath}') + print(f'Unable to read file: {filepath}') pass def create_license_file(self, licenses, filepath='NOTICES.txt'): @@ -89,18 +117,44 @@ class LicenseScanner: :param licenses: Dict with package paths and their corresponding license file contents :param filepath: Path to write the file - """ - package_separator = '------------------------------------' - with open(filepath, 'w', encoding='utf8') as f: + """ + license_separator = '------------------------------------' + with open(filepath, 'w', encoding='utf8') as lf: for directory, license in licenses.items(): - license_output = '\n\n'.join([ - f'{package_separator}', - f'Package path: {directory}', - 'License:', - f'{license}\n' - ]) - f.write(license_output) + if not self.package_info.match(os.path.basename(directory)): + license_output = '\n\n'.join([ + f'{license_separator}', + f'Package path: {os.path.relpath(directory)}', + 'License:', + f'{license}\n' + ]) + lf.write(license_output) return None + + def create_package_file(self, packages, filepath='SPDX-Licenses.json', get_contents=False): + """Creates file with all the provided SPDX package info summaries in json. + Optional dirpath parameter will follow the license file path in the package info and return its contents in a dictionary + + :param licenses: Dict with package info paths and their corresponding file contents + :param filepath: Path to write the file + :param dirpath: Root path for packages + :rtype: Ordered dict + """ + licenses = OrderedDict() + package_json = [] + + with open(filepath, 'w', encoding='utf8') as pf: + for directory, package in packages.items(): + if self.package_info.match(os.path.basename(directory)): + package_obj = json.loads(package) + package_json.append(package_obj) + if get_contents: + license_path = os.path.join(os.path.dirname(directory), pathlib.Path(package_obj['LicenseFile'])) + licenses[license_path] = self._get_file_contents(license_path) + else: + licenses[directory] = package + pf.write(json.dumps(package_json, indent=4)) + return licenses def parse_args(): @@ -108,7 +162,8 @@ def parse_args(): description='Script to run LicenseScanner and generate license file') parser.add_argument('--config-file', '-c', type=pathlib.Path, help='Config file for LicenseScanner') parser.add_argument('--license-file-path', '-l', type=pathlib.Path, help='Create license file in the provided path') - parser.add_argument('--scan-path', '-s', default=os.curdir, type=pathlib.Path, help='Path to scan') + parser.add_argument('--package-file-path', '-p', type=pathlib.Path, help='Create package summary file in the provided path') + parser.add_argument('--scan-path', '-s', default=os.curdir, type=pathlib.Path, nargs='+', help='Path to scan, multiple space separated paths can be used') return parser.parse_args() @@ -116,10 +171,15 @@ def main(): try: args = parse_args() ls = LicenseScanner(args.config_file) - licenses = ls.scan(args.scan_path) + scanned_path_data = ls.scan(args.scan_path) if args.license_file_path: - ls.create_license_file(licenses, args.license_file_path) + ls.create_license_file(scanned_path_data, args.license_file_path) + if args.package_file_path: + ls.create_package_file(scanned_path_data, args.package_file_path) + if args.license_file_path and args.package_file_path: + license_files = ls.create_package_file(scanned_path_data, args.package_file_path, True) + ls.create_license_file(license_files, args.license_file_path) except FileNotFoundError as e: print(f'Type: {type(e).__name__}, Error: {e}') return 1 diff --git a/scripts/license_scanner/scanner_config.json b/scripts/license_scanner/scanner_config.json index b5863a7d31..2e8f5206db 100644 --- a/scripts/license_scanner/scanner_config.json +++ b/scripts/license_scanner/scanner_config.json @@ -8,5 +8,8 @@ "license_patterns": [ "LICENSE*", "COPYING*" + ], + "package_patterns": [ + "PackageInfo.json" ] } From 6b6eb2c93638b498185b8f9b2bcc51d0f4366494 Mon Sep 17 00:00:00 2001 From: Jeremy Ong Date: Fri, 5 Nov 2021 15:34:35 -0600 Subject: [PATCH 34/34] Remove markers that occupy <2us for 99% of events Signed-off-by: Jeremy Ong --- .../AzCore/AzCore/Jobs/Internal/JobManagerWorkStealing.cpp | 2 -- Gems/Atom/RHI/Code/Source/RHI/FrameScheduler.cpp | 3 --- Gems/Atom/RPI/Code/Source/RPI.Public/Pass/RasterPass.cpp | 2 -- 3 files changed, 7 deletions(-) diff --git a/Code/Framework/AzCore/AzCore/Jobs/Internal/JobManagerWorkStealing.cpp b/Code/Framework/AzCore/AzCore/Jobs/Internal/JobManagerWorkStealing.cpp index 230bf959f6..c05590ca91 100644 --- a/Code/Framework/AzCore/AzCore/Jobs/Internal/JobManagerWorkStealing.cpp +++ b/Code/Framework/AzCore/AzCore/Jobs/Internal/JobManagerWorkStealing.cpp @@ -457,8 +457,6 @@ void JobManagerWorkStealing::ProcessJobsInternal(ThreadInfo* info, Job* suspende else { //attempt to steal a job from another thread's queue - AZ_PROFILE_SCOPE(AzCore, "JobManagerWorkStealing::ProcessJobsInternal:WorkStealing"); - unsigned int numStealAttempts = 0; const unsigned int maxStealAttempts = (unsigned int)m_workerThreads.size() * 3; //try every thread a few times before giving up while (!job) diff --git a/Gems/Atom/RHI/Code/Source/RHI/FrameScheduler.cpp b/Gems/Atom/RHI/Code/Source/RHI/FrameScheduler.cpp index 11ae78c69a..eb4fb46cc9 100644 --- a/Gems/Atom/RHI/Code/Source/RHI/FrameScheduler.cpp +++ b/Gems/Atom/RHI/Code/Source/RHI/FrameScheduler.cpp @@ -152,8 +152,6 @@ namespace AZ ResultCode FrameScheduler::ImportScopeProducer(ScopeProducer& scopeProducer) { - AZ_PROFILE_SCOPE(RHI, "FrameScheduler: ImportScopeProducer"); - if (!ValidateIsProcessing()) { return RHI::ResultCode::InvalidOperation; @@ -266,7 +264,6 @@ namespace AZ // Execute all queued resource invalidations, which will mark SRG's for compilation. { - AZ_PROFILE_SCOPE(RHI, "Invalidate Resources"); ResourceInvalidateBus::ExecuteQueuedEvents(); } diff --git a/Gems/Atom/RPI/Code/Source/RPI.Public/Pass/RasterPass.cpp b/Gems/Atom/RPI/Code/Source/RPI.Public/Pass/RasterPass.cpp index d9f98c11d3..5001951377 100644 --- a/Gems/Atom/RPI/Code/Source/RPI.Public/Pass/RasterPass.cpp +++ b/Gems/Atom/RPI/Code/Source/RPI.Public/Pass/RasterPass.cpp @@ -216,8 +216,6 @@ namespace AZ void RasterPass::CompileResources(const RHI::FrameGraphCompileContext& context) { - AZ_PROFILE_SCOPE(RPI, "RasterPass: CompileResources"); - if (m_shaderResourceGroup == nullptr) { return;