diff --git a/AutomatedTesting/Gem/PythonTests/CMakeLists.txt b/AutomatedTesting/Gem/PythonTests/CMakeLists.txt index 26c03e78c2..421d095be7 100644 --- a/AutomatedTesting/Gem/PythonTests/CMakeLists.txt +++ b/AutomatedTesting/Gem/PythonTests/CMakeLists.txt @@ -67,3 +67,6 @@ add_subdirectory(Multiplayer) ## Integration tests for editor testing framework ## add_subdirectory(editor_test_testing) + +## Performance ## +add_subdirectory(Performance) diff --git a/AutomatedTesting/Gem/PythonTests/Performance/CMakeLists.txt b/AutomatedTesting/Gem/PythonTests/Performance/CMakeLists.txt new file mode 100644 index 0000000000..c2ec14db9d --- /dev/null +++ b/AutomatedTesting/Gem/PythonTests/Performance/CMakeLists.txt @@ -0,0 +1,39 @@ +# +# Copyright (c) Contributors to the Open 3D Engine Project. +# For complete copyright and license terms please see the LICENSE at the root of this distribution. +# +# SPDX-License-Identifier: Apache-2.0 OR MIT +# +# + +if(NOT PAL_TRAIT_BUILD_TESTS_SUPPORTED OR NOT PAL_TRAIT_BUILD_HOST_TOOLS) + return() +endif() + +ly_add_pytest( + NAME AutomatedTesting::EditorLevelLoadingPerfTests_DX12 + TEST_SUITE periodic + TEST_REQUIRES gpu + TEST_SERIAL + PATH ${CMAKE_CURRENT_LIST_DIR}/TestSuite_Periodic_DX12.py + RUNTIME_DEPENDENCIES + Legacy::Editor + AZ::AssetProcessor + AutomatedTesting.Assets + COMPONENT + Performance +) + +ly_add_pytest( + NAME AutomatedTesting::EditorLevelLoadingPerfTests_Vulkan + TEST_SUITE periodic + TEST_REQUIRES gpu + TEST_SERIAL + PATH ${CMAKE_CURRENT_LIST_DIR}/TestSuite_Periodic_Vulkan.py + RUNTIME_DEPENDENCIES + Legacy::Editor + AZ::AssetProcessor + AutomatedTesting.Assets + COMPONENT + Performance +) diff --git a/AutomatedTesting/Gem/PythonTests/Performance/TestSuite_Periodic_DX12.py b/AutomatedTesting/Gem/PythonTests/Performance/TestSuite_Periodic_DX12.py new file mode 100644 index 0000000000..ac3152a090 --- /dev/null +++ b/AutomatedTesting/Gem/PythonTests/Performance/TestSuite_Periodic_DX12.py @@ -0,0 +1,31 @@ +""" +Copyright (c) Contributors to the Open 3D Engine Project. +For complete copyright and license terms please see the LICENSE at the root of this distribution. + +SPDX-License-Identifier: Apache-2.0 OR MIT + +""" + +# This suite consists of all test cases that are passing and have been verified. + +import pytest +import os +import sys + +from ly_test_tools.o3de.editor_test import EditorTestSuite, EditorSingleTest + +@pytest.mark.parametrize("project", ["AutomatedTesting"]) +@pytest.mark.parametrize("launcher_platform", ['windows_editor']) +class TestAutomation(EditorTestSuite): + + class Time_EditorLevelLoading_10KEntityCpuPerfTest(EditorSingleTest): + extra_cmdline_args = ['-rhi=dx12'] + use_null_renderer = False # needs renderer to validate test + + from .tests import EditorLevelLoading_10KEntityCpuPerfTest as test_module + + class Time_EditorLevelLoading_10kVegInstancesTest(EditorSingleTest): + extra_cmdline_args = ['-rhi=dx12'] + use_null_renderer = False # needs renderer to validate test + + from .tests import EditorLevelLoading_10kVegInstancesTest as test_module diff --git a/AutomatedTesting/Gem/PythonTests/Performance/TestSuite_Periodic_Vulkan.py b/AutomatedTesting/Gem/PythonTests/Performance/TestSuite_Periodic_Vulkan.py new file mode 100644 index 0000000000..c71ede27dc --- /dev/null +++ b/AutomatedTesting/Gem/PythonTests/Performance/TestSuite_Periodic_Vulkan.py @@ -0,0 +1,34 @@ +""" +Copyright (c) Contributors to the Open 3D Engine Project. +For complete copyright and license terms please see the LICENSE at the root of this distribution. + +SPDX-License-Identifier: Apache-2.0 OR MIT + +""" + +# This suite consists of all test cases that are passing and have been verified. + +import pytest +import os +import sys + +from ly_test_tools.o3de.editor_test import EditorTestSuite, EditorSingleTest + +@pytest.mark.parametrize("project", ["AutomatedTesting"]) +@pytest.mark.parametrize("launcher_platform", ['windows_editor']) +class TestAutomation(EditorTestSuite): + + class Time_EditorLevelLoading_10KEntityCpuPerfTest(EditorSingleTest): + # there is currently a huge discrepancy loading this level with vulkan compared to dx12 which requires the 10 min timeout + # this should be removed once that issue has been sorted out + timeout = 600 + extra_cmdline_args = ['-rhi=vulkan'] + use_null_renderer = False # needs renderer to validate test + + from .tests import EditorLevelLoading_10KEntityCpuPerfTest as test_module + + class Time_EditorLevelLoading_10kVegInstancesTest(EditorSingleTest): + extra_cmdline_args = ['-rhi=vulkan'] + use_null_renderer = False # needs renderer to validate test + + from .tests import EditorLevelLoading_10kVegInstancesTest as test_module diff --git a/AutomatedTesting/Gem/PythonTests/Performance/__init__.py b/AutomatedTesting/Gem/PythonTests/Performance/__init__.py new file mode 100644 index 0000000000..f5193b300e --- /dev/null +++ b/AutomatedTesting/Gem/PythonTests/Performance/__init__.py @@ -0,0 +1,6 @@ +""" +Copyright (c) Contributors to the Open 3D Engine Project. +For complete copyright and license terms please see the LICENSE at the root of this distribution. + +SPDX-License-Identifier: Apache-2.0 OR MIT +""" diff --git a/AutomatedTesting/Gem/PythonTests/Performance/tests/EditorLevelLoading_10KEntityCpuPerfTest.py b/AutomatedTesting/Gem/PythonTests/Performance/tests/EditorLevelLoading_10KEntityCpuPerfTest.py new file mode 100644 index 0000000000..fb7707044a --- /dev/null +++ b/AutomatedTesting/Gem/PythonTests/Performance/tests/EditorLevelLoading_10KEntityCpuPerfTest.py @@ -0,0 +1,15 @@ +""" +Copyright (c) Contributors to the Open 3D Engine Project. +For complete copyright and license terms please see the LICENSE at the root of this distribution. + +SPDX-License-Identifier: Apache-2.0 OR MIT +""" + +from Performance.utils.perf_timer import time_editor_level_loading + +def EditorLevelLoading_10KEntityCpuPerfTest(): + time_editor_level_loading('Performance', '10KEntityCpuPerfTest') + +if __name__ == "__main__": + from editor_python_test_tools.utils import Report + Report.start_test(EditorLevelLoading_10KEntityCpuPerfTest) diff --git a/AutomatedTesting/Gem/PythonTests/Performance/tests/EditorLevelLoading_10kVegInstancesTest.py b/AutomatedTesting/Gem/PythonTests/Performance/tests/EditorLevelLoading_10kVegInstancesTest.py new file mode 100644 index 0000000000..1c71cc806e --- /dev/null +++ b/AutomatedTesting/Gem/PythonTests/Performance/tests/EditorLevelLoading_10kVegInstancesTest.py @@ -0,0 +1,15 @@ +""" +Copyright (c) Contributors to the Open 3D Engine Project. +For complete copyright and license terms please see the LICENSE at the root of this distribution. + +SPDX-License-Identifier: Apache-2.0 OR MIT +""" + +from Performance.utils.perf_timer import time_editor_level_loading + +def EditorLevelLoading_10kVegInstancesTest(): + time_editor_level_loading('Performance', '10kVegInstancesTest') + +if __name__ == "__main__": + from editor_python_test_tools.utils import Report + Report.start_test(EditorLevelLoading_10kVegInstancesTest) diff --git a/AutomatedTesting/Gem/PythonTests/Performance/tests/__init__.py b/AutomatedTesting/Gem/PythonTests/Performance/tests/__init__.py new file mode 100644 index 0000000000..f5193b300e --- /dev/null +++ b/AutomatedTesting/Gem/PythonTests/Performance/tests/__init__.py @@ -0,0 +1,6 @@ +""" +Copyright (c) Contributors to the Open 3D Engine Project. +For complete copyright and license terms please see the LICENSE at the root of this distribution. + +SPDX-License-Identifier: Apache-2.0 OR MIT +""" diff --git a/AutomatedTesting/Gem/PythonTests/Performance/utils/__init__.py b/AutomatedTesting/Gem/PythonTests/Performance/utils/__init__.py new file mode 100644 index 0000000000..f5193b300e --- /dev/null +++ b/AutomatedTesting/Gem/PythonTests/Performance/utils/__init__.py @@ -0,0 +1,6 @@ +""" +Copyright (c) Contributors to the Open 3D Engine Project. +For complete copyright and license terms please see the LICENSE at the root of this distribution. + +SPDX-License-Identifier: Apache-2.0 OR MIT +""" diff --git a/AutomatedTesting/Gem/PythonTests/Performance/utils/perf_timer.py b/AutomatedTesting/Gem/PythonTests/Performance/utils/perf_timer.py new file mode 100644 index 0000000000..54497e96b2 --- /dev/null +++ b/AutomatedTesting/Gem/PythonTests/Performance/utils/perf_timer.py @@ -0,0 +1,72 @@ +""" +Copyright (c) Contributors to the Open 3D Engine Project. +For complete copyright and license terms please see the LICENSE at the root of this distribution. + +SPDX-License-Identifier: Apache-2.0 OR MIT +""" + +import time + +ENTER_MSG = ("Entered game mode", "Failed to enter game mode") +EXIT_MSG = ("Exited game mode", "Couldn't exit game mode") + +class Timer: + unit_divisor = 60 + hour_divisor = unit_divisor * unit_divisor + + def start(self): + self._start_time = time.perf_counter() + + def log_time(self, message): + from editor_python_test_tools.utils import Report + + elapsed_time = time.perf_counter() - self._start_time + hours = int(elapsed_time / Timer.hour_divisor) + minutes = int(elapsed_time % Timer.hour_divisor / Timer.unit_divisor) + seconds = elapsed_time % Timer.unit_divisor + + Report.info(f'{message}: {hours:0>2d}:{minutes:0>2d}:{seconds:0>5.2f}\n') + +def time_editor_level_loading(level_dir, level_name): + + """ + Summary: + Time how long it takes to load an arbitrary level, entering game mode, and exiting game mode + + Level Description: + Preferably a level with a large number of entities + + Expected Behavior: + Level loads within a reasonable time frame e.i. doesn't trip the framework timeout + + Benchmark Steps: + 1) Time opening the level + 2) Time entering game mode + 3) Time exiting game mode + 4) Close the editor + + :return: None + """ + from editor_python_test_tools.utils import TestHelper as helper + + timer = Timer() + + helper.init_idle() + + # 1) Open level + timer.start() + helper.open_level(level_dir, level_name) + timer.log_time('Level load time') + + # 2) Time how long it takes to enter game mode + timer.start() + helper.enter_game_mode(ENTER_MSG) + timer.log_time('Enter game mode') + + # 3) Exit game mode + timer.start() + helper.exit_game_mode(EXIT_MSG) + timer.log_time('Exit game mode') + + # 4) Close the editor + helper.close_editor() diff --git a/scripts/build/Platform/Windows/build_config.json b/scripts/build/Platform/Windows/build_config.json index 017f56bb68..f951221d8e 100644 --- a/scripts/build/Platform/Windows/build_config.json +++ b/scripts/build/Platform/Windows/build_config.json @@ -239,7 +239,29 @@ "CMAKE_LY_PROJECTS": "AutomatedTesting", "CMAKE_TARGET": "TEST_SUITE_periodic", "CMAKE_NATIVE_BUILD_ARGS": "/m /nologo", - "CTEST_OPTIONS": "-L \"(SUITE_periodic)\" -T Test --no-tests=error", + "CTEST_OPTIONS": "-L \"(SUITE_periodic)\" -LE \"(REQUIRES_gpu)\" -T Test --no-tests=error", + "TEST_METRICS": "True", + "TEST_RESULTS": "True" + } + }, + "periodic_test_gpu_profile": { + "TAGS": [ + "nightly-incremental", + "nightly-clean", + "weekly-build-metrics" + ], + "PIPELINE_ENV":{ + "NODE_LABEL":"windows-gpu" + }, + "COMMAND": "build_test_windows.cmd", + "PARAMETERS": { + "CONFIGURATION": "profile", + "OUTPUT_DIRECTORY": "build\\windows", + "CMAKE_OPTIONS": "-G \"Visual Studio 16 2019\" -DCMAKE_SYSTEM_VERSION=10.0", + "CMAKE_LY_PROJECTS": "AutomatedTesting", + "CMAKE_TARGET": "TEST_SUITE_periodic", + "CMAKE_NATIVE_BUILD_ARGS": "/m /nologo", + "CTEST_OPTIONS": "-L \"(SUITE_periodic_REQUIRES_gpu)\" -T Test --no-tests=error", "TEST_METRICS": "True", "TEST_RESULTS": "True" }