[development] automated tests for timing editor loading of the 10k levels (#7472)
It has been observed that when entering game mode in the editor with the 10k levels, the Vulkan backend performs significantly worse compared to DX12 on Windows. These tests are intended to detect if either backend gets worse with a currently generous timeout of 3 minutes to complete (10 for the Vulkan outlier) and should lowered over time as this metric stabilizes. Added a new periodic tests job to specifically run on a GPU instance. Signed-off-by: AMZN-ScottR <24445312+AMZN-ScottR@users.noreply.github.com>monroegm-disable-blank-issue-2
Scott Romero
4 years ago
committed by
GitHub
parent
4bf6b6ba15
commit
aaa8b9df7c
@ -0,0 +1,39 @@
|
||||
#
|
||||
# Copyright (c) Contributors to the Open 3D Engine Project.
|
||||
# For complete copyright and license terms please see the LICENSE at the root of this distribution.
|
||||
#
|
||||
# SPDX-License-Identifier: Apache-2.0 OR MIT
|
||||
#
|
||||
#
|
||||
|
||||
if(NOT PAL_TRAIT_BUILD_TESTS_SUPPORTED OR NOT PAL_TRAIT_BUILD_HOST_TOOLS)
|
||||
return()
|
||||
endif()
|
||||
|
||||
ly_add_pytest(
|
||||
NAME AutomatedTesting::EditorLevelLoadingPerfTests_DX12
|
||||
TEST_SUITE periodic
|
||||
TEST_REQUIRES gpu
|
||||
TEST_SERIAL
|
||||
PATH ${CMAKE_CURRENT_LIST_DIR}/TestSuite_Periodic_DX12.py
|
||||
RUNTIME_DEPENDENCIES
|
||||
Legacy::Editor
|
||||
AZ::AssetProcessor
|
||||
AutomatedTesting.Assets
|
||||
COMPONENT
|
||||
Performance
|
||||
)
|
||||
|
||||
ly_add_pytest(
|
||||
NAME AutomatedTesting::EditorLevelLoadingPerfTests_Vulkan
|
||||
TEST_SUITE periodic
|
||||
TEST_REQUIRES gpu
|
||||
TEST_SERIAL
|
||||
PATH ${CMAKE_CURRENT_LIST_DIR}/TestSuite_Periodic_Vulkan.py
|
||||
RUNTIME_DEPENDENCIES
|
||||
Legacy::Editor
|
||||
AZ::AssetProcessor
|
||||
AutomatedTesting.Assets
|
||||
COMPONENT
|
||||
Performance
|
||||
)
|
||||
@ -0,0 +1,31 @@
|
||||
"""
|
||||
Copyright (c) Contributors to the Open 3D Engine Project.
|
||||
For complete copyright and license terms please see the LICENSE at the root of this distribution.
|
||||
|
||||
SPDX-License-Identifier: Apache-2.0 OR MIT
|
||||
|
||||
"""
|
||||
|
||||
# This suite consists of all test cases that are passing and have been verified.
|
||||
|
||||
import pytest
|
||||
import os
|
||||
import sys
|
||||
|
||||
from ly_test_tools.o3de.editor_test import EditorTestSuite, EditorSingleTest
|
||||
|
||||
@pytest.mark.parametrize("project", ["AutomatedTesting"])
|
||||
@pytest.mark.parametrize("launcher_platform", ['windows_editor'])
|
||||
class TestAutomation(EditorTestSuite):
|
||||
|
||||
class Time_EditorLevelLoading_10KEntityCpuPerfTest(EditorSingleTest):
|
||||
extra_cmdline_args = ['-rhi=dx12']
|
||||
use_null_renderer = False # needs renderer to validate test
|
||||
|
||||
from .tests import EditorLevelLoading_10KEntityCpuPerfTest as test_module
|
||||
|
||||
class Time_EditorLevelLoading_10kVegInstancesTest(EditorSingleTest):
|
||||
extra_cmdline_args = ['-rhi=dx12']
|
||||
use_null_renderer = False # needs renderer to validate test
|
||||
|
||||
from .tests import EditorLevelLoading_10kVegInstancesTest as test_module
|
||||
@ -0,0 +1,34 @@
|
||||
"""
|
||||
Copyright (c) Contributors to the Open 3D Engine Project.
|
||||
For complete copyright and license terms please see the LICENSE at the root of this distribution.
|
||||
|
||||
SPDX-License-Identifier: Apache-2.0 OR MIT
|
||||
|
||||
"""
|
||||
|
||||
# This suite consists of all test cases that are passing and have been verified.
|
||||
|
||||
import pytest
|
||||
import os
|
||||
import sys
|
||||
|
||||
from ly_test_tools.o3de.editor_test import EditorTestSuite, EditorSingleTest
|
||||
|
||||
@pytest.mark.parametrize("project", ["AutomatedTesting"])
|
||||
@pytest.mark.parametrize("launcher_platform", ['windows_editor'])
|
||||
class TestAutomation(EditorTestSuite):
|
||||
|
||||
class Time_EditorLevelLoading_10KEntityCpuPerfTest(EditorSingleTest):
|
||||
# there is currently a huge discrepancy loading this level with vulkan compared to dx12 which requires the 10 min timeout
|
||||
# this should be removed once that issue has been sorted out
|
||||
timeout = 600
|
||||
extra_cmdline_args = ['-rhi=vulkan']
|
||||
use_null_renderer = False # needs renderer to validate test
|
||||
|
||||
from .tests import EditorLevelLoading_10KEntityCpuPerfTest as test_module
|
||||
|
||||
class Time_EditorLevelLoading_10kVegInstancesTest(EditorSingleTest):
|
||||
extra_cmdline_args = ['-rhi=vulkan']
|
||||
use_null_renderer = False # needs renderer to validate test
|
||||
|
||||
from .tests import EditorLevelLoading_10kVegInstancesTest as test_module
|
||||
@ -0,0 +1,6 @@
|
||||
"""
|
||||
Copyright (c) Contributors to the Open 3D Engine Project.
|
||||
For complete copyright and license terms please see the LICENSE at the root of this distribution.
|
||||
|
||||
SPDX-License-Identifier: Apache-2.0 OR MIT
|
||||
"""
|
||||
@ -0,0 +1,15 @@
|
||||
"""
|
||||
Copyright (c) Contributors to the Open 3D Engine Project.
|
||||
For complete copyright and license terms please see the LICENSE at the root of this distribution.
|
||||
|
||||
SPDX-License-Identifier: Apache-2.0 OR MIT
|
||||
"""
|
||||
|
||||
from Performance.utils.perf_timer import time_editor_level_loading
|
||||
|
||||
def EditorLevelLoading_10KEntityCpuPerfTest():
|
||||
time_editor_level_loading('Performance', '10KEntityCpuPerfTest')
|
||||
|
||||
if __name__ == "__main__":
|
||||
from editor_python_test_tools.utils import Report
|
||||
Report.start_test(EditorLevelLoading_10KEntityCpuPerfTest)
|
||||
@ -0,0 +1,15 @@
|
||||
"""
|
||||
Copyright (c) Contributors to the Open 3D Engine Project.
|
||||
For complete copyright and license terms please see the LICENSE at the root of this distribution.
|
||||
|
||||
SPDX-License-Identifier: Apache-2.0 OR MIT
|
||||
"""
|
||||
|
||||
from Performance.utils.perf_timer import time_editor_level_loading
|
||||
|
||||
def EditorLevelLoading_10kVegInstancesTest():
|
||||
time_editor_level_loading('Performance', '10kVegInstancesTest')
|
||||
|
||||
if __name__ == "__main__":
|
||||
from editor_python_test_tools.utils import Report
|
||||
Report.start_test(EditorLevelLoading_10kVegInstancesTest)
|
||||
@ -0,0 +1,6 @@
|
||||
"""
|
||||
Copyright (c) Contributors to the Open 3D Engine Project.
|
||||
For complete copyright and license terms please see the LICENSE at the root of this distribution.
|
||||
|
||||
SPDX-License-Identifier: Apache-2.0 OR MIT
|
||||
"""
|
||||
@ -0,0 +1,6 @@
|
||||
"""
|
||||
Copyright (c) Contributors to the Open 3D Engine Project.
|
||||
For complete copyright and license terms please see the LICENSE at the root of this distribution.
|
||||
|
||||
SPDX-License-Identifier: Apache-2.0 OR MIT
|
||||
"""
|
||||
@ -0,0 +1,72 @@
|
||||
"""
|
||||
Copyright (c) Contributors to the Open 3D Engine Project.
|
||||
For complete copyright and license terms please see the LICENSE at the root of this distribution.
|
||||
|
||||
SPDX-License-Identifier: Apache-2.0 OR MIT
|
||||
"""
|
||||
|
||||
import time
|
||||
|
||||
ENTER_MSG = ("Entered game mode", "Failed to enter game mode")
|
||||
EXIT_MSG = ("Exited game mode", "Couldn't exit game mode")
|
||||
|
||||
class Timer:
|
||||
unit_divisor = 60
|
||||
hour_divisor = unit_divisor * unit_divisor
|
||||
|
||||
def start(self):
|
||||
self._start_time = time.perf_counter()
|
||||
|
||||
def log_time(self, message):
|
||||
from editor_python_test_tools.utils import Report
|
||||
|
||||
elapsed_time = time.perf_counter() - self._start_time
|
||||
hours = int(elapsed_time / Timer.hour_divisor)
|
||||
minutes = int(elapsed_time % Timer.hour_divisor / Timer.unit_divisor)
|
||||
seconds = elapsed_time % Timer.unit_divisor
|
||||
|
||||
Report.info(f'{message}: {hours:0>2d}:{minutes:0>2d}:{seconds:0>5.2f}\n')
|
||||
|
||||
def time_editor_level_loading(level_dir, level_name):
|
||||
|
||||
"""
|
||||
Summary:
|
||||
Time how long it takes to load an arbitrary level, entering game mode, and exiting game mode
|
||||
|
||||
Level Description:
|
||||
Preferably a level with a large number of entities
|
||||
|
||||
Expected Behavior:
|
||||
Level loads within a reasonable time frame e.i. doesn't trip the framework timeout
|
||||
|
||||
Benchmark Steps:
|
||||
1) Time opening the level
|
||||
2) Time entering game mode
|
||||
3) Time exiting game mode
|
||||
4) Close the editor
|
||||
|
||||
:return: None
|
||||
"""
|
||||
from editor_python_test_tools.utils import TestHelper as helper
|
||||
|
||||
timer = Timer()
|
||||
|
||||
helper.init_idle()
|
||||
|
||||
# 1) Open level
|
||||
timer.start()
|
||||
helper.open_level(level_dir, level_name)
|
||||
timer.log_time('Level load time')
|
||||
|
||||
# 2) Time how long it takes to enter game mode
|
||||
timer.start()
|
||||
helper.enter_game_mode(ENTER_MSG)
|
||||
timer.log_time('Enter game mode')
|
||||
|
||||
# 3) Exit game mode
|
||||
timer.start()
|
||||
helper.exit_game_mode(EXIT_MSG)
|
||||
timer.log_time('Exit game mode')
|
||||
|
||||
# 4) Close the editor
|
||||
helper.close_editor()
|
||||
Loading…
Reference in New Issue