[development] automated tests for timing editor loading of the 10k levels (#7472)

It has been observed that when entering game mode in the editor with the 10k levels, the Vulkan backend performs significantly worse compared to DX12 on Windows. These tests are intended to detect if either backend gets worse with a currently generous timeout of 3 minutes to complete (10 for the Vulkan outlier) and should lowered over time as this metric stabilizes. Added a new periodic tests job to specifically run on a GPU instance.

Signed-off-by: AMZN-ScottR <24445312+AMZN-ScottR@users.noreply.github.com>
This commit is contained in:
Scott Romero
2022-02-15 11:17:15 -08:00
committed by GitHub
parent 4bf6b6ba15
commit aaa8b9df7c
11 changed files with 250 additions and 1 deletions
@@ -239,7 +239,29 @@
"CMAKE_LY_PROJECTS": "AutomatedTesting",
"CMAKE_TARGET": "TEST_SUITE_periodic",
"CMAKE_NATIVE_BUILD_ARGS": "/m /nologo",
"CTEST_OPTIONS": "-L \"(SUITE_periodic)\" -T Test --no-tests=error",
"CTEST_OPTIONS": "-L \"(SUITE_periodic)\" -LE \"(REQUIRES_gpu)\" -T Test --no-tests=error",
"TEST_METRICS": "True",
"TEST_RESULTS": "True"
}
},
"periodic_test_gpu_profile": {
"TAGS": [
"nightly-incremental",
"nightly-clean",
"weekly-build-metrics"
],
"PIPELINE_ENV":{
"NODE_LABEL":"windows-gpu"
},
"COMMAND": "build_test_windows.cmd",
"PARAMETERS": {
"CONFIGURATION": "profile",
"OUTPUT_DIRECTORY": "build\\windows",
"CMAKE_OPTIONS": "-G \"Visual Studio 16 2019\" -DCMAKE_SYSTEM_VERSION=10.0",
"CMAKE_LY_PROJECTS": "AutomatedTesting",
"CMAKE_TARGET": "TEST_SUITE_periodic",
"CMAKE_NATIVE_BUILD_ARGS": "/m /nologo",
"CTEST_OPTIONS": "-L \"(SUITE_periodic_REQUIRES_gpu)\" -T Test --no-tests=error",
"TEST_METRICS": "True",
"TEST_RESULTS": "True"
}