From fee95f3f5eb6c7d62ea97158a74f071f9e80a7de Mon Sep 17 00:00:00 2001 From: jromnoa <80134229+jromnoa@users.noreply.github.com> Date: Wed, 9 Feb 2022 08:10:33 -0800 Subject: [PATCH] sandboxing the sponza level from the level test because it causes a hardlock 4%-12% of the time (#7513) Signed-off-by: jromnoa <80134229+jromnoa@users.noreply.github.com> --- .../Gem/PythonTests/Atom/TestSuite_Sandbox.py | 7 +- .../Atom/atom_utils/atom_constants.py | 4 +- .../tests/hydra_Atom_LevelLoadTest_Sandbox.py | 71 +++++++++++++++++++ 3 files changed, 80 insertions(+), 2 deletions(-) create mode 100644 AutomatedTesting/Gem/PythonTests/Atom/tests/hydra_Atom_LevelLoadTest_Sandbox.py diff --git a/AutomatedTesting/Gem/PythonTests/Atom/TestSuite_Sandbox.py b/AutomatedTesting/Gem/PythonTests/Atom/TestSuite_Sandbox.py index 5299e55a2b..54e3d8cb41 100644 --- a/AutomatedTesting/Gem/PythonTests/Atom/TestSuite_Sandbox.py +++ b/AutomatedTesting/Gem/PythonTests/Atom/TestSuite_Sandbox.py @@ -20,9 +20,14 @@ TEST_DIRECTORY = os.path.join(os.path.dirname(__file__), "tests") @pytest.mark.parametrize("launcher_platform", ['windows_editor']) class TestAutomation(EditorTestSuite): - enable_prefab_system = False + enable_prefab_system = True # this test is intermittently timing out without ever having executed. sandboxing while we investigate cause. @pytest.mark.test_case_id("C36525660") class AtomEditorComponents_DisplayMapperAdded(EditorSharedTest): from Atom.tests import hydra_AtomEditorComponents_DisplayMapperAdded as test_module + + # The "Sponza" level is failing with a hard lock 4-12% of the time, needs root causing and fixing. + @pytest.mark.test_case_id("C36529679") + class AtomLevelLoadTest_Editor_Sandbox(EditorSharedTest): + from Atom.tests import hydra_Atom_LevelLoadTest_Sandbox as test_module diff --git a/AutomatedTesting/Gem/PythonTests/Atom/atom_utils/atom_constants.py b/AutomatedTesting/Gem/PythonTests/Atom/atom_utils/atom_constants.py index 6525536f96..778f9bd3fb 100644 --- a/AutomatedTesting/Gem/PythonTests/Atom/atom_utils/atom_constants.py +++ b/AutomatedTesting/Gem/PythonTests/Atom/atom_utils/atom_constants.py @@ -33,7 +33,9 @@ GLOBAL_ILLUMINATION_QUALITY = { } # Level list used in Editor Level Load Test -LEVEL_LIST = ["hermanubis", "hermanubis_high", "macbeth_shaderballs", "PbrMaterialChart", "ShadowTest", "Sponza"] +# WARNING: "Sponza" level is sandboxed due to an intermittent failure. +LEVEL_LIST = ["hermanubis", "hermanubis_high", "macbeth_shaderballs", "PbrMaterialChart", "ShadowTest"] + class AtomComponentProperties: """ diff --git a/AutomatedTesting/Gem/PythonTests/Atom/tests/hydra_Atom_LevelLoadTest_Sandbox.py b/AutomatedTesting/Gem/PythonTests/Atom/tests/hydra_Atom_LevelLoadTest_Sandbox.py new file mode 100644 index 0000000000..6cee7b2840 --- /dev/null +++ b/AutomatedTesting/Gem/PythonTests/Atom/tests/hydra_Atom_LevelLoadTest_Sandbox.py @@ -0,0 +1,71 @@ +""" +Copyright (c) Contributors to the Open 3D Engine Project. +For complete copyright and license terms please see the LICENSE at the root of this distribution. + +SPDX-License-Identifier: Apache-2.0 OR MIT +""" + + +def Atom_LevelLoadTest(): + """ + Summary: + Loads all graphics levels within the AutomatedTesting project in editor. For each level this script will verify that + the level loads, and can enter/exit gameplay without crashing the editor. + + Test setup: + - Store all available levels in a list. + - Set up a for loop to run all checks for each level. + + Expected Behavior: + Test verifies that each level loads, enters/exits game mode, and reports success for all test actions. + + Test Steps for each level: + 1) Create tuple with level load success and failure messages + 2) Open the level using the python test tools command + 3) Verify level is loaded using a separate command, and report success/failure + 4) Enter gameplay and report result using a tuple + 5) Exit Gameplay and report result using a tuple + 6) Look for errors or asserts. + + :return: None + """ + SANDBOX_LEVEL_LIST = ["Sponza"] + + import azlmbr.legacy.general as general + + from editor_python_test_tools.utils import Report, Tracer, TestHelper + + with Tracer() as error_tracer: + + for level in SANDBOX_LEVEL_LIST: + + # 1. Create tuple with level load success and failure messages + level_check_tuple = (f"loaded {level}", f"failed to load {level}") + + # 2. Open the level using the python test tools command + TestHelper.init_idle() + TestHelper.open_level("Graphics", level) + + # 3. Verify level is loaded using a separate command, and report success/failure + Report.result(level_check_tuple, level == general.get_current_level_name()) + + # 4. Enter gameplay and report result using a tuple + enter_game_mode_tuple = (f"{level} entered gameplay successfully ", f"{level} failed to enter gameplay") + TestHelper.enter_game_mode(enter_game_mode_tuple) + general.idle_wait_frames(1) + + # 5. Exit gameplay and report result using a tuple + exit_game_mode_tuple = (f"{level} exited gameplay successfully ", f"{level} failed to exit gameplay") + TestHelper.exit_game_mode(exit_game_mode_tuple) + + # 6. Look for errors or asserts. + TestHelper.wait_for_condition(lambda: error_tracer.has_errors or error_tracer.has_asserts, 1.0) + for error_info in error_tracer.errors: + Report.info(f"Error: {error_info.filename} {error_info.function} | {error_info.message}") + for assert_info in error_tracer.asserts: + Report.info(f"Assert: {assert_info.filename} {assert_info.function} | {assert_info.message}") + + +if __name__ == "__main__": + from editor_python_test_tools.utils import Report + Report.start_test(Atom_LevelLoadTest)