From 6308d01456252a1a12b23c844a8235ef93b07252 Mon Sep 17 00:00:00 2001 From: evanchia Date: Wed, 30 Jun 2021 15:29:06 -0700 Subject: [PATCH 1/4] Minor changes to system example test to match docs Signed-off-by: evanchia --- .../tests/example/test_system_example.py | 94 +++++++++---------- 1 file changed, 46 insertions(+), 48 deletions(-) diff --git a/Tools/LyTestTools/tests/example/test_system_example.py b/Tools/LyTestTools/tests/example/test_system_example.py index 3cb6128802..c4f4812428 100755 --- a/Tools/LyTestTools/tests/example/test_system_example.py +++ b/Tools/LyTestTools/tests/example/test_system_example.py @@ -3,7 +3,7 @@ Copyright (c) Contributors to the Open 3D Engine Project. For complete copyright SPDX-License-Identifier: Apache-2.0 OR MIT -Example test using LyTestTools to test Lumberyard. +Example test using LyTestTools to test Open 3D Engine. """ # Python built-in dependencies. import logging @@ -11,7 +11,7 @@ import logging # Third party dependencies. import pytest -# ly_test_tools dependencies. +# Optional ly_test_tools dependencies. import ly_test_tools.log.log_monitor import ly_remote_console.remote_console_commands as remote_console_commands @@ -22,28 +22,6 @@ import ly_remote_console.remote_console_commands as remote_console_commands logger = logging.getLogger(__name__) -@pytest.fixture -def remote_console(request): - """ - Creates a RemoteConsole() class instance to send console commands to the - Lumberyard client console. - :param request: _pytest.fixtures.SubRequest class that handles getting - a pytest fixture from a pytest function/fixture. - :return: ly_remote_console.remote_console_commands.RemoteConsole class instance - representing the Lumberyard remote console executable. - """ - # Initialize the RemoteConsole object to send commands to the Lumberyard client console. - console = remote_console_commands.RemoteConsole() - - # Custom teardown method for this remote_console fixture. - def teardown(): - console.stop() - - # Utilize request.addfinalizer() to add custom teardown() methods. - request.addfinalizer(teardown) # This pattern must be used in pytest version - - return console - # Shared parameters & fixtures for all test methods inside the TestSystemExample class. @pytest.mark.usefixtures("automatic_process_killer") @pytest.mark.parametrize('project', ['AutomatedTesting']) @@ -55,6 +33,41 @@ class TestSystemExample(object): For this test, we placed unique test values in test methods and shared test values in the test class. We also assume building has already been done, but the test should error if the build is mis-configured. """ + + # This test method only needs pytest.mark report values and shared test class parameters. + @pytest.mark.parametrize('processes_to_kill', ['Editor.exe']) + @pytest.mark.parametrize("launcher_platform", ['windows_editor']) + def test_SystemTestExample_WindowsPlatform_LaunchEditor(self, editor, processes_to_kill, launcher_platform): + """ + Tests launching the O3DE Editor is successful with the current build. + """ + # Launch the O3DE Editor & verify load is successful: + with editor.start(): + assert editor.is_alive(), ( + 'Editor failed to launch for the current O3DE build.') + + @pytest.fixture + def remote_console(request): + """ + Creates a RemoteConsole() class instance to send console commands to the + O3DE client console. + :param request: _pytest.fixtures.SubRequest class that handles getting + a pytest fixture from a pytest function/fixture. + :return: ly_remote_console.remote_console_commands.RemoteConsole class instance + representing the O3DE Remote Console executable. + """ + # Initialize the RemoteConsole object to send commands to the O3DE client console. + console = remote_console_commands.RemoteConsole() + + # Custom teardown method for this remote_console fixture. + def teardown(): + console.stop() + + # Utilize request.addfinalizer() to add custom teardown() methods. + request.addfinalizer(teardown) # This pattern must be used in pytest version + + return console + # This test method needs specific parameters not shared by all other tests in the class. # For targeting specific launchers, use the 'launcher_platform' pytest param like below: # @pytest.mark.parametrize("launcher_platform", ['android']) @@ -62,39 +75,24 @@ class TestSystemExample(object): # @pytest.mark.parametrize("asset_processor_platform", ['android']) @pytest.mark.parametrize('level', ['simple_jacklocomotion']) @pytest.mark.parametrize('load_wait', [120]) - @pytest.mark.test_case_id('C16806863') - def test_SystemTestExample_AllSupportedPlatforms_LaunchAutomatedTesting( - # launcher_platform, asset_processor_platform, # Re-add these here if you plan to use them. - self, launcher, remote_console, level, load_wait): + def test_SystemTestExample_AllSupportedPlatforms_LaunchAutomatedTesting(self, launcher, remote_console, level, + load_wait): """ - Tests launching the AutomatedTesting then launches the Lumberyard client & + Tests launching the AutomatedTesting then launches the O3DE client & loads the "simple_jacklocomotion" level using the remote console. Assumes the user already setup & built their machine for the test. """ - # Launch the Lumberyard client & remote console test case: + # Launch the O3DE client & remote console test case: with launcher.start(): remote_console.start() + # Wait for the expected log line to appear launcher_load = remote_console.expect_log_line( match_string='Level system is loading "simple_jacklocomotion"', timeout=load_wait) # Assert loading was successful using remote console logs: - assert launcher_load, ( - 'Launcher failed to load Lumberyard client with the ' - f'"{level}" level - waited "{load_wait}" seconds.') - - # This test method only needs pytest.mark report values and shared test class parameters. - @pytest.mark.parametrize('processes_to_kill', ['Editor.exe']) - @pytest.mark.parametrize("launcher_platform", ['windows_editor']) - @pytest.mark.test_case_id('C16806864') - def test_SystemTestExample_AllSupportedPlatforms_LaunchEditor(self, editor, processes_to_kill, launcher_platform): - """ - Tests launching the Lumberyard Editor is successful with the current build. - """ - # Launch the Lumberyard editor & verify load is successful: - with editor.start(): - assert editor.is_alive(), ( - 'Editor failed to launch for the current Lumberyard build.') + assert launcher_load, ('Launcher failed to load O3DE client with the "{level}" level - waited "{load_wait}" ' + 'seconds.') # Log monitoring example test. @pytest.mark.parametrize('level', ['simple_jacklocomotion']) @@ -106,7 +104,7 @@ class TestSystemExample(object): """ Tests that the logging paths created by LyTestTools can be monitored for results using the log monitor. """ - # Launch the Lumberyard client & initialize the log monitor. + # Launch the O3DE client & initialize the log monitor. file_to_monitor = launcher.workspace.info_log_path log_monitor = ly_test_tools.log.log_monitor.LogMonitor(launcher=launcher, log_file_path=file_to_monitor) @@ -115,7 +113,7 @@ class TestSystemExample(object): for expected_line in expected_lines: logger.info(expected_line) - # Start the Lumberyard client & test that the lines we logged can be viewed by the log monitor. + # Start the O3DE client & test that the lines we logged can be viewed by the log monitor. with launcher.start(): log_test = log_monitor.monitor_log_for_lines( expected_lines=expected_lines, # Defaults to None. From 9067fc3b99af1ff01384a873136e05262f1ed1c2 Mon Sep 17 00:00:00 2001 From: evanchia Date: Thu, 8 Jul 2021 10:24:49 -0700 Subject: [PATCH 2/4] Updating/Removing LyTestTools sample tests Signed-off-by: evanchia --- Tools/LyTestTools/tests/example/__init__.py | 5 - .../tests/example/test_system_example.py | 127 ------------------ Tools/LyTestTools/tests/integ/sanity_tests.py | 17 ++- 3 files changed, 14 insertions(+), 135 deletions(-) delete mode 100755 Tools/LyTestTools/tests/example/__init__.py delete mode 100755 Tools/LyTestTools/tests/example/test_system_example.py diff --git a/Tools/LyTestTools/tests/example/__init__.py b/Tools/LyTestTools/tests/example/__init__.py deleted file mode 100755 index e200fa77d0..0000000000 --- a/Tools/LyTestTools/tests/example/__init__.py +++ /dev/null @@ -1,5 +0,0 @@ -""" -Copyright (c) Contributors to the Open 3D Engine Project. For complete copyright and license terms please see the LICENSE at the root of this distribution. - -SPDX-License-Identifier: Apache-2.0 OR MIT -""" diff --git a/Tools/LyTestTools/tests/example/test_system_example.py b/Tools/LyTestTools/tests/example/test_system_example.py deleted file mode 100755 index c4f4812428..0000000000 --- a/Tools/LyTestTools/tests/example/test_system_example.py +++ /dev/null @@ -1,127 +0,0 @@ -""" -Copyright (c) Contributors to the Open 3D Engine Project. For complete copyright and license terms please see the LICENSE at the root of this distribution. - -SPDX-License-Identifier: Apache-2.0 OR MIT - -Example test using LyTestTools to test Open 3D Engine. -""" -# Python built-in dependencies. -import logging - -# Third party dependencies. -import pytest - -# Optional ly_test_tools dependencies. -import ly_test_tools.log.log_monitor -import ly_remote_console.remote_console_commands as remote_console_commands - -# Configuring the logging is done in ly_test_tools at the following location: -# ~/dev/Tools/LyTestTools/ly_test_tools/_internal/log/py_logging_util.py - -# Use the following logging pattern to hook all test logging together: -logger = logging.getLogger(__name__) - - -# Shared parameters & fixtures for all test methods inside the TestSystemExample class. -@pytest.mark.usefixtures("automatic_process_killer") -@pytest.mark.parametrize('project', ['AutomatedTesting']) -class TestSystemExample(object): - """ - Example test case class to hold a set of test case methods. - - The amount of tests run is based on the parametrization stacking made in each test method or class. - For this test, we placed unique test values in test methods and shared test values in the test class. - We also assume building has already been done, but the test should error if the build is mis-configured. - """ - - # This test method only needs pytest.mark report values and shared test class parameters. - @pytest.mark.parametrize('processes_to_kill', ['Editor.exe']) - @pytest.mark.parametrize("launcher_platform", ['windows_editor']) - def test_SystemTestExample_WindowsPlatform_LaunchEditor(self, editor, processes_to_kill, launcher_platform): - """ - Tests launching the O3DE Editor is successful with the current build. - """ - # Launch the O3DE Editor & verify load is successful: - with editor.start(): - assert editor.is_alive(), ( - 'Editor failed to launch for the current O3DE build.') - - @pytest.fixture - def remote_console(request): - """ - Creates a RemoteConsole() class instance to send console commands to the - O3DE client console. - :param request: _pytest.fixtures.SubRequest class that handles getting - a pytest fixture from a pytest function/fixture. - :return: ly_remote_console.remote_console_commands.RemoteConsole class instance - representing the O3DE Remote Console executable. - """ - # Initialize the RemoteConsole object to send commands to the O3DE client console. - console = remote_console_commands.RemoteConsole() - - # Custom teardown method for this remote_console fixture. - def teardown(): - console.stop() - - # Utilize request.addfinalizer() to add custom teardown() methods. - request.addfinalizer(teardown) # This pattern must be used in pytest version - - return console - - # This test method needs specific parameters not shared by all other tests in the class. - # For targeting specific launchers, use the 'launcher_platform' pytest param like below: - # @pytest.mark.parametrize("launcher_platform", ['android']) - # If you want to target different AssetProcessor platforms, use asset_processor_platform: - # @pytest.mark.parametrize("asset_processor_platform", ['android']) - @pytest.mark.parametrize('level', ['simple_jacklocomotion']) - @pytest.mark.parametrize('load_wait', [120]) - def test_SystemTestExample_AllSupportedPlatforms_LaunchAutomatedTesting(self, launcher, remote_console, level, - load_wait): - """ - Tests launching the AutomatedTesting then launches the O3DE client & - loads the "simple_jacklocomotion" level using the remote console. - Assumes the user already setup & built their machine for the test. - """ - # Launch the O3DE client & remote console test case: - with launcher.start(): - remote_console.start() - # Wait for the expected log line to appear - launcher_load = remote_console.expect_log_line( - match_string='Level system is loading "simple_jacklocomotion"', - timeout=load_wait) - - # Assert loading was successful using remote console logs: - assert launcher_load, ('Launcher failed to load O3DE client with the "{level}" level - waited "{load_wait}" ' - 'seconds.') - - # Log monitoring example test. - @pytest.mark.parametrize('level', ['simple_jacklocomotion']) - @pytest.mark.parametrize('expected_lines', [['Log Monitoring test 1', 'Log Monitoring test 2']]) - @pytest.mark.parametrize('unexpected_lines', [['Unexpected test 1', 'Unexpected test 2']]) - @pytest.mark.test_case_id('C21202585') - def test_SystemTestExample_AllSupportedPlatforms_LogMonitoring(self, level, launcher, expected_lines, - unexpected_lines): - """ - Tests that the logging paths created by LyTestTools can be monitored for results using the log monitor. - """ - # Launch the O3DE client & initialize the log monitor. - file_to_monitor = launcher.workspace.info_log_path - log_monitor = ly_test_tools.log.log_monitor.LogMonitor(launcher=launcher, - log_file_path=file_to_monitor) - - # Generate log lines to the info log using logger. - for expected_line in expected_lines: - logger.info(expected_line) - - # Start the O3DE client & test that the lines we logged can be viewed by the log monitor. - with launcher.start(): - log_test = log_monitor.monitor_log_for_lines( - expected_lines=expected_lines, # Defaults to None. - unexpected_lines=unexpected_lines, # Defaults to None. - halt_on_unexpected=True, # Defaults to False. - timeout=60) # Defaults to 30 - - # Assert the log monitor detected expected lines and did not detect any unexpected lines. - assert log_test, ( - f'Log monitoring failed. Used expected_lines values: {expected_lines} & ' - f'unexpected_lines values: {unexpected_lines}') diff --git a/Tools/LyTestTools/tests/integ/sanity_tests.py b/Tools/LyTestTools/tests/integ/sanity_tests.py index c6c5ce0c31..6c1f1079a9 100755 --- a/Tools/LyTestTools/tests/integ/sanity_tests.py +++ b/Tools/LyTestTools/tests/integ/sanity_tests.py @@ -15,8 +15,6 @@ import ly_test_tools.builtin.helpers as helpers import ly_test_tools.environment.process_utils as process_utils import ly_test_tools.environment.waiter as waiter -pytestmark = pytest.mark.SUITE_smoke - logger = logging.getLogger(__name__) # Note: For device testing, device ids must exist in ~/ly_test_tools/devices.ini, see README.txt for more info. @@ -24,31 +22,44 @@ logger = logging.getLogger(__name__) @pytest.mark.parametrize("project", ["AutomatedTesting"]) class TestAutomatedTestingProject(object): + def test_StartGameLauncher_Sanity(self, project): + # Kill processes that may interfere with the test process_utils.kill_processes_named(names=process_utils.LY_PROCESS_KILL_LIST, ignore_extensions=True) try: + # Create the Workspace object workspace = helpers.create_builtin_workspace(project=project) + # Create the Launcher object and add args launcher = launcher_helper.create_launcher(workspace) - launcher.args.extend(['-NullRenderer', '-BatchMode']) + launcher.args.extend(['-NullRenderer']) + # Call the game client executable with launcher.start(): + # Wait for the process to exist waiter.wait_for(lambda: process_utils.process_exists(f"{project}.GameLauncher.exe", ignore_extensions=True)) finally: + # Clean up processes after the test is finished process_utils.kill_processes_named(names=process_utils.LY_PROCESS_KILL_LIST, ignore_extensions=True) @pytest.mark.skipif(not ly_test_tools.WINDOWS, reason="Editor currently only functions on Windows") def test_StartEditor_Sanity(self, project): + # Kill processes that may interfere with the test process_utils.kill_processes_named(names=process_utils.LY_PROCESS_KILL_LIST, ignore_extensions=True) try: + # Create the Workspace object workspace = helpers.create_builtin_workspace(project=project) + # Create the Launcher object and add args editor = launcher_helper.create_editor(workspace) editor.args.extend(['-NullRenderer', '-autotest_mode']) + # Call the Editor executable with editor.start(): + # Wait for the process to exist waiter.wait_for(lambda: process_utils.process_exists("Editor", ignore_extensions=True)) finally: + # Clean up processes after the test is finished process_utils.kill_processes_named(names=process_utils.LY_PROCESS_KILL_LIST, ignore_extensions=True) From bc7e12278cd39d4b495fa9dde226fe74423d8bdb Mon Sep 17 00:00:00 2001 From: evanchia Date: Thu, 8 Jul 2021 15:55:06 -0700 Subject: [PATCH 3/4] changed nullrenderer arg for sanity test Signed-off-by: evanchia --- Tools/LyTestTools/tests/integ/sanity_tests.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Tools/LyTestTools/tests/integ/sanity_tests.py b/Tools/LyTestTools/tests/integ/sanity_tests.py index 6c1f1079a9..0aacfff8c6 100755 --- a/Tools/LyTestTools/tests/integ/sanity_tests.py +++ b/Tools/LyTestTools/tests/integ/sanity_tests.py @@ -33,7 +33,7 @@ class TestAutomatedTestingProject(object): # Create the Launcher object and add args launcher = launcher_helper.create_launcher(workspace) - launcher.args.extend(['-NullRenderer']) + launcher.args.extend(['-rhi=Null']) # Call the game client executable with launcher.start(): @@ -54,7 +54,7 @@ class TestAutomatedTestingProject(object): # Create the Launcher object and add args editor = launcher_helper.create_editor(workspace) - editor.args.extend(['-NullRenderer', '-autotest_mode']) + editor.args.extend(['-rhi=Null', '-autotest_mode']) # Call the Editor executable with editor.start(): From 12a8cc7cd75edabb235086e92be2ab93eb673117 Mon Sep 17 00:00:00 2001 From: evanchia Date: Mon, 12 Jul 2021 15:31:09 -0700 Subject: [PATCH 4/4] Removing deleted test from CMakeLists and updated in line comments for sample test Signed-off-by: evanchia --- Tools/LyTestTools/tests/CMakeLists.txt | 14 ---------- Tools/LyTestTools/tests/integ/sanity_tests.py | 26 +++++++++++++++++++ 2 files changed, 26 insertions(+), 14 deletions(-) diff --git a/Tools/LyTestTools/tests/CMakeLists.txt b/Tools/LyTestTools/tests/CMakeLists.txt index e5f270197c..5e57cbf123 100644 --- a/Tools/LyTestTools/tests/CMakeLists.txt +++ b/Tools/LyTestTools/tests/CMakeLists.txt @@ -52,18 +52,4 @@ if(PAL_TRAIT_BUILD_HOST_TOOLS AND PAL_TRAIT_BUILD_TESTS_SUPPORTED AND AutomatedT AutomatedTesting.Assets COMPONENT TestTools ) - - # Example tests. - ly_add_pytest( - NAME LyTestTools_ExampleTests_periodic_no_gpu - PATH ${CMAKE_CURRENT_LIST_DIR}/example/test_system_example.py - TEST_SERIAL - TEST_SUITE periodic - RUNTIME_DEPENDENCIES - Legacy::Editor - AssetProcessor - AutomatedTesting.GameLauncher - AutomatedTesting.Assets - COMPONENT TestTools - ) endif() diff --git a/Tools/LyTestTools/tests/integ/sanity_tests.py b/Tools/LyTestTools/tests/integ/sanity_tests.py index 0aacfff8c6..f68bc9ae57 100755 --- a/Tools/LyTestTools/tests/integ/sanity_tests.py +++ b/Tools/LyTestTools/tests/integ/sanity_tests.py @@ -6,24 +6,43 @@ SPDX-License-Identifier: Apache-2.0 OR MIT A sanity test for the built-in fixtures. Launch the windows launcher attached to the currently installed instance. """ +# Import any dependencies for the test. import logging import pytest +# Import any desired LTT modules from the package `ly_test_tools`. All LTT modules can be viewed at `Tools/LyTestTools/ly_test_tools`. import ly_test_tools +# The `launchers.launcher_helper` module helps create Launcher objects which control the Open 3D Engine (O3DE) Editor and game clients. import ly_test_tools.launchers.launcher_helper as launcher_helper +# The `builtin.helpers` module helps create the Workspace object, which controls the testing workspace in LTT. import ly_test_tools.builtin.helpers as helpers +# The `environment` module contains tools that involve the system's environment such as processes or timed waiters. import ly_test_tools.environment.process_utils as process_utils import ly_test_tools.environment.waiter as waiter +# Initialize a logger instance to hook all test logs together. The sub-logger pattern below makes it easy to track which file creates a log line. logger = logging.getLogger(__name__) # Note: For device testing, device ids must exist in ~/ly_test_tools/devices.ini, see README.txt for more info. +# First define the class `TestAutomatedTestingProject` to group test functions together. +# The example test contains two test functions: `test_StartGameLauncher_Sanity` and `test_StartEditor_Sanity`. @pytest.mark.parametrize("project", ["AutomatedTesting"]) +# The example test utilizes Pytest parameterization. The following sets the `project` parameter to `AutomatedTesting` +# for both test functions. Notice that the Pytest mark is defined at the class level to affect both test functions. class TestAutomatedTestingProject(object): def test_StartGameLauncher_Sanity(self, project): + """ + The `test_StartGameLauncher_Sanity` test function verifies that the O3DE game client launches successfully. + Start the test by utilizing the `kill_processes_named` function to close any open O3DE processes that may + interfere with the test. The Workspace object emulates the O3DE package by locating the engine and project + directories. The Launcher object controls the O3DE game client and requires a Workspace object for + initialization. Add the `-rhi=Null` arg to the executable call to disable GPU rendering. This allows the + test to run on instances without a GPU. We launch the game client executable and wait for the process to exist. + A try/finally block ensures proper test cleanup if issues occur during the test. + """ # Kill processes that may interfere with the test process_utils.kill_processes_named(names=process_utils.LY_PROCESS_KILL_LIST, ignore_extensions=True) @@ -45,6 +64,13 @@ class TestAutomatedTestingProject(object): @pytest.mark.skipif(not ly_test_tools.WINDOWS, reason="Editor currently only functions on Windows") def test_StartEditor_Sanity(self, project): + """ + The `test_StartEditor_Sanity` test function is similar to the previous example with minor adjustments. A + PyTest mark skips the test if the operating system is not Windows. We use the `create_editor` function instead + of `create_launcher` to create an Editor type launcher instead of a game client type launcher. The additional + `-autotest_mode` arg supresses modal dialogs from interfering with our test. We launch the Editor executable and + wait for the process to exist. + """ # Kill processes that may interfere with the test process_utils.kill_processes_named(names=process_utils.LY_PROCESS_KILL_LIST, ignore_extensions=True)