birkeh/o3de
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Merge pull request #1979 from aws-lumberyard-dev/system_test_example_update

Browse Source 
System test example update
This commit is contained in:
Sean Sweeney
2021-07-14 16:59:01 -07:00
committed by GitHub
parent 72ea2ca1b4 12a8cc7cd7
commit 9e81cedf80
4 changed files with 41 additions and 152 deletions
Download Patch File Download Diff File Expand all files Collapse all files
Tools/LyTestTools/tests/CMakeLists.txt
-14
View File
@@ -52,18 +52,4 @@ if(PAL_TRAIT_BUILD_HOST_TOOLS AND PAL_TRAIT_BUILD_TESTS_SUPPORTED AND AutomatedT
AutomatedTesting.Assets
COMPONENT TestTools
)
# Example tests.
ly_add_pytest(
NAME LyTestTools_ExampleTests_periodic_no_gpu
PATH ${CMAKE_CURRENT_LIST_DIR}/example/test_system_example.py
TEST_SERIAL
TEST_SUITE periodic
RUNTIME_DEPENDENCIES
Legacy::Editor
AssetProcessor
AutomatedTesting.GameLauncher
AutomatedTesting.Assets
COMPONENT TestTools
)
endif()
Tools/LyTestTools/tests/example/__init__.py
-5
View File
@@ -1,5 +0,0 @@
"""
Copyright (c) Contributors to the Open 3D Engine Project. For complete copyright and license terms please see the LICENSE at the root of this distribution.
SPDX-License-Identifier: Apache-2.0 OR MIT
"""
Tools/LyTestTools/tests/example/test_system_example.py
-129
View File
@@ -1,129 +0,0 @@
"""
Copyright (c) Contributors to the Open 3D Engine Project. For complete copyright and license terms please see the LICENSE at the root of this distribution.
SPDX-License-Identifier: Apache-2.0 OR MIT
Example test using LyTestTools to test Lumberyard.
"""
# Python built-in dependencies.
import logging
# Third party dependencies.
import pytest
# ly_test_tools dependencies.
import ly_test_tools.log.log_monitor
import ly_remote_console.remote_console_commands as remote_console_commands
# Configuring the logging is done in ly_test_tools at the following location:
# ~/dev/Tools/LyTestTools/ly_test_tools/_internal/log/py_logging_util.py
# Use the following logging pattern to hook all test logging together:
logger = logging.getLogger(__name__)
@pytest.fixture
def remote_console(request):
"""
Creates a RemoteConsole() class instance to send console commands to the
Lumberyard client console.
:param request: _pytest.fixtures.SubRequest class that handles getting
a pytest fixture from a pytest function/fixture.
:return: ly_remote_console.remote_console_commands.RemoteConsole class instance
representing the Lumberyard remote console executable.
"""
# Initialize the RemoteConsole object to send commands to the Lumberyard client console.
console = remote_console_commands.RemoteConsole()
# Custom teardown method for this remote_console fixture.
def teardown():
console.stop()
# Utilize request.addfinalizer() to add custom teardown() methods.
request.addfinalizer(teardown) # This pattern must be used in pytest version
return console
# Shared parameters & fixtures for all test methods inside the TestSystemExample class.
@pytest.mark.usefixtures("automatic_process_killer")
@pytest.mark.parametrize('project', ['AutomatedTesting'])
class TestSystemExample(object):
"""
Example test case class to hold a set of test case methods.
The amount of tests run is based on the parametrization stacking made in each test method or class.
For this test, we placed unique test values in test methods and shared test values in the test class.
We also assume building has already been done, but the test should error if the build is mis-configured.
"""
# This test method needs specific parameters not shared by all other tests in the class.
# For targeting specific launchers, use the 'launcher_platform' pytest param like below:
# @pytest.mark.parametrize("launcher_platform", ['android'])
# If you want to target different AssetProcessor platforms, use asset_processor_platform:
# @pytest.mark.parametrize("asset_processor_platform", ['android'])
@pytest.mark.parametrize('level', ['simple_jacklocomotion'])
@pytest.mark.parametrize('load_wait', [120])
@pytest.mark.test_case_id('C16806863')
def test_SystemTestExample_AllSupportedPlatforms_LaunchAutomatedTesting(
# launcher_platform, asset_processor_platform, # Re-add these here if you plan to use them.
self, launcher, remote_console, level, load_wait):
"""
Tests launching the AutomatedTesting then launches the Lumberyard client &
loads the "simple_jacklocomotion" level using the remote console.
Assumes the user already setup & built their machine for the test.
"""
# Launch the Lumberyard client & remote console test case:
with launcher.start():
remote_console.start()
launcher_load = remote_console.expect_log_line(
match_string='Level system is loading "simple_jacklocomotion"',
timeout=load_wait)
# Assert loading was successful using remote console logs:
assert launcher_load, (
'Launcher failed to load Lumberyard client with the '
f'"{level}" level - waited "{load_wait}" seconds.')
# This test method only needs pytest.mark report values and shared test class parameters.
@pytest.mark.parametrize('processes_to_kill', ['Editor.exe'])
@pytest.mark.parametrize("launcher_platform", ['windows_editor'])
@pytest.mark.test_case_id('C16806864')
def test_SystemTestExample_AllSupportedPlatforms_LaunchEditor(self, editor, processes_to_kill, launcher_platform):
"""
Tests launching the Lumberyard Editor is successful with the current build.
"""
# Launch the Lumberyard editor & verify load is successful:
with editor.start():
assert editor.is_alive(), (
'Editor failed to launch for the current Lumberyard build.')
# Log monitoring example test.
@pytest.mark.parametrize('level', ['simple_jacklocomotion'])
@pytest.mark.parametrize('expected_lines', [['Log Monitoring test 1', 'Log Monitoring test 2']])
@pytest.mark.parametrize('unexpected_lines', [['Unexpected test 1', 'Unexpected test 2']])
@pytest.mark.test_case_id('C21202585')
def test_SystemTestExample_AllSupportedPlatforms_LogMonitoring(self, level, launcher, expected_lines,
unexpected_lines):
"""
Tests that the logging paths created by LyTestTools can be monitored for results using the log monitor.
"""
# Launch the Lumberyard client & initialize the log monitor.
file_to_monitor = launcher.workspace.info_log_path
log_monitor = ly_test_tools.log.log_monitor.LogMonitor(launcher=launcher,
log_file_path=file_to_monitor)
# Generate log lines to the info log using logger.
for expected_line in expected_lines:
logger.info(expected_line)
# Start the Lumberyard client & test that the lines we logged can be viewed by the log monitor.
with launcher.start():
log_test = log_monitor.monitor_log_for_lines(
expected_lines=expected_lines, # Defaults to None.
unexpected_lines=unexpected_lines, # Defaults to None.
halt_on_unexpected=True, # Defaults to False.
timeout=60) # Defaults to 30
# Assert the log monitor detected expected lines and did not detect any unexpected lines.
assert log_test, (
f'Log monitoring failed. Used expected_lines values: {expected_lines} & '
f'unexpected_lines values: {unexpected_lines}')
Tools/LyTestTools/tests/integ/sanity_tests.py
+41 -4
View File
@@ -6,49 +6,86 @@ SPDX-License-Identifier: Apache-2.0 OR MIT
A sanity test for the built-in fixtures.
Launch the windows launcher attached to the currently installed instance.
"""
# Import any dependencies for the test.
import logging
import pytest
# Import any desired LTT modules from the package `ly_test_tools`. All LTT modules can be viewed at `Tools/LyTestTools/ly_test_tools`.
import ly_test_tools
# The `launchers.launcher_helper` module helps create Launcher objects which control the Open 3D Engine (O3DE) Editor and game clients.
import ly_test_tools.launchers.launcher_helper as launcher_helper
# The `builtin.helpers` module helps create the Workspace object, which controls the testing workspace in LTT.
import ly_test_tools.builtin.helpers as helpers
# The `environment` module contains tools that involve the system's environment such as processes or timed waiters.
import ly_test_tools.environment.process_utils as process_utils
import ly_test_tools.environment.waiter as waiter
pytestmark = pytest.mark.SUITE_smoke
# Initialize a logger instance to hook all test logs together. The sub-logger pattern below makes it easy to track which file creates a log line.
logger = logging.getLogger(__name__)
# Note: For device testing, device ids must exist in ~/ly_test_tools/devices.ini, see README.txt for more info.
# First define the class `TestAutomatedTestingProject` to group test functions together.
# The example test contains two test functions: `test_StartGameLauncher_Sanity` and `test_StartEditor_Sanity`.
@pytest.mark.parametrize("project", ["AutomatedTesting"])
# The example test utilizes Pytest parameterization. The following sets the `project` parameter to `AutomatedTesting`
# for both test functions. Notice that the Pytest mark is defined at the class level to affect both test functions.
class TestAutomatedTestingProject(object):
def test_StartGameLauncher_Sanity(self, project):
"""
The `test_StartGameLauncher_Sanity` test function verifies that the O3DE game client launches successfully.
Start the test by utilizing the `kill_processes_named` function to close any open O3DE processes that may
interfere with the test. The Workspace object emulates the O3DE package by locating the engine and project
directories. The Launcher object controls the O3DE game client and requires a Workspace object for
initialization. Add the `-rhi=Null` arg to the executable call to disable GPU rendering. This allows the
test to run on instances without a GPU. We launch the game client executable and wait for the process to exist.
A try/finally block ensures proper test cleanup if issues occur during the test.
"""
# Kill processes that may interfere with the test
process_utils.kill_processes_named(names=process_utils.LY_PROCESS_KILL_LIST, ignore_extensions=True)
try:
# Create the Workspace object
workspace = helpers.create_builtin_workspace(project=project)
# Create the Launcher object and add args
launcher = launcher_helper.create_launcher(workspace)
launcher.args.extend(['-NullRenderer', '-BatchMode'])
launcher.args.extend(['-rhi=Null'])
# Call the game client executable
with launcher.start():
# Wait for the process to exist
waiter.wait_for(lambda: process_utils.process_exists(f"{project}.GameLauncher.exe", ignore_extensions=True))
finally:
# Clean up processes after the test is finished
process_utils.kill_processes_named(names=process_utils.LY_PROCESS_KILL_LIST, ignore_extensions=True)
@pytest.mark.skipif(not ly_test_tools.WINDOWS, reason="Editor currently only functions on Windows")
def test_StartEditor_Sanity(self, project):
"""
The `test_StartEditor_Sanity` test function is similar to the previous example with minor adjustments. A
PyTest mark skips the test if the operating system is not Windows. We use the `create_editor` function instead
of `create_launcher` to create an Editor type launcher instead of a game client type launcher. The additional
`-autotest_mode` arg supresses modal dialogs from interfering with our test. We launch the Editor executable and
wait for the process to exist.
"""
# Kill processes that may interfere with the test
process_utils.kill_processes_named(names=process_utils.LY_PROCESS_KILL_LIST, ignore_extensions=True)
try:
# Create the Workspace object
workspace = helpers.create_builtin_workspace(project=project)
# Create the Launcher object and add args
editor = launcher_helper.create_editor(workspace)
editor.args.extend(['-NullRenderer', '-autotest_mode'])
editor.args.extend(['-rhi=Null', '-autotest_mode'])
# Call the Editor executable
with editor.start():
# Wait for the process to exist
waiter.wait_for(lambda: process_utils.process_exists("Editor", ignore_extensions=True))
finally:
# Clean up processes after the test is finished
process_utils.kill_processes_named(names=process_utils.LY_PROCESS_KILL_LIST, ignore_extensions=True)