From 27d2437308539e5c4d02d38b1a1216077050b163 Mon Sep 17 00:00:00 2001 From: evanchia Date: Thu, 18 Nov 2021 16:34:06 -0800 Subject: [PATCH 1/4] Defaults the LyTestTools TestResults folder to /Testing. Also removes the default saving of logs on launcher cleanup and automatically saves logs for tests using the editor_test module Signed-off-by: evanchia --- .../ly_test_tools/launchers/platforms/base.py | 1 - Tools/LyTestTools/ly_test_tools/o3de/editor_test.py | 11 ++++++++++- cmake/LYTestWrappers.cmake | 4 +++- 3 files changed, 13 insertions(+), 3 deletions(-) diff --git a/Tools/LyTestTools/ly_test_tools/launchers/platforms/base.py b/Tools/LyTestTools/ly_test_tools/launchers/platforms/base.py index d32be10562..52675f2ce5 100755 --- a/Tools/LyTestTools/ly_test_tools/launchers/platforms/base.py +++ b/Tools/LyTestTools/ly_test_tools/launchers/platforms/base.py @@ -153,7 +153,6 @@ class Launcher(object): :return: None """ self.workspace.asset_processor.stop() - self.save_project_log_files() def save_project_log_files(self): # type: () -> None diff --git a/Tools/LyTestTools/ly_test_tools/o3de/editor_test.py b/Tools/LyTestTools/ly_test_tools/o3de/editor_test.py index ae6ed16321..bd035b26ed 100644 --- a/Tools/LyTestTools/ly_test_tools/o3de/editor_test.py +++ b/Tools/LyTestTools/ly_test_tools/o3de/editor_test.py @@ -776,6 +776,11 @@ class EditorTestSuite(): editor.kill() editor_log_content = editor_utils.retrieve_editor_log_content(run_id, log_name, workspace) test_result = Result.Timeout.create(test_spec, output, test_spec.timeout, editor_log_content) + finally: + log_folder = editor_utils.retrieve_log_path(run_id, workspace) + if run_id == 0: + log_folder = workspace.paths.project_log() + workspace.artifact_manager.save_artifact(log_folder) editor_log_content = editor_utils.retrieve_editor_log_content(run_id, log_name, workspace) results = self._get_results_using_output([test_spec], output, editor_log_content) @@ -890,7 +895,11 @@ class EditorTestSuite(): results[test_spec_name] = Result.Timeout.create(timed_out_result.test_spec, results[test_spec_name].output, self.timeout_editor_shared_test, result.editor_log) - + finally: + log_folder = editor_utils.retrieve_log_path(run_id, workspace) + if run_id == 0: + log_folder = workspace.paths.project_log() + workspace.artifact_manager.save_artifact(log_folder) return results def _run_single_test(self, request: Request, workspace: AbstractWorkspace, editor: Editor, diff --git a/cmake/LYTestWrappers.cmake b/cmake/LYTestWrappers.cmake index 9b139645fb..80ac70f294 100644 --- a/cmake/LYTestWrappers.cmake +++ b/cmake/LYTestWrappers.cmake @@ -313,12 +313,14 @@ function(ly_add_pytest) # Add the script path to the test target params set(LY_TEST_PARAMS "${ly_add_pytest_PATH}") + string(REPLACE "::" "_" pytest_output_directory "${CMAKE_BINARY_DIR}/Testing/LyTestTools/${ly_add_pytest_NAME}") + ly_add_test( NAME ${ly_add_pytest_NAME} PARENT_NAME ${ly_add_pytest_NAME} TEST_SUITE ${ly_add_pytest_TEST_SUITE} LABELS FRAMEWORK_pytest - TEST_COMMAND ${LY_PYTEST_EXECUTABLE} ${ly_add_pytest_PATH} ${ly_add_pytest_EXTRA_ARGS} --junitxml=${pytest_report_directory} ${custom_marks_args} + TEST_COMMAND ${LY_PYTEST_EXECUTABLE} ${ly_add_pytest_PATH} ${ly_add_pytest_EXTRA_ARGS} --output-path ${pytest_output_directory} --junitxml=${pytest_report_directory} ${custom_marks_args} TEST_LIBRARY pytest COMPONENT ${ly_add_pytest_COMPONENT} ${ly_add_pytest_UNPARSED_ARGUMENTS} From c12ae72164739273bf07d862fa2d573480a6ae3a Mon Sep 17 00:00:00 2001 From: evanchia Date: Mon, 22 Nov 2021 16:02:19 -0800 Subject: [PATCH 2/4] Saving logs to test artifact folder directly instead of moving them Signed-off-by: evanchia --- .../LyTestTools/ly_test_tools/o3de/editor_test.py | 15 +++------------ 1 file changed, 3 insertions(+), 12 deletions(-) diff --git a/Tools/LyTestTools/ly_test_tools/o3de/editor_test.py b/Tools/LyTestTools/ly_test_tools/o3de/editor_test.py index bd035b26ed..94f67c2714 100644 --- a/Tools/LyTestTools/ly_test_tools/o3de/editor_test.py +++ b/Tools/LyTestTools/ly_test_tools/o3de/editor_test.py @@ -48,6 +48,7 @@ import ly_test_tools.environment.waiter as waiter import ly_test_tools.environment.process_utils as process_utils import ly_test_tools.o3de.editor_test import ly_test_tools.o3de.editor_test_utils as editor_utils +import ly_test_tools._internal.pytest_plugin from ly_test_tools.o3de.asset_processor import AssetProcessor from ly_test_tools.launchers.exceptions import WaitTimeoutError @@ -751,7 +752,7 @@ class EditorTestSuite(): cmdline = [ "--runpythontest", test_filename, "-logfile", f"@log@/{log_name}", - "-project-log-path", editor_utils.retrieve_log_path(run_id, workspace)] + test_cmdline_args + "-project-log-path", ly_test_tools._internal.pytest_plugin.output_path] + test_cmdline_args editor.args.extend(cmdline) editor.start(backupFiles = False, launch_ap = False, configure_settings=False) @@ -776,11 +777,6 @@ class EditorTestSuite(): editor.kill() editor_log_content = editor_utils.retrieve_editor_log_content(run_id, log_name, workspace) test_result = Result.Timeout.create(test_spec, output, test_spec.timeout, editor_log_content) - finally: - log_folder = editor_utils.retrieve_log_path(run_id, workspace) - if run_id == 0: - log_folder = workspace.paths.project_log() - workspace.artifact_manager.save_artifact(log_folder) editor_log_content = editor_utils.retrieve_editor_log_content(run_id, log_name, workspace) results = self._get_results_using_output([test_spec], output, editor_log_content) @@ -818,7 +814,7 @@ class EditorTestSuite(): cmdline = [ "--runpythontest", test_filenames_str, "-logfile", f"@log@/{log_name}", - "-project-log-path", editor_utils.retrieve_log_path(run_id, workspace)] + test_cmdline_args + "-project-log-path", ly_test_tools._internal.pytest_plugin.output_path] + test_cmdline_args editor.args.extend(cmdline) editor.start(backupFiles = False, launch_ap = False, configure_settings=False) @@ -895,11 +891,6 @@ class EditorTestSuite(): results[test_spec_name] = Result.Timeout.create(timed_out_result.test_spec, results[test_spec_name].output, self.timeout_editor_shared_test, result.editor_log) - finally: - log_folder = editor_utils.retrieve_log_path(run_id, workspace) - if run_id == 0: - log_folder = workspace.paths.project_log() - workspace.artifact_manager.save_artifact(log_folder) return results def _run_single_test(self, request: Request, workspace: AbstractWorkspace, editor: Editor, From 9a8f26fab50f558fa1fee951b566d0801df20c3d Mon Sep 17 00:00:00 2001 From: evanchia Date: Mon, 29 Nov 2021 10:45:35 -0800 Subject: [PATCH 3/4] moved cmake variables Signed-off-by: evanchia --- cmake/LYTestWrappers.cmake | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/cmake/LYTestWrappers.cmake b/cmake/LYTestWrappers.cmake index 80ac70f294..03146b6045 100644 --- a/cmake/LYTestWrappers.cmake +++ b/cmake/LYTestWrappers.cmake @@ -32,6 +32,7 @@ endif() # Set and create folders for PyTest and GTest xml output ly_set(PYTEST_XML_OUTPUT_DIR ${CMAKE_BINARY_DIR}/Testing/Pytest) ly_set(GTEST_XML_OUTPUT_DIR ${CMAKE_BINARY_DIR}/Testing/Gtest) +ly_set(LYTESTTOOLS_OUTPUT_DIR ${CMAKE_BINARY_DIR}/Testing/LyTestTools) file(MAKE_DIRECTORY ${PYTEST_XML_OUTPUT_DIR}) file(MAKE_DIRECTORY ${GTEST_XML_OUTPUT_DIR}) @@ -309,12 +310,11 @@ function(ly_add_pytest) endif() string(REPLACE "::" "_" pytest_report_directory "${PYTEST_XML_OUTPUT_DIR}/${ly_add_pytest_NAME}.xml") + string(REPLACE "::" "_" pytest_output_directory "${LYTESTTOOLS_OUTPUT_DIR}/${ly_add_pytest_NAME}") # Add the script path to the test target params set(LY_TEST_PARAMS "${ly_add_pytest_PATH}") - string(REPLACE "::" "_" pytest_output_directory "${CMAKE_BINARY_DIR}/Testing/LyTestTools/${ly_add_pytest_NAME}") - ly_add_test( NAME ${ly_add_pytest_NAME} PARENT_NAME ${ly_add_pytest_NAME} From 22273f6808b4062932cee6376b85da9e80454abf Mon Sep 17 00:00:00 2001 From: evanchia Date: Thu, 2 Dec 2021 13:47:34 -0800 Subject: [PATCH 4/4] fixing unit tests Signed-off-by: evanchia --- Tools/LyTestTools/tests/unit/test_launcher_base.py | 9 --------- 1 file changed, 9 deletions(-) diff --git a/Tools/LyTestTools/tests/unit/test_launcher_base.py b/Tools/LyTestTools/tests/unit/test_launcher_base.py index 5264e1bc28..4fe72b889e 100755 --- a/Tools/LyTestTools/tests/unit/test_launcher_base.py +++ b/Tools/LyTestTools/tests/unit/test_launcher_base.py @@ -146,15 +146,6 @@ class TestBaseLauncher: mock_stop_ap.assert_called_once() - @mock.patch('ly_test_tools.launchers.platforms.base.Launcher.save_project_log_files') - def test_Teardown_TeardownCalled_CallsSaveProjectLogFiles(self, under_test): - mock_workspace = mock.MagicMock() - mock_args = ['foo'] - mock_launcher = ly_test_tools.launchers.Launcher(mock_workspace, mock_args) - - mock_launcher.teardown() - under_test.assert_called_once() - @mock.patch('os.path.exists', mock.MagicMock(return_value=True)) @mock.patch('ly_test_tools._internal.managers.artifact_manager.ArtifactManager.save_artifact') @mock.patch('os.listdir')