From 84493760caa0718c46789d6bcc4ec60fd5f5e385 Mon Sep 17 00:00:00 2001 From: evanchia Date: Mon, 18 Oct 2021 13:27:40 -0700 Subject: [PATCH] addressing PR feedback Signed-off-by: evanchia --- .../_internal/pytest_plugin/editor_test.py | 11 +- .../pytest_plugin/test_tools_fixtures.py | 16 -- .../ly_test_tools/o3de/editor_test.py | 55 +++---- .../ly_test_tools/o3de/editor_test_utils.py | 9 +- .../tests/unit/test_editor_test_utils.py | 26 +-- Tools/LyTestTools/tests/unit/test_fixtures.py | 11 -- .../tests/unit/test_o3de_editor_test.py | 149 ++++++++---------- .../unit/test_pytest_plugin_editor_test.py | 2 +- 8 files changed, 118 insertions(+), 161 deletions(-) diff --git a/Tools/LyTestTools/ly_test_tools/_internal/pytest_plugin/editor_test.py b/Tools/LyTestTools/ly_test_tools/_internal/pytest_plugin/editor_test.py index 2df5810047..19caf7e090 100644 --- a/Tools/LyTestTools/ly_test_tools/_internal/pytest_plugin/editor_test.py +++ b/Tools/LyTestTools/ly_test_tools/_internal/pytest_plugin/editor_test.py @@ -4,7 +4,8 @@ For complete copyright and license terms please see the LICENSE at the root of t SPDX-License-Identifier: Apache-2.0 OR MIT -Utility for specifying an Editor test, supports seamless parallelization and/or batching of tests. +Utility for specifying an Editor test, supports seamless parallelization and/or batching of tests. This is not a set of +tools to directly invoke, but a plugin with functions intended to be called by only the Pytest framework. """ import pytest @@ -15,7 +16,7 @@ __test__ = False def pytest_addoption(parser): # type (argparse.ArgumentParser) -> None """ - Options when running tests in batches or parallel. + Options when running editor tests in batches or parallel. :param parser: The ArgumentParser object :return: None """ @@ -27,8 +28,8 @@ def pytest_pycollect_makeitem(collector, name, obj): # type (PyCollector, str, object) -> Collector """ Create a custom custom item collection if the class defines pytest_custom_makeitem function. This is used for - automtically generating test functions with a custom collector. - :param collector: The Python test collector + automatically generating test functions with a custom collector. + :param collector: The Pytest collector :param name: Name of the collector :param obj: The custom collector, normally an EditorTestSuite.EditorTestClass object :return: Returns the custom collector @@ -40,7 +41,7 @@ def pytest_pycollect_makeitem(collector, name, obj): @pytest.hookimpl(hookwrapper=True) def pytest_collection_modifyitems(session, items, config): - # type (Session, list, Config) -> None + # type (Session, List[EditorTestBase], Config) -> None """ Add custom modification of items. This is used for adding the runners into the item list. :param session: The Pytest Session diff --git a/Tools/LyTestTools/ly_test_tools/_internal/pytest_plugin/test_tools_fixtures.py b/Tools/LyTestTools/ly_test_tools/_internal/pytest_plugin/test_tools_fixtures.py index 3115c28406..af29064766 100755 --- a/Tools/LyTestTools/ly_test_tools/_internal/pytest_plugin/test_tools_fixtures.py +++ b/Tools/LyTestTools/ly_test_tools/_internal/pytest_plugin/test_tools_fixtures.py @@ -55,22 +55,6 @@ def pytest_configure(config): ly_test_tools._internal.pytest_plugin.build_directory = _get_build_directory(config) ly_test_tools._internal.pytest_plugin.output_path = _get_output_path(config) -def pytest_pycollect_makeitem(collector, name, obj): - # type (PyCollector, str, object) -> Collector - """ - Create a custom custom item collection if the class defines pytest_custom_makeitem function. This is used for - automtically generating test functions with a custom collector. - :param collector: The Python test collector - :param name: Name of the collector - :param obj: The custom collector, normally an EditorTestSuite.EditorTestClass object - :return: Returns the custom collector - """ - import inspect - if inspect.isclass(obj): - for base in obj.__bases__: - if hasattr(base, "pytest_custom_makeitem"): - return base.pytest_custom_makeitem(collector, name, obj) - def _get_build_directory(config): """ Fetch and verify the cmake build directory CLI arg, without creating an error when unset diff --git a/Tools/LyTestTools/ly_test_tools/o3de/editor_test.py b/Tools/LyTestTools/ly_test_tools/o3de/editor_test.py index 39413c22a3..789d07f200 100644 --- a/Tools/LyTestTools/ly_test_tools/o3de/editor_test.py +++ b/Tools/LyTestTools/ly_test_tools/o3de/editor_test.py @@ -321,8 +321,9 @@ class EditorTestSuite(): def editor_test_data(self, request): # type (request) -> TestData """ - Yields a generator to capture the test results and an AssetProcessor object. - :request: The pytest request + Yields a per-testsuite structure to store the data of each test result and an AssetProcessor object that will be + re-used on the whole suite + :request: The Pytest request :yield: The TestData object """ self._editor_test_data(request) @@ -513,12 +514,15 @@ class EditorTestSuite(): @classmethod def pytest_custom_modify_items(cls, session, items, config): - # type () -> None + # type (Session, List[EditorTestBase], Config) -> None """ - + Adds the runners' functions and filters the tests that will run. The runners will be added if they have any + selected tests + :param session: The Pytest Session + :param items: The test case functions + :param config: The Pytest Config object + :return: None """ - # Add here the runners functions and filter the tests that will be run. - # The runners will be added if they have any selected tests new_items = [] for runner in cls._runners: runner.tests[:] = cls.filter_session_shared_tests(items, runner.tests) @@ -535,7 +539,7 @@ class EditorTestSuite(): @classmethod def get_single_tests(cls): - # type () -> list + # type () -> List """ Grabs all of the EditorSingleTests subclassed tests from the EditorTestSuite class Usage example: @@ -549,7 +553,7 @@ class EditorTestSuite(): @classmethod def get_shared_tests(cls): - # type () -> list + # type () -> List """ Grabs all of the EditorSharedTests from the EditorTestSuite Usage example: @@ -563,7 +567,7 @@ class EditorTestSuite(): @classmethod def get_session_shared_tests(cls, session): - # type (Session) -> list[EditorTestBase] + # type (Session) -> List[EditorTestBase] """ Filters and returns all of the shared tests in a given session. :session: The test session @@ -574,7 +578,7 @@ class EditorTestSuite(): @staticmethod def filter_session_shared_tests(session_items, shared_tests): - # type (list, list) -> list[EditorTestBase] + # type (List[EditorTestBase, List[EditorSharedTest]) -> List[EditorTestBase] """ Retrieve the test sub-set that was collected this can be less than the original set if were overriden via -k argument or similars @@ -596,7 +600,7 @@ class EditorTestSuite(): @staticmethod def filter_shared_tests(shared_tests, is_batchable=False, is_parallelizable=False): - # type (list, bool, bool) -> list + # type (List[EditorSharedTest], bool, bool) -> List[EditorSharedTest] """ Filters and returns all tests based off of if they are batchable and/or parallelizable :shared_tests: All shared tests @@ -654,7 +658,7 @@ class EditorTestSuite(): @staticmethod def _get_results_using_output(test_spec_list, output, editor_log_content): - # type(list, str, str) -> dict{str: Result} + # type(List[EditorTestBase], str, str) -> dict{str: Result} """ Utility function for parsing the output information from the editor. It deserializes the JSON content printed in the output for every test and returns that information. @@ -732,7 +736,7 @@ class EditorTestSuite(): ### Running tests ### def _exec_editor_test(self, request, workspace, editor, run_id, log_name, test_spec, cmdline_args = []): - # type (Request, AbstractWorkspace, Editor, int, str, EditorTestBase, list[str] -> dict{str: Result} + # type (Request, AbstractWorkspace, Editor, int, str, EditorTestBase, List[str] -> dict{str: Result} """ Starts the editor with the given test and retuns an result dict with a single element specifying the result :request: The pytest request @@ -794,7 +798,7 @@ class EditorTestSuite(): return results def _exec_editor_multitest(self, request, workspace, editor, run_id, log_name, test_spec_list, cmdline_args=[]): - # type (Request, AbstractWorkspace, Editor, int, str, list[EditorTestBase], list[str]) -> dict{str: Result} + # type (Request, AbstractWorkspace, Editor, int, str, List[EditorTestBase], List[str]) -> dict{str: Result} """ Starts an editor executable with a list of tests and returns a dict of the result of every test ran within that editor instance. In case of failure this function also parses the editor output to find out what specific tests @@ -804,7 +808,7 @@ class EditorTestSuite(): :editor: The LyTestTools Editor object :run_id: The unique run id :log_name: The name of the editor log to retrieve - :test_spec_list: A list of EditorTestBase tests to run + :test_spec_list: A list of EditorTestBase tests to run in the same editor instance :cmdline_args: Any additional command line args :return: A dict of Result objects """ @@ -820,8 +824,7 @@ class EditorTestSuite(): editor_utils.cycle_crash_report(run_id, workspace) results = {} - test_filenames_str = ";".join(editor_utils.get_testcase_module_filepath(test_spec.test_module) for - test_spec in test_spec_list) + test_filenames_str = ";".join(editor_utils.get_testcase_module_filepath(test_spec.test_module) for test_spec in test_spec_list) cmdline = [ "--runpythontest", test_filenames_str, "-logfile", f"@log@/{log_name}", @@ -846,8 +849,7 @@ class EditorTestSuite(): # Scrap the output to attempt to find out which tests failed. # This function should always populate the result list, if it didn't find it, it will have "Unknown" type of result results = self._get_results_using_output(test_spec_list, output, editor_log_content) - assert len(results) == len(test_spec_list), "bug in _get_results_using_output(), the number of results" \ - "don't match the tests ran" + assert len(results) == len(test_spec_list), "bug in _get_results_using_output(), the number of results don't match the tests ran" # If the editor crashed, find out in which test it happened and update the results has_crashed = return_code != EditorTestSuite._TEST_FAIL_RETCODE @@ -866,9 +868,9 @@ class EditorTestSuite(): else: # If there are remaning "Unknown" results, these couldn't execute because of the crash, # update with info about the offender - results[test_spec_name].extra_info = f"This test has unknown result, test " \ - f"'{crashed_result.test_spec.__name__}' crashed " \ - f"before this test could be executed" + results[test_spec_name].extra_info = f"This test has unknown result," \ + f"test '{crashed_result.test_spec.__name__}'" \ + f"crashed before this test could be executed" # if all the tests ran, the one that has caused the crash is the last test if not crashed_result: crash_error = editor_utils.retrieve_crash_output(run_id, workspace, self._TIMEOUT_CRASH_LOG) @@ -882,8 +884,7 @@ class EditorTestSuite(): # The editor timed out when running the tests, get the data from the output to find out which ones ran results = self._get_results_using_output(test_spec_list, output, editor_log_content) - assert len(results) == len(test_spec_list), "bug in _get_results_using_output(), the number of results " \ - "don't match the tests ran" + assert len(results) == len(test_spec_list), "bug in _get_results_using_output(), the number of results don't match the tests ran" # Similar logic here as crashes, the first test that has no result is the one that timed out timed_out_result = None for test_spec_name, result in results.items(): @@ -929,7 +930,7 @@ class EditorTestSuite(): self._report_result(test_name, test_result) def _run_batched_tests(self, request, workspace, editor, editor_test_data, test_spec_list, extra_cmdline_args=[]): - # type (Request, AbstractWorkspace, Editor, TestData, list[EditorSharedTest], list[str]) -> None + # type (Request, AbstractWorkspace, Editor, TestData, List[EditorSharedTest], List[str]) -> None """ Runs a batch of tests in one single editor with the given spec list (one editor, multiple tests) :request: The Pytest Request @@ -950,7 +951,7 @@ class EditorTestSuite(): editor_test_data.results.update(results) def _run_parallel_tests(self, request, workspace, editor, editor_test_data, test_spec_list, extra_cmdline_args=[]): - # type(Request, AbstractWorkspace, Editor, TestData, list[EditorSharedTest], list[str]) -> None + # type(Request, AbstractWorkspace, Editor, TestData, List[EditorSharedTest], List[str]) -> None """ Runs multiple editors with one test on each editor (multiple editor, one test each) :request: The Pytest Request @@ -999,7 +1000,7 @@ class EditorTestSuite(): def _run_parallel_batched_tests(self, request, workspace, editor, editor_test_data, test_spec_list, extra_cmdline_args=[]): - # type(Request, AbstractWorkspace, Editor, TestData, list[EditorSharedTest], list[str] -> None + # type(Request, AbstractWorkspace, Editor, TestData, List[EditorSharedTest], List[str] -> None """ Runs multiple editors with a batch of tests for each editor (multiple editor, multiple tests each) :request: The Pytest Request diff --git a/Tools/LyTestTools/ly_test_tools/o3de/editor_test_utils.py b/Tools/LyTestTools/ly_test_tools/o3de/editor_test_utils.py index 838f929cfa..6e319bb860 100644 --- a/Tools/LyTestTools/ly_test_tools/o3de/editor_test_utils.py +++ b/Tools/LyTestTools/ly_test_tools/o3de/editor_test_utils.py @@ -4,7 +4,7 @@ For complete copyright and license terms please see the LICENSE at the root of t SPDX-License-Identifier: Apache-2.0 OR MIT -Utility functions for the editor_test module +Utility functions mostly for the editor_test module. They can also be used for assisting Editor tests. """ import os @@ -19,7 +19,8 @@ logger = logging.getLogger(__name__) def kill_all_ly_processes(include_asset_processor=True): # type (bool) -> None """ - Kills all common O3DE processes such as the Editor, Game Launchers, and Asset Processor. + Kills all common O3DE processes such as the Editor, Game Launchers, and optionally Asset Processor. Defaults to + killing the Asset Processor. :param include_asset_processor: Boolean flag whether or not to kill the AP :return: None """ @@ -65,7 +66,7 @@ def retrieve_log_path(run_id, workspace): """ return os.path.join(workspace.paths.project(), "user", f"log_test_{run_id}") -def retrieve_crash_output(run_id, workspace, timeout): +def retrieve_crash_output(run_id, workspace, timeout=10): # type (int, ly_test_tools._internal.managers.workspace.AbstractWorkspaceManager, float) -> str """ returns the crash output string for the given test run. @@ -138,7 +139,7 @@ def retrieve_editor_log_content(run_id, log_name, workspace, timeout=10): return editor_info def retrieve_last_run_test_index_from_output(test_spec_list, output): - # type (list, str) -> int + # type (List[EditorTestBase], str) -> int """ Finds out what was the last test that was run by inspecting the input. This is used for determining what was the batched test has crashed the editor diff --git a/Tools/LyTestTools/tests/unit/test_editor_test_utils.py b/Tools/LyTestTools/tests/unit/test_editor_test_utils.py index 134d1e1ad8..b096f6d6b0 100644 --- a/Tools/LyTestTools/tests/unit/test_editor_test_utils.py +++ b/Tools/LyTestTools/tests/unit/test_editor_test_utils.py @@ -16,18 +16,20 @@ pytestmark = pytest.mark.SUITE_smoke class TestEditorTestUtils(unittest.TestCase): @mock.patch('ly_test_tools.environment.process_utils.kill_processes_named') - def test_KillAllLyProcesses_IncludeAP_CallsCorrectly(self, under_test): + def test_KillAllLyProcesses_IncludeAP_CallsCorrectly(self, mock_kill_processes_named): process_list = ['Editor', 'Profiler', 'RemoteConsole', 'AssetProcessor', 'AssetProcessorBatch', 'AssetBuilder'] editor_test_utils.kill_all_ly_processes(include_asset_processor=True) - under_test.assert_called_once_with(process_list, ignore_extensions=True) + mock_kill_processes_named.assert_called_once_with(process_list, ignore_extensions=True) @mock.patch('ly_test_tools.environment.process_utils.kill_processes_named') - def test_KillAllLyProcesses_NotIncludeAP_CallsCorrectly(self, under_test): + def test_KillAllLyProcesses_NotIncludeAP_CallsCorrectly(self, mock_kill_processes_named): process_list = ['Editor', 'Profiler', 'RemoteConsole'] + ap_process_list = ['AssetProcessor', 'AssetProcessorBatch', 'AssetBuilder'] editor_test_utils.kill_all_ly_processes(include_asset_processor=False) - under_test.assert_called_once_with(process_list, ignore_extensions=True) + mock_kill_processes_named.assert_called_once() + assert ap_process_list not in mock_kill_processes_named.call_args[0] def test_GetTestcaseModuleFilepath_NoExtension_ReturnsPYExtension(self): mock_module = mock.MagicMock() @@ -81,30 +83,30 @@ class TestEditorTestUtils(unittest.TestCase): @mock.patch('ly_test_tools.o3de.editor_test_utils.retrieve_log_path') @mock.patch('os.path.exists') def test_CycleCrashReport_LogExists_NamedCorrectly(self, mock_exists, mock_retrieve_log_path, mock_getmtime, - under_test): + mock_rename): mock_exists.side_effect = [True, False] mock_retrieve_log_path.return_value = 'mock_log_path' mock_workspace = mock.MagicMock() mock_getmtime.return_value = 1 editor_test_utils.cycle_crash_report(0, mock_workspace) - under_test.assert_called_once_with(os.path.join('mock_log_path', 'error.log'), - os.path.join('mock_log_path', 'error_1969_12_31_16_00_01.log')) + mock_rename.assert_called_once_with(os.path.join('mock_log_path', 'error.log'), + os.path.join('mock_log_path', 'error_1969_12_31_16_00_01.log')) @mock.patch('os.rename') @mock.patch('os.path.getmtime') @mock.patch('ly_test_tools.o3de.editor_test_utils.retrieve_log_path') @mock.patch('os.path.exists') def test_CycleCrashReport_DmpExists_NamedCorrectly(self, mock_exists, mock_retrieve_log_path, mock_getmtime, - under_test): + mock_rename): mock_exists.side_effect = [False, True] mock_retrieve_log_path.return_value = 'mock_log_path' mock_workspace = mock.MagicMock() mock_getmtime.return_value = 1 editor_test_utils.cycle_crash_report(0, mock_workspace) - under_test.assert_called_once_with(os.path.join('mock_log_path', 'error.dmp'), - os.path.join('mock_log_path', 'error_1969_12_31_16_00_01.dmp')) + mock_rename.assert_called_once_with(os.path.join('mock_log_path', 'error.dmp'), + os.path.join('mock_log_path', 'error_1969_12_31_16_00_01.dmp')) @mock.patch('ly_test_tools.o3de.editor_test_utils.retrieve_log_path') @mock.patch('ly_test_tools.environment.waiter.wait_for', mock.MagicMock()) @@ -123,9 +125,9 @@ class TestEditorTestUtils(unittest.TestCase): mock_retrieve_log_path.return_value = 'mock_log_path' mock_logname = 'mock_log.log' mock_workspace = mock.MagicMock() - expected = f"-- Error reading editor.log: [Errno 2] No such file or directory: 'mock_log_path\\\\mock_log.log' --" + expected = f"-- Error reading editor.log" - assert expected == editor_test_utils.retrieve_editor_log_content(0, mock_logname, mock_workspace) + assert expected in editor_test_utils.retrieve_editor_log_content(0, mock_logname, mock_workspace) def test_RetrieveLastRunTestIndexFromOutput_SecondTestFailed_Returns0(self): mock_test = mock.MagicMock() diff --git a/Tools/LyTestTools/tests/unit/test_fixtures.py b/Tools/LyTestTools/tests/unit/test_fixtures.py index 911e6fef91..32dc204bd2 100755 --- a/Tools/LyTestTools/tests/unit/test_fixtures.py +++ b/Tools/LyTestTools/tests/unit/test_fixtures.py @@ -369,14 +369,3 @@ class TestFixtures(object): mock_request.addfinalizer.call_args[0][0]() mock_stop.assert_called_once() - - @mock.patch('inspect.isclass', mock.MagicMock(return_value=True)) - def test_PytestPycollectMakeitem_ValidArgs_CallsCorrectly(self): - mock_collector = mock.MagicMock() - mock_name = mock.MagicMock() - mock_obj = mock.MagicMock() - mock_base = mock.MagicMock() - mock_obj.__bases__ = [mock_base] - - test_tools_fixtures.pytest_pycollect_makeitem(mock_collector, mock_name, mock_obj) - mock_base.pytest_custom_makeitem.assert_called_once_with(mock_collector, mock_name, mock_obj) diff --git a/Tools/LyTestTools/tests/unit/test_o3de_editor_test.py b/Tools/LyTestTools/tests/unit/test_o3de_editor_test.py index 05d1ebc305..2ba5a8ed6c 100644 --- a/Tools/LyTestTools/tests/unit/test_o3de_editor_test.py +++ b/Tools/LyTestTools/tests/unit/test_o3de_editor_test.py @@ -68,14 +68,9 @@ class TestPass(unittest.TestCase): mock_test_spec = mock.MagicMock() mock_output = 'mock_output' mock_editor_log = mock.MagicMock() - expected = f"Test Passed\n"\ - f"------------\n"\ - f"| Output |\n"\ - f"------------\n"\ - f"{mock_output}\n" mock_pass = editor_test.Result.Pass.create(mock_test_spec, mock_output, mock_editor_log) - assert str(mock_pass) == expected + assert mock_output in str(mock_pass) class TestFail(unittest.TestCase): @@ -93,18 +88,10 @@ class TestFail(unittest.TestCase): mock_test_spec = mock.MagicMock() mock_output = 'mock_output' mock_editor_log = 'mock_editor_log' - expected = f"Test FAILED\n"\ - f"------------\n"\ - f"| Output |\n"\ - f"------------\n"\ - f"{mock_output}\n"\ - f"--------------\n"\ - f"| Editor log |\n"\ - f"--------------\n"\ - f"{mock_editor_log}\n" mock_pass = editor_test.Result.Fail.create(mock_test_spec, mock_output, mock_editor_log) - assert str(mock_pass) == expected + assert mock_output in str(mock_pass) + assert mock_editor_log in str(mock_pass) class TestCrash(unittest.TestCase): @@ -129,23 +116,12 @@ class TestCrash(unittest.TestCase): mock_editor_log = 'mock_editor_log' mock_return_code = 0 mock_stacktrace = 'mock stacktrace' - expected = f"Test CRASHED, return code {hex(mock_return_code)}\n"\ - f"---------------\n"\ - f"| Stacktrace |\n"\ - f"---------------\n"\ - f"{mock_stacktrace}"\ - f"------------\n" \ - f"| Output |\n" \ - f"------------\n" \ - f"{mock_output}\n" \ - f"--------------\n" \ - f"| Editor log |\n" \ - f"--------------\n" \ - f"{mock_editor_log}\n" mock_pass = editor_test.Result.Crash.create(mock_test_spec, mock_output, mock_return_code, mock_stacktrace, mock_editor_log) - assert str(mock_pass) == expected + assert mock_stacktrace in str(mock_pass) + assert mock_output in str(mock_pass) + assert mock_editor_log in str(mock_pass) def test_Str_MissingStackTrace_ReturnsCorrectly(self): mock_test_spec = mock.MagicMock() @@ -153,23 +129,10 @@ class TestCrash(unittest.TestCase): mock_editor_log = 'mock_editor_log' mock_return_code = 0 mock_stacktrace = None - expected = f"Test CRASHED, return code {hex(mock_return_code)}\n"\ - f"---------------\n"\ - f"| Stacktrace |\n"\ - f"---------------\n"\ - f"-- No stacktrace data found --\n"\ - f"------------\n" \ - f"| Output |\n" \ - f"------------\n" \ - f"{mock_output}\n" \ - f"--------------\n" \ - f"| Editor log |\n" \ - f"--------------\n" \ - f"{mock_editor_log}\n" - mock_pass = editor_test.Result.Crash.create(mock_test_spec, mock_output, mock_return_code, mock_stacktrace, mock_editor_log) - assert str(mock_pass) == expected + assert mock_output in str(mock_pass) + assert mock_editor_log in str(mock_pass) class Timeout(unittest.TestCase): @@ -190,18 +153,10 @@ class Timeout(unittest.TestCase): mock_output = 'mock_output' mock_editor_log = 'mock_editor_log' mock_timeout = 0 - expected = f"Test TIMED OUT after {mock_timeout} seconds\n"\ - f"------------\n" \ - f"| Output |\n" \ - f"------------\n" \ - f"{mock_output}\n" \ - f"--------------\n" \ - f"| Editor log |\n" \ - f"--------------\n" \ - f"{mock_editor_log}\n" mock_pass = editor_test.Result.Timeout.create(mock_test_spec, mock_output, mock_timeout, mock_editor_log) - assert str(mock_pass) == expected + assert mock_output in str(mock_pass) + assert mock_editor_log in str(mock_pass) class Unknown(unittest.TestCase): @@ -222,18 +177,10 @@ class Unknown(unittest.TestCase): mock_output = 'mock_output' mock_editor_log = 'mock_editor_log' mock_extra_info = 'mock extra info' - expected = f"Unknown test result, possible cause: {mock_extra_info}\n"\ - f"------------\n" \ - f"| Output |\n" \ - f"------------\n" \ - f"{mock_output}\n" \ - f"--------------\n" \ - f"| Editor log |\n" \ - f"--------------\n" \ - f"{mock_editor_log}\n" mock_pass = editor_test.Result.Unknown.create(mock_test_spec, mock_output, mock_extra_info, mock_editor_log) - assert str(mock_pass) == expected + assert mock_output in str(mock_pass) + assert mock_editor_log in str(mock_pass) class TestEditorTestSuite(unittest.TestCase): @@ -274,12 +221,31 @@ class TestEditorTestSuite(unittest.TestCase): mock.MagicMock()) assert isinstance(mock_test_class, editor_test.EditorTestSuite.EditorTestClass) - def test_PytestCustomModifyItems(self): - pass + @mock.patch('ly_test_tools.o3de.editor_test.EditorTestSuite.filter_session_shared_tests') + def test_PytestCustomModifyItems_FunctionsMatch_AddsRunners(self, mock_filter_tests): + class MockTestSuite(editor_test.EditorTestSuite): + pass + mock_func_1 = mock.MagicMock() + mock_test = mock.MagicMock() + runner_1 = editor_test.EditorTestSuite.Runner('mock_runner_1', mock_func_1, [mock_test]) + mock_run_pytest_func = mock.MagicMock() + runner_1.run_pytestfunc = mock_run_pytest_func + mock_result_pytestfuncs = [mock.MagicMock()] + runner_1.result_pytestfuncs = mock_result_pytestfuncs + mock_items = [] + mock_items.extend(mock_result_pytestfuncs) + + MockTestSuite._runners = [runner_1] + mock_test_1 = mock.MagicMock() + mock_test_2 = mock.MagicMock() + mock_filter_tests.return_value = [mock_test_1, mock_test_2] + + MockTestSuite.pytest_custom_modify_items(mock.MagicMock(), mock_items, mock.MagicMock()) + assert mock_items == [mock_run_pytest_func, mock_result_pytestfuncs[0]] def test_GetSingleTests_NoSingleTests_EmptyList(self): class MockTestSuite(editor_test.EditorTestSuite): - pass + pass mock_test_suite = MockTestSuite() tests = mock_test_suite.get_single_tests() assert len(tests) == 0 @@ -291,7 +257,8 @@ class TestEditorTestSuite(unittest.TestCase): mock_test_suite = MockTestSuite() tests = mock_test_suite.get_single_tests() assert len(tests) == 1 - + assert tests[0].__name__ == "MockSingleTest" + assert issubclass(tests[0], editor_test.EditorSingleTest) def test_GetSingleTests_AllTests_ReturnsOnlySingles(self): class MockTestSuite(editor_test.EditorTestSuite): @@ -304,6 +271,8 @@ class TestEditorTestSuite(unittest.TestCase): mock_test_suite = MockTestSuite() tests = mock_test_suite.get_single_tests() assert len(tests) == 2 + for test in tests: + assert issubclass(test, editor_test.EditorSingleTest) def test_GetSharedTests_NoSharedTests_EmptyList(self): class MockTestSuite(editor_test.EditorTestSuite): @@ -319,6 +288,8 @@ class TestEditorTestSuite(unittest.TestCase): mock_test_suite = MockTestSuite() tests = mock_test_suite.get_shared_tests() assert len(tests) == 1 + assert tests[0].__name__ == 'MockSharedTest' + assert issubclass(tests[0], editor_test.EditorSharedTest) def test_GetSharedTests_AllTests_ReturnsOnlyShared(self): class MockTestSuite(editor_test.EditorTestSuite): @@ -331,6 +302,8 @@ class TestEditorTestSuite(unittest.TestCase): mock_test_suite = MockTestSuite() tests = mock_test_suite.get_shared_tests() assert len(tests) == 2 + for test in tests: + assert issubclass(test, editor_test.EditorSharedTest) @mock.patch('ly_test_tools.o3de.editor_test.EditorTestSuite.filter_session_shared_tests') @mock.patch('ly_test_tools.o3de.editor_test.EditorTestSuite.get_shared_tests') @@ -343,8 +316,6 @@ class TestEditorTestSuite(unittest.TestCase): def test_FilterSessionSharedTests_OneSharedTest_ReturnsOne(self): def mock_test(): pass - mock_session_items = mock.MagicMock() - mock_shared_tests = mock.MagicMock() mock_test.originalname = 'mock_test' mock_test.__name__ = mock_test.originalname mock_session_items = [mock_test] @@ -361,8 +332,6 @@ class TestEditorTestSuite(unittest.TestCase): pass def mock_test_3(): pass - mock_session_items = mock.MagicMock() - mock_shared_tests = mock.MagicMock() mock_test.originalname = 'mock_test' mock_test.__name__ = mock_test.originalname mock_test_2.originalname = 'mock_test_2' @@ -379,8 +348,6 @@ class TestEditorTestSuite(unittest.TestCase): def test_FilterSessionSharedTests_SkippingPytestRaises_SkipsAddingTest(self): def mock_test(): pass - mock_session_items = mock.MagicMock() - mock_shared_tests = mock.MagicMock() mock_test.originalname = 'mock_test' mock_test.__name__ = mock_test.originalname mock_session_items = [mock_test] @@ -495,12 +462,13 @@ class TestUtils(unittest.TestCase): mock_output = 'JSON_START(' \ '{"name": "mock_module_name", "output": "mock_std_out", "success": "mock_success_data"}' \ ')JSON_END' - mock_editor_log = 'JSON_START(' \ - ')JSON_END' + mock_pass = mock.MagicMock() + mock_create.return_value = mock_pass - results = mock_test_suite._get_results_using_output(mock_test_list, mock_output, mock_editor_log) + results = mock_test_suite._get_results_using_output(mock_test_list, mock_output, '') assert mock_create.called assert len(results) == 1 + assert results[mock_test.__name__] == mock_pass @mock.patch('ly_test_tools.o3de.editor_test.Result.Fail.create') @mock.patch('ly_test_tools.o3de.editor_test_utils.get_module_filename') @@ -513,12 +481,13 @@ class TestUtils(unittest.TestCase): mock_output = 'JSON_START(' \ '{"name": "mock_module_name", "output": "mock_std_out", "failed": "mock_fail_data"}' \ ')JSON_END' - mock_editor_log = 'JSON_START(' \ - ')JSON_END' + mock_fail = mock.MagicMock() + mock_create.return_value = mock_fail - results = mock_test_suite._get_results_using_output(mock_test_list, mock_output, mock_editor_log) + results = mock_test_suite._get_results_using_output(mock_test_list, mock_output, '') assert mock_create.called assert len(results) == 1 + assert results[mock_test.__name__] == mock_fail @mock.patch('ly_test_tools.o3de.editor_test.Result.Unknown.create') @mock.patch('ly_test_tools.o3de.editor_test_utils.get_module_filename') @@ -531,12 +500,13 @@ class TestUtils(unittest.TestCase): mock_output = 'JSON_START(' \ '{"name": "mock_module_name", "output": "mock_std_out", "failed": "mock_fail_data"}' \ ')JSON_END' - mock_editor_log = 'JSON_START(' \ - ')JSON_END' + mock_unknown = mock.MagicMock() + mock_create.return_value = mock_unknown - results = mock_test_suite._get_results_using_output(mock_test_list, mock_output, mock_editor_log) + results = mock_test_suite._get_results_using_output(mock_test_list, mock_output, '') assert mock_create.called assert len(results) == 1 + assert results[mock_test.__name__] == mock_unknown @mock.patch('ly_test_tools.o3de.editor_test.Result.Pass.create') @mock.patch('ly_test_tools.o3de.editor_test.Result.Fail.create') @@ -567,7 +537,13 @@ class TestUtils(unittest.TestCase): ')JSON_END' \ 'JSON_START(' \ '{"name": "mock_module_name_fail"}' \ - ')JSON_END' \ + ')JSON_END' + mock_unknown = mock.MagicMock() + mock_pass = mock.MagicMock() + mock_fail = mock.MagicMock() + mock_create_unknown.return_value = mock_unknown + mock_create_pass.return_value = mock_pass + mock_create_fail.return_value = mock_fail results = mock_test_suite._get_results_using_output(mock_test_list, mock_output, mock_editor_log) mock_create_pass.assert_called_with( @@ -577,6 +553,9 @@ class TestUtils(unittest.TestCase): mock_create_unknown.assert_called_with( mock_test_unknown, mock_output, "Couldn't find any test run information on stdout", mock_editor_log) assert len(results) == 3 + assert results[mock_test_pass.__name__] == mock_pass + assert results[mock_test_fail.__name__] == mock_fail + assert results[mock_test_unknown.__name__] == mock_unknown @mock.patch('builtins.print') def test_ReportResult_TestPassed_ReportsCorrectly(self, mock_print): diff --git a/Tools/LyTestTools/tests/unit/test_pytest_plugin_editor_test.py b/Tools/LyTestTools/tests/unit/test_pytest_plugin_editor_test.py index 968c07aca6..10c779e546 100644 --- a/Tools/LyTestTools/tests/unit/test_pytest_plugin_editor_test.py +++ b/Tools/LyTestTools/tests/unit/test_pytest_plugin_editor_test.py @@ -38,4 +38,4 @@ class TestEditorTest(unittest.TestCase): generator = editor_test.pytest_collection_modifyitems(mock_session, mock_items, mock_config) for x in generator: pass - assert mock_class.pytest_custom_modify_items.call_count == 1 \ No newline at end of file + assert mock_class.pytest_custom_modify_items.call_count == 1