From ea442b80d1e8725269dc41bb039c0ad8dbb6a676 Mon Sep 17 00:00:00 2001 From: AMZN-AlexOteiza <82234181+AMZN-AlexOteiza@users.noreply.github.com> Date: Thu, 21 Oct 2021 22:03:18 +0100 Subject: [PATCH] Renamed ctest_pytest.ini to pytest.ini so it is used by default, added TestSuite_ as collection file (#4822) * Fixed warnings of unused marks, renamed ctest_pytest.ini to pytest.ini to better consistency on runs * Fixed some test suites to run propertly * Fix missing arguments * Fixed missing cmakelists and renamed missing file * Temp disable editor_testing_tests as timeout in jenkins --- .../Gem/PythonTests/Atom/TestSuite_Sandbox.py | 4 ---- .../Gem/PythonTests/CMakeLists.txt | 3 +++ .../PythonTests/Physics/TestSuite_Utils.py | 21 +++++++--------- .../editor_test_testing/CMakeLists.txt | 24 +++++++++++++++++++ .../editor_test_testing/TestSuite_Main.py | 9 +++---- .../editor_test_testing/conftest.py | 8 ------- Tools/LyTestTools/setup.py | 3 ++- cmake/LYTestWrappers.cmake | 2 +- cmake/Platform/Common/Install_common.cmake | 2 +- ctest_pytest.ini => pytest.ini | 3 ++- 10 files changed, 47 insertions(+), 32 deletions(-) create mode 100644 AutomatedTesting/Gem/PythonTests/editor_test_testing/CMakeLists.txt delete mode 100644 AutomatedTesting/Gem/PythonTests/editor_test_testing/conftest.py rename ctest_pytest.ini => pytest.ini (91%) diff --git a/AutomatedTesting/Gem/PythonTests/Atom/TestSuite_Sandbox.py b/AutomatedTesting/Gem/PythonTests/Atom/TestSuite_Sandbox.py index 9bb7f9c50e..ad45e51080 100644 --- a/AutomatedTesting/Gem/PythonTests/Atom/TestSuite_Sandbox.py +++ b/AutomatedTesting/Gem/PythonTests/Atom/TestSuite_Sandbox.py @@ -14,10 +14,6 @@ import editor_python_test_tools.hydra_test_utils as hydra logger = logging.getLogger(__name__) TEST_DIRECTORY = os.path.join(os.path.dirname(__file__), "tests") - -@pytest.mark.parametrize("project", ["AutomatedTesting"]) -@pytest.mark.parametrize("launcher_platform", ['windows_editor']) -@pytest.mark.parametrize("level", ["auto_test"]) class TestAtomEditorComponentsSandbox(object): # It requires at least one test diff --git a/AutomatedTesting/Gem/PythonTests/CMakeLists.txt b/AutomatedTesting/Gem/PythonTests/CMakeLists.txt index 2ca7f7ea2b..fd3222ba83 100644 --- a/AutomatedTesting/Gem/PythonTests/CMakeLists.txt +++ b/AutomatedTesting/Gem/PythonTests/CMakeLists.txt @@ -58,3 +58,6 @@ add_subdirectory(smoke) ## AWS ## add_subdirectory(AWS) + +## Integration tests for editor testing framework ## +add_subdirectory(editor_test_testing) diff --git a/AutomatedTesting/Gem/PythonTests/Physics/TestSuite_Utils.py b/AutomatedTesting/Gem/PythonTests/Physics/TestSuite_Utils.py index a3fd5c741f..61c2816f50 100755 --- a/AutomatedTesting/Gem/PythonTests/Physics/TestSuite_Utils.py +++ b/AutomatedTesting/Gem/PythonTests/Physics/TestSuite_Utils.py @@ -9,20 +9,17 @@ import pytest import sys import ly_test_tools.environment.file_system as fs -from .FileManagement import FileManagement as fm +from .utils.FileManagement import FileManagement as fm sys.path.append(os.path.dirname(os.path.abspath(__file__)) + '/../automatedtesting_shared') -from .base import TestAutomationBase +from base import TestAutomationBase - -@pytest.mark.parametrize("platform", ["win_x64_vs2017"]) -@pytest.mark.parametrize("configuration", ["profile"]) -@pytest.mark.parametrize("spec", ["all"]) +@pytest.mark.parametrize("launcher_platform", ['windows_editor']) @pytest.mark.parametrize("project", ["AutomatedTesting"]) class TestUtils(TestAutomationBase): @fm.file_revert("UtilTest_Physmaterial_Editor_TestLibrary.physmaterial", r"AutomatedTesting\Levels\Physics\Physmaterial_Editor_Test") - def test_physmaterial_editor(self, request, workspace, editor): + def test_physmaterial_editor(self, request, workspace, launcher_platform, editor): """ Tests functionality of physmaterial editing utility :param workspace: Fixture containing platform and project detail @@ -35,11 +32,11 @@ class TestUtils(TestAutomationBase): unexpected_lines = ["Assert"] self._run_test(request, workspace, editor, physmaterial_editor_test_module, expected_lines, unexpected_lines) - def test_UtilTest_Tracer_PicksErrorsAndWarnings(self, request, workspace, editor): + def test_UtilTest_Tracer_PicksErrorsAndWarnings(self, request, workspace, launcher_platform, editor): from .utils import UtilTest_Tracer_PicksErrorsAndWarnings as testcase_module self._run_test(request, workspace, editor, testcase_module, [], []) - def test_FileManagement_FindingFiles(self, workspace): + def test_FileManagement_FindingFiles(self, workspace, launcher_platform): """ Tests the functionality of "searching for files" with FileManagement._find_files() :param workspace: ly_test_tools workspace fixture @@ -110,7 +107,7 @@ class TestUtils(TestAutomationBase): find_me_too_path, found_me["FindMeToo.txt"] ) - def test_FileManagement_FileBackup(self, workspace): + def test_FileManagement_FileBackup(self, workspace, launcher_platform): """ Tests the functionality of the file back up system via the FileManagement class :param workspace: ly_test_tools workspace fixture @@ -167,7 +164,7 @@ class TestUtils(TestAutomationBase): del file_map[target_file_path] fm._save_file_map(file_map) - def test_FileManagement_FileRestoration(self, workspace): + def test_FileManagement_FileRestoration(self, workspace, launcher_platform): """ Tests the restore file system via the FileManagement class :param workspace: ly_test_tools workspace fixture @@ -261,7 +258,7 @@ class TestUtils(TestAutomationBase): ["FindMe.txt", "FindMeToo.txt"], parent_path=r"AutomatedTesting\levels\Utils\Managed_files", search_subdirs=True ) @fm.file_override("default.physxconfiguration", "UtilTest_PhysxConfig_Override.physxconfiguration") - def test_UtilTest_Managed_Files(self, request, workspace, editor): + def test_UtilTest_Managed_Files(self, request, workspace, editor, launcher_platform): from .utils import UtilTest_Managed_Files as test_module expected_lines = [] diff --git a/AutomatedTesting/Gem/PythonTests/editor_test_testing/CMakeLists.txt b/AutomatedTesting/Gem/PythonTests/editor_test_testing/CMakeLists.txt new file mode 100644 index 0000000000..dbf26d66cd --- /dev/null +++ b/AutomatedTesting/Gem/PythonTests/editor_test_testing/CMakeLists.txt @@ -0,0 +1,24 @@ +# +# Copyright (c) Contributors to the Open 3D Engine Project. +# For complete copyright and license terms please see the LICENSE at the root of this distribution. +# +# SPDX-License-Identifier: Apache-2.0 OR MIT +# +# + +# This timeouts on jenkins, investigation is needed. Commment for now +# +#if(PAL_TRAIT_BUILD_TESTS_SUPPORTED AND PAL_TRAIT_BUILD_HOST_TOOLS) +# ly_add_pytest( +# NAME AutomatedTesting::EditorTestTesting +# TEST_SUITE main +# TEST_SERIAL +# PATH ${CMAKE_CURRENT_LIST_DIR}/TestSuite_Main.py +# RUNTIME_DEPENDENCIES +# Legacy::Editor +# AZ::AssetProcessor +# AutomatedTesting.Assets +# COMPONENT +# TestTools +# ) +#endif() diff --git a/AutomatedTesting/Gem/PythonTests/editor_test_testing/TestSuite_Main.py b/AutomatedTesting/Gem/PythonTests/editor_test_testing/TestSuite_Main.py index 7c7e063951..15ba6690a1 100644 --- a/AutomatedTesting/Gem/PythonTests/editor_test_testing/TestSuite_Main.py +++ b/AutomatedTesting/Gem/PythonTests/editor_test_testing/TestSuite_Main.py @@ -35,10 +35,11 @@ class TestEditorTest: @classmethod def setup_class(cls): TestEditorTest.args = sys.argv.copy() - build_dir_arg_index = TestEditorTest.args.index("--build-directory") - if build_dir_arg_index < 0: - print("Error: Must pass --build-directory argument in order to run this test") - sys.exit(-2) + build_dir_arg_index = -1 + try: + build_dir_arg_index = TestEditorTest.args.index("--build-directory") + except ValueError as ex: + raise ValueError("Must pass --build-directory argument in order to run this test") TestEditorTest.args[build_dir_arg_index+1] = os.path.abspath(TestEditorTest.args[build_dir_arg_index+1]) TestEditorTest.args.append("-s") diff --git a/AutomatedTesting/Gem/PythonTests/editor_test_testing/conftest.py b/AutomatedTesting/Gem/PythonTests/editor_test_testing/conftest.py deleted file mode 100644 index 1f49f7111b..0000000000 --- a/AutomatedTesting/Gem/PythonTests/editor_test_testing/conftest.py +++ /dev/null @@ -1,8 +0,0 @@ -""" -Copyright (c) Contributors to the Open 3D Engine Project. -For complete copyright and license terms please see the LICENSE at the root of this distribution. - -SPDX-License-Identifier: Apache-2.0 OR MIT -""" - -pytest_plugins = ["pytester"] diff --git a/Tools/LyTestTools/setup.py b/Tools/LyTestTools/setup.py index e23f914dab..a523cbc2dd 100755 --- a/Tools/LyTestTools/setup.py +++ b/Tools/LyTestTools/setup.py @@ -48,7 +48,8 @@ if __name__ == '__main__': 'ly_test_tools=ly_test_tools._internal.pytest_plugin.test_tools_fixtures', 'testrail_filter=ly_test_tools._internal.pytest_plugin.case_id', 'terminal_report=ly_test_tools._internal.pytest_plugin.terminal_report', - 'editor_test=ly_test_tools._internal.pytest_plugin.editor_test' + 'editor_test=ly_test_tools._internal.pytest_plugin.editor_test', + 'pytester=_pytest.pytester' ], }, ) diff --git a/cmake/LYTestWrappers.cmake b/cmake/LYTestWrappers.cmake index 87649c5be6..9b139645fb 100644 --- a/cmake/LYTestWrappers.cmake +++ b/cmake/LYTestWrappers.cmake @@ -17,7 +17,7 @@ set(LY_GOOGLETEST_EXTRA_PARAMS CACHE STRING "Allows injection of additional opti find_package(Python REQUIRED MODULE) -ly_set(LY_PYTEST_EXECUTABLE ${LY_PYTHON_CMD} -B -m pytest -v --tb=short --show-capture=log -c ${LY_ROOT_FOLDER}/ctest_pytest.ini --build-directory "${CMAKE_RUNTIME_OUTPUT_DIRECTORY}/$") +ly_set(LY_PYTEST_EXECUTABLE ${LY_PYTHON_CMD} -B -m pytest -v --tb=short --show-capture=log -c ${LY_ROOT_FOLDER}/pytest.ini --build-directory "${CMAKE_RUNTIME_OUTPUT_DIRECTORY}/$") ly_set(LY_TEST_GLOBAL_KNOWN_SUITE_NAMES "smoke" "main" "periodic" "benchmark" "sandbox" "awsi") ly_set(LY_TEST_GLOBAL_KNOWN_REQUIREMENTS "gpu") diff --git a/cmake/Platform/Common/Install_common.cmake b/cmake/Platform/Common/Install_common.cmake index c453fadd2e..44cdeb1994 100644 --- a/cmake/Platform/Common/Install_common.cmake +++ b/cmake/Platform/Common/Install_common.cmake @@ -364,7 +364,7 @@ function(ly_setup_o3de_install) # Misc install(FILES - ${LY_ROOT_FOLDER}/ctest_pytest.ini + ${LY_ROOT_FOLDER}/pytest.ini ${LY_ROOT_FOLDER}/LICENSE.txt ${LY_ROOT_FOLDER}/README.md DESTINATION . diff --git a/ctest_pytest.ini b/pytest.ini similarity index 91% rename from ctest_pytest.ini rename to pytest.ini index 93310a522d..65c93e0eb2 100644 --- a/ctest_pytest.ini +++ b/pytest.ini @@ -7,10 +7,11 @@ # [pytest] -python_files = 'test_*.py' , '*_test.py' , '*_tests.py' +python_files = 'test_*.py' , '*_test.py' , '*_tests.py', 'TestSuite_*.py' norecursedirs = Python/2.7.* Python/3.7.5 BinTemp Cache SDKs JenkinsScripts cmake junit_family=legacy log_format=%(asctime)s [%(levelname)8s] %(message)s (%(filename)s:%(lineno)s) +addopts='--tb=short' '--show-capture=log' # primary suite markers which should appear on every filterable test and be mutually exclusive: markers = SUITE_smoke: Tiny, quick tests of fundamental operation (tests with no suite marker will also execute here in CI)