Renamed ctest_pytest.ini to pytest.ini so it is used by default, added TestSuite_ as collection file (#4822)

* Fixed warnings of unused marks, renamed ctest_pytest.ini to pytest.ini to better consistency on runs

* Fixed some test suites to run propertly

* Fix missing arguments

* Fixed missing cmakelists and renamed missing file

* Temp disable editor_testing_tests as timeout in jenkins

AutomatedTesting/Gem/PythonTests/Atom/TestSuite_Sandbox.py

@@ -14,10 +14,6 @@ import editor_python_test_tools.hydra_test_utils as hydra
 logger = logging.getLogger(__name__)
 TEST_DIRECTORY = os.path.join(os.path.dirname(__file__), "tests")
 
-
-@pytest.mark.parametrize("project", ["AutomatedTesting"])
-@pytest.mark.parametrize("launcher_platform", ['windows_editor'])
-@pytest.mark.parametrize("level", ["auto_test"])
 class TestAtomEditorComponentsSandbox(object):
 
     # It requires at least one test

AutomatedTesting/Gem/PythonTests/CMakeLists.txt

@@ -58,3 +58,6 @@ add_subdirectory(smoke)
 
 ## AWS ##
 add_subdirectory(AWS)
+
+## Integration tests for editor testing framework ##
+add_subdirectory(editor_test_testing)

AutomatedTesting/Gem/PythonTests/Physics/TestSuite_Utils.py

@@ -9,20 +9,17 @@ import pytest
 import sys
 
 import ly_test_tools.environment.file_system as fs
-from .FileManagement import FileManagement as fm
+from .utils.FileManagement import FileManagement as fm
 
 sys.path.append(os.path.dirname(os.path.abspath(__file__)) + '/../automatedtesting_shared')
 
-from .base import TestAutomationBase
+from base import TestAutomationBase
 
-
+@pytest.mark.parametrize("launcher_platform", ['windows_editor'])
-@pytest.mark.parametrize("platform", ["win_x64_vs2017"])
-@pytest.mark.parametrize("configuration", ["profile"])
-@pytest.mark.parametrize("spec", ["all"])
 @pytest.mark.parametrize("project", ["AutomatedTesting"])
 class TestUtils(TestAutomationBase):
     @fm.file_revert("UtilTest_Physmaterial_Editor_TestLibrary.physmaterial", r"AutomatedTesting\Levels\Physics\Physmaterial_Editor_Test")
-    def test_physmaterial_editor(self, request, workspace, editor):
+    def test_physmaterial_editor(self, request, workspace, launcher_platform, editor):
         """
         Tests functionality of physmaterial editing utility
         :param workspace: Fixture containing platform and project detail
@@ -35,11 +32,11 @@ class TestUtils(TestAutomationBase):
         unexpected_lines = ["Assert"]
         self._run_test(request, workspace, editor, physmaterial_editor_test_module, expected_lines, unexpected_lines)
     
-    def test_UtilTest_Tracer_PicksErrorsAndWarnings(self, request, workspace, editor):
+    def test_UtilTest_Tracer_PicksErrorsAndWarnings(self, request, workspace, launcher_platform, editor):
         from .utils import UtilTest_Tracer_PicksErrorsAndWarnings as testcase_module
         self._run_test(request, workspace, editor, testcase_module, [], [])
 
-    def test_FileManagement_FindingFiles(self, workspace):
+    def test_FileManagement_FindingFiles(self, workspace, launcher_platform):
         """
         Tests the functionality of "searching for files" with FileManagement._find_files()
         :param workspace: ly_test_tools workspace fixture
@@ -110,7 +107,7 @@ class TestUtils(TestAutomationBase):
             find_me_too_path, found_me["FindMeToo.txt"]
         )
 
-    def test_FileManagement_FileBackup(self, workspace):
+    def test_FileManagement_FileBackup(self, workspace, launcher_platform):
         """
         Tests the functionality of the file back up system via the FileManagement class
         :param workspace: ly_test_tools workspace fixture
@@ -167,7 +164,7 @@ class TestUtils(TestAutomationBase):
         del file_map[target_file_path]
         fm._save_file_map(file_map)
 
-    def test_FileManagement_FileRestoration(self, workspace):
+    def test_FileManagement_FileRestoration(self, workspace, launcher_platform):
         """
         Tests the restore file system via the FileManagement class
         :param workspace: ly_test_tools workspace fixture
@@ -261,7 +258,7 @@ class TestUtils(TestAutomationBase):
         ["FindMe.txt", "FindMeToo.txt"], parent_path=r"AutomatedTesting\levels\Utils\Managed_files", search_subdirs=True
     )
     @fm.file_override("default.physxconfiguration", "UtilTest_PhysxConfig_Override.physxconfiguration")
-    def test_UtilTest_Managed_Files(self, request, workspace, editor):
+    def test_UtilTest_Managed_Files(self, request, workspace, editor, launcher_platform):
         from .utils import UtilTest_Managed_Files as test_module
 
         expected_lines = []

AutomatedTesting/Gem/PythonTests/editor_test_testing/CMakeLists.txt

@@ -0,0 +1,24 @@
+#
+# Copyright (c) Contributors to the Open 3D Engine Project.
+# For complete copyright and license terms please see the LICENSE at the root of this distribution.
+#
+# SPDX-License-Identifier: Apache-2.0 OR MIT
+#
+#
+
+# This timeouts on jenkins, investigation is needed. Commment for now
+# 
+#if(PAL_TRAIT_BUILD_TESTS_SUPPORTED AND PAL_TRAIT_BUILD_HOST_TOOLS)
+#    ly_add_pytest(
+#        NAME AutomatedTesting::EditorTestTesting
+#        TEST_SUITE main
+#        TEST_SERIAL
+#        PATH ${CMAKE_CURRENT_LIST_DIR}/TestSuite_Main.py
+#        RUNTIME_DEPENDENCIES
+#            Legacy::Editor
+#            AZ::AssetProcessor
+#            AutomatedTesting.Assets
+#        COMPONENT
+#            TestTools
+#    )
+#endif()

AutomatedTesting/Gem/PythonTests/editor_test_testing/TestSuite_Main.py

@@ -35,10 +35,11 @@ class TestEditorTest:
     @classmethod
     def setup_class(cls):
         TestEditorTest.args = sys.argv.copy()
+        build_dir_arg_index = -1
+        try:
             build_dir_arg_index = TestEditorTest.args.index("--build-directory")
-        if build_dir_arg_index < 0:
+        except ValueError as ex:
-            print("Error: Must pass --build-directory argument in order to run this test")
+            raise ValueError("Must pass --build-directory argument in order to run this test")
-            sys.exit(-2)
 
         TestEditorTest.args[build_dir_arg_index+1] = os.path.abspath(TestEditorTest.args[build_dir_arg_index+1])
         TestEditorTest.args.append("-s")

AutomatedTesting/Gem/PythonTests/editor_test_testing/conftest.py

@@ -1,8 +0,0 @@
-"""
-Copyright (c) Contributors to the Open 3D Engine Project.
-For complete copyright and license terms please see the LICENSE at the root of this distribution.
-
-SPDX-License-Identifier: Apache-2.0 OR MIT
-"""
-
-pytest_plugins = ["pytester"]

Tools/LyTestTools/setup.py

@@ -48,7 +48,8 @@ if __name__ == '__main__':
                 'ly_test_tools=ly_test_tools._internal.pytest_plugin.test_tools_fixtures',
                 'testrail_filter=ly_test_tools._internal.pytest_plugin.case_id',
                 'terminal_report=ly_test_tools._internal.pytest_plugin.terminal_report',
-                'editor_test=ly_test_tools._internal.pytest_plugin.editor_test'
+                'editor_test=ly_test_tools._internal.pytest_plugin.editor_test',
+                'pytester=_pytest.pytester'
             ],
         },
     )

cmake/LYTestWrappers.cmake

@@ -17,7 +17,7 @@ set(LY_GOOGLETEST_EXTRA_PARAMS CACHE STRING "Allows injection of additional opti
 
 find_package(Python REQUIRED MODULE)
 
-ly_set(LY_PYTEST_EXECUTABLE ${LY_PYTHON_CMD} -B -m pytest -v --tb=short --show-capture=log -c ${LY_ROOT_FOLDER}/ctest_pytest.ini --build-directory "${CMAKE_RUNTIME_OUTPUT_DIRECTORY}/$<CONFIG>")
+ly_set(LY_PYTEST_EXECUTABLE ${LY_PYTHON_CMD} -B -m pytest -v --tb=short --show-capture=log -c ${LY_ROOT_FOLDER}/pytest.ini --build-directory "${CMAKE_RUNTIME_OUTPUT_DIRECTORY}/$<CONFIG>")
 ly_set(LY_TEST_GLOBAL_KNOWN_SUITE_NAMES "smoke" "main" "periodic" "benchmark" "sandbox" "awsi")
 ly_set(LY_TEST_GLOBAL_KNOWN_REQUIREMENTS "gpu")
 

cmake/Platform/Common/Install_common.cmake

@@ -364,7 +364,7 @@ function(ly_setup_o3de_install)
 
     # Misc
     install(FILES
-        ${LY_ROOT_FOLDER}/ctest_pytest.ini
+        ${LY_ROOT_FOLDER}/pytest.ini
         ${LY_ROOT_FOLDER}/LICENSE.txt
         ${LY_ROOT_FOLDER}/README.md
         DESTINATION .

pytest.ini

@@ -7,10 +7,11 @@
 #
 
 [pytest]
-python_files = 'test_*.py' , '*_test.py' , '*_tests.py'
+python_files = 'test_*.py' , '*_test.py' , '*_tests.py', 'TestSuite_*.py'
 norecursedirs = Python/2.7.* Python/3.7.5 BinTemp Cache SDKs JenkinsScripts cmake
 junit_family=legacy
 log_format=%(asctime)s [%(levelname)8s] %(message)s (%(filename)s:%(lineno)s)
+addopts='--tb=short' '--show-capture=log'
 
 # primary suite markers which should appear on every filterable test and be mutually exclusive:
 markers = SUITE_smoke: Tiny, quick tests of fundamental operation (tests with no suite marker will also execute here in CI)