Renamed ctest_pytest.ini to pytest.ini so it is used by default, added TestSuite_ as collection file (#4822)

* Fixed warnings of unused marks, renamed ctest_pytest.ini to pytest.ini to better consistency on runs

* Fixed some test suites to run propertly

* Fix missing arguments

* Fixed missing cmakelists and renamed missing file

* Temp disable editor_testing_tests as timeout in jenkins

AutomatedTesting/Gem/PythonTests/Atom/TestSuite_Sandbox.py

@@ -14,10 +14,6 @@ import editor_python_test_tools.hydra_test_utils as hydra
 logger = logging.getLogger(__name__)
 TEST_DIRECTORY = os.path.join(os.path.dirname(__file__), "tests")
 
-
-@pytest.mark.parametrize("project", ["AutomatedTesting"])
-@pytest.mark.parametrize("launcher_platform", ['windows_editor'])
-@pytest.mark.parametrize("level", ["auto_test"])
 class TestAtomEditorComponentsSandbox(object):
 
     # It requires at least one test

AutomatedTesting/Gem/PythonTests/CMakeLists.txt

@@ -58,3 +58,6 @@ add_subdirectory(smoke)
 
 ## AWS ##
 add_subdirectory(AWS)
+
+## Integration tests for editor testing framework ##
+add_subdirectory(editor_test_testing)

AutomatedTesting/Gem/PythonTests/Physics/TestSuite_Utils.py

@@ -9,20 +9,17 @@ import pytest
 import sys
 
 import ly_test_tools.environment.file_system as fs
-from .FileManagement import FileManagement as fm
+from .utils.FileManagement import FileManagement as fm
 
 sys.path.append(os.path.dirname(os.path.abspath(__file__)) + '/../automatedtesting_shared')
 
-from .base import TestAutomationBase
+from base import TestAutomationBase
 
-
-@pytest.mark.parametrize("platform", ["win_x64_vs2017"])
-@pytest.mark.parametrize("configuration", ["profile"])
-@pytest.mark.parametrize("spec", ["all"])
+@pytest.mark.parametrize("launcher_platform", ['windows_editor'])
 @pytest.mark.parametrize("project", ["AutomatedTesting"])
 class TestUtils(TestAutomationBase):
     @fm.file_revert("UtilTest_Physmaterial_Editor_TestLibrary.physmaterial", r"AutomatedTesting\Levels\Physics\Physmaterial_Editor_Test")
-    def test_physmaterial_editor(self, request, workspace, editor):
+    def test_physmaterial_editor(self, request, workspace, launcher_platform, editor):
         """
         Tests functionality of physmaterial editing utility
         :param workspace: Fixture containing platform and project detail
@@ -35,11 +32,11 @@ class TestUtils(TestAutomationBase):
         unexpected_lines = ["Assert"]
         self._run_test(request, workspace, editor, physmaterial_editor_test_module, expected_lines, unexpected_lines)
     
-    def test_UtilTest_Tracer_PicksErrorsAndWarnings(self, request, workspace, editor):
+    def test_UtilTest_Tracer_PicksErrorsAndWarnings(self, request, workspace, launcher_platform, editor):
         from .utils import UtilTest_Tracer_PicksErrorsAndWarnings as testcase_module
         self._run_test(request, workspace, editor, testcase_module, [], [])
 
-    def test_FileManagement_FindingFiles(self, workspace):
+    def test_FileManagement_FindingFiles(self, workspace, launcher_platform):
         """
         Tests the functionality of "searching for files" with FileManagement._find_files()
         :param workspace: ly_test_tools workspace fixture
@@ -110,7 +107,7 @@ class TestUtils(TestAutomationBase):
             find_me_too_path, found_me["FindMeToo.txt"]
         )
 
-    def test_FileManagement_FileBackup(self, workspace):
+    def test_FileManagement_FileBackup(self, workspace, launcher_platform):
         """
         Tests the functionality of the file back up system via the FileManagement class
         :param workspace: ly_test_tools workspace fixture
@@ -167,7 +164,7 @@ class TestUtils(TestAutomationBase):
         del file_map[target_file_path]
         fm._save_file_map(file_map)
 
-    def test_FileManagement_FileRestoration(self, workspace):
+    def test_FileManagement_FileRestoration(self, workspace, launcher_platform):
         """
         Tests the restore file system via the FileManagement class
         :param workspace: ly_test_tools workspace fixture
@@ -261,7 +258,7 @@ class TestUtils(TestAutomationBase):
         ["FindMe.txt", "FindMeToo.txt"], parent_path=r"AutomatedTesting\levels\Utils\Managed_files", search_subdirs=True
     )
     @fm.file_override("default.physxconfiguration", "UtilTest_PhysxConfig_Override.physxconfiguration")
-    def test_UtilTest_Managed_Files(self, request, workspace, editor):
+    def test_UtilTest_Managed_Files(self, request, workspace, editor, launcher_platform):
         from .utils import UtilTest_Managed_Files as test_module
 
         expected_lines = []

AutomatedTesting/Gem/PythonTests/editor_test_testing/CMakeLists.txt

@@ -0,0 +1,24 @@
+#
+# Copyright (c) Contributors to the Open 3D Engine Project.
+# For complete copyright and license terms please see the LICENSE at the root of this distribution.
+#
+# SPDX-License-Identifier: Apache-2.0 OR MIT
+#
+#
+
+# This timeouts on jenkins, investigation is needed. Commment for now
+# 
+#if(PAL_TRAIT_BUILD_TESTS_SUPPORTED AND PAL_TRAIT_BUILD_HOST_TOOLS)
+#    ly_add_pytest(
+#        NAME AutomatedTesting::EditorTestTesting
+#        TEST_SUITE main
+#        TEST_SERIAL
+#        PATH ${CMAKE_CURRENT_LIST_DIR}/TestSuite_Main.py
+#        RUNTIME_DEPENDENCIES
+#            Legacy::Editor
+#            AZ::AssetProcessor
+#            AutomatedTesting.Assets
+#        COMPONENT
+#            TestTools
+#    )
+#endif()

AutomatedTesting/Gem/PythonTests/editor_test_testing/TestSuite_Main.py

@@ -35,10 +35,11 @@ class TestEditorTest:
     @classmethod
     def setup_class(cls):
         TestEditorTest.args = sys.argv.copy()
-        build_dir_arg_index = TestEditorTest.args.index("--build-directory")
-        if build_dir_arg_index < 0:
-            print("Error: Must pass --build-directory argument in order to run this test")
-            sys.exit(-2)
+        build_dir_arg_index = -1
+        try:
+            build_dir_arg_index = TestEditorTest.args.index("--build-directory")
+        except ValueError as ex:
+            raise ValueError("Must pass --build-directory argument in order to run this test")
 
         TestEditorTest.args[build_dir_arg_index+1] = os.path.abspath(TestEditorTest.args[build_dir_arg_index+1])
         TestEditorTest.args.append("-s")

AutomatedTesting/Gem/PythonTests/editor_test_testing/conftest.py

@@ -1,8 +0,0 @@
-"""
-Copyright (c) Contributors to the Open 3D Engine Project.
-For complete copyright and license terms please see the LICENSE at the root of this distribution.
-
-SPDX-License-Identifier: Apache-2.0 OR MIT
-"""
-
-pytest_plugins = ["pytester"]