diff --git a/AutomatedTesting/Gem/PythonTests/physics/CMakeLists.txt b/AutomatedTesting/Gem/PythonTests/physics/CMakeLists.txt index eb37db1943..2557faee43 100644 --- a/AutomatedTesting/Gem/PythonTests/physics/CMakeLists.txt +++ b/AutomatedTesting/Gem/PythonTests/physics/CMakeLists.txt @@ -19,6 +19,19 @@ if(PAL_TRAIT_BUILD_TESTS_SUPPORTED AND PAL_TRAIT_BUILD_HOST_TOOLS) COMPONENT Physics ) + ly_add_pytest( + NAME AutomatedTesting::PhysicsTests_Main_Optimized + TEST_SUITE main + TEST_SERIAL + PATH ${CMAKE_CURRENT_LIST_DIR}/TestSuite_Main_Optimized.py + TIMEOUT 1500 + RUNTIME_DEPENDENCIES + Legacy::Editor + AZ::AssetProcessor + AutomatedTesting.Assets + COMPONENT + Physics + ) ly_add_pytest( NAME AutomatedTesting::PhysicsTests_Periodic TEST_SUITE periodic diff --git a/AutomatedTesting/Gem/PythonTests/physics/TestSuite_Main_Optimized.py b/AutomatedTesting/Gem/PythonTests/physics/TestSuite_Main_Optimized.py new file mode 100644 index 0000000000..75e226fba1 --- /dev/null +++ b/AutomatedTesting/Gem/PythonTests/physics/TestSuite_Main_Optimized.py @@ -0,0 +1,349 @@ +""" +Copyright (c) Contributors to the Open 3D Engine Project. +For complete copyright and license terms please see the LICENSE at the root of this distribution. + +SPDX-License-Identifier: Apache-2.0 OR MIT +""" + +import pytest +import os +import sys +import inspect + +from ly_test_tools import LAUNCHERS +from ly_test_tools.o3de.editor_test import EditorSingleTest, EditorSharedTest, EditorParallelTest, EditorTestSuite +from .FileManagement import FileManagement as fm + +# Custom test spec, it provides functionality to override files +class EditorSingleTest_WithFileOverrides(EditorSingleTest): + # Specify here what files to override, [(original, override), ...] + files_to_override = [()] + # Base directory of the files (Default path is {ProjectName}) + base_dir = None + # True will will search sub-directories for the files in base + search_subdirs = False + + @classmethod + def wrap_run(cls, instance, request, workspace, editor, editor_test_results, launcher_platform): + root_path = cls.base_dir + if root_path is not None: + root_path = os.path.join(workspace.paths.engine_root(), root_path) + else: + # Default to project folder + root_path = workspace.paths.project() + + # Try to locate both target and source files + original_file_list, override_file_list = zip(*cls.files_to_override) + try: + file_list = fm._find_files(original_file_list + override_file_list, root_path, cls.search_subdirs) + except RuntimeWarning as w: + assert False, ( + w.message + + " Please check use of search_subdirs; make sure you are using the correct parent directory." + ) + + for f in original_file_list: + fm._restore_file(f, file_list[f]) + fm._backup_file(f, file_list[f]) + + for original, override in cls.files_to_override: + fm._copy_file(override, file_list[override], original, file_list[override]) + + yield # Run Test + for f in original_file_list: + fm._restore_file(f, file_list[f]) + + +@pytest.mark.xfail(reason="Optimized tests are experimental, we will enable xfail and monitor them temporarly.") +@pytest.mark.SUITE_main +@pytest.mark.parametrize("launcher_platform", ['windows_editor']) +@pytest.mark.parametrize("project", ["AutomatedTesting"]) +class TestAutomation(EditorTestSuite): + + @staticmethod + def get_number_parallel_editors(): + return 16 + + ######################################### + # Non-atomic tests: These need to be run in a single editor because they have custom setup and teardown + class C4044459_Material_DynamicFriction(EditorSingleTest_WithFileOverrides): + from . import C4044459_Material_DynamicFriction as test_module + files_to_override = [ + ('physxsystemconfiguration.setreg', 'C4044459_Material_DynamicFriction.setreg_override') + ] + base_dir = "AutomatedTesting/Registry" + + class C4982593_PhysXCollider_CollisionLayerTest(EditorSingleTest_WithFileOverrides): + from . import C4982593_PhysXCollider_CollisionLayerTest as test_module + files_to_override = [ + ('physxsystemconfiguration.setreg', 'C4982593_PhysXCollider_CollisionLayer.setreg_override') + ] + base_dir = "AutomatedTesting/Registry" + ######################################### + + class C111111_RigidBody_EnablingGravityWorksUsingNotificationsPoC(EditorSharedTest): + from . import C111111_RigidBody_EnablingGravityWorksUsingNotificationsPoC as test_module + + class C5932041_PhysXForceRegion_LocalSpaceForceOnRigidBodies(EditorSharedTest): + from . import C5932041_PhysXForceRegion_LocalSpaceForceOnRigidBodies as test_module + + class C15425929_Undo_Redo(EditorSharedTest): + from . import C15425929_Undo_Redo as test_module + + class C4976243_Collision_SameCollisionGroupDiffCollisionLayers(EditorSharedTest): + from . import C4976243_Collision_SameCollisionGroupDiffCollisionLayers as test_module + + class C14654881_CharacterController_SwitchLevels(EditorSharedTest): + from . import C14654881_CharacterController_SwitchLevels as test_module + + class C17411467_AddPhysxRagdollComponent(EditorSharedTest): + from . import C17411467_AddPhysxRagdollComponent as test_module + + class C12712453_ScriptCanvas_MultipleRaycastNode(EditorSharedTest): + from . import C12712453_ScriptCanvas_MultipleRaycastNode as test_module + + class C18243586_Joints_HingeLeadFollowerCollide(EditorSharedTest): + from . import C18243586_Joints_HingeLeadFollowerCollide as test_module + + class C4982803_Enable_PxMesh_Option(EditorSharedTest): + from . import C4982803_Enable_PxMesh_Option as test_module + + class C24308873_CylinderShapeCollider_CollidesWithPhysXTerrain(EditorSharedTest): + from . import C24308873_CylinderShapeCollider_CollidesWithPhysXTerrain as test_module + + class C3510642_Terrain_NotCollideWithTerrain(EditorSharedTest): + from . import C3510642_Terrain_NotCollideWithTerrain as test_module + + class C4976195_RigidBodies_InitialLinearVelocity(EditorSharedTest): + from . import C4976195_RigidBodies_InitialLinearVelocity as test_module + + class C4976206_RigidBodies_GravityEnabledActive(EditorSharedTest): + from . import C4976206_RigidBodies_GravityEnabledActive as test_module + + class C4976207_PhysXRigidBodies_KinematicBehavior(EditorSharedTest): + from . import C4976207_PhysXRigidBodies_KinematicBehavior as test_module + + class C5932042_PhysXForceRegion_LinearDamping(EditorSharedTest): + from . import C5932042_PhysXForceRegion_LinearDamping as test_module + + class C5932043_ForceRegion_SimpleDragOnRigidBodies(EditorSharedTest): + from . import C5932043_ForceRegion_SimpleDragOnRigidBodies as test_module + + class C5959760_PhysXForceRegion_PointForceExertion(EditorSharedTest): + from . import C5959760_PhysXForceRegion_PointForceExertion as test_module + + class C5959764_ForceRegion_ForceRegionImpulsesCapsule(EditorSharedTest): + from . import C5959764_ForceRegion_ForceRegionImpulsesCapsule as test_module + + class C5340400_RigidBody_ManualMomentOfInertia(EditorSharedTest): + from . import C5340400_RigidBody_ManualMomentOfInertia as test_module + + class C4976210_COM_ManualSetting(EditorSharedTest): + from . import C4976210_COM_ManualSetting as test_module + + class C4976194_RigidBody_PhysXComponentIsValid(EditorSharedTest): + from . import C4976194_RigidBody_PhysXComponentIsValid as test_module + + class C5932045_ForceRegion_Spline(EditorSharedTest): + from . import C5932045_ForceRegion_Spline as test_module + + class C4982797_Collider_ColliderOffset(EditorSharedTest): + from . import C4982797_Collider_ColliderOffset as test_module + + class C4976200_RigidBody_AngularDampingObjectRotation(EditorSharedTest): + from . import C4976200_RigidBody_AngularDampingObjectRotation as test_module + + class C5689529_Verify_Terrain_RigidBody_Collider_Mesh(EditorSharedTest): + from . import C5689529_Verify_Terrain_RigidBody_Collider_Mesh as test_module + + class C5959810_ForceRegion_ForceRegionCombinesForces(EditorSharedTest): + from . import C5959810_ForceRegion_ForceRegionCombinesForces as test_module + + class C5959765_ForceRegion_AssetGetsImpulsed(EditorSharedTest): + from . import C5959765_ForceRegion_AssetGetsImpulsed as test_module + + class C6274125_ScriptCanvas_TriggerEvents(EditorSharedTest): + from . import C6274125_ScriptCanvas_TriggerEvents as test_module + # needs to be updated to log for unexpected lines + # expected_lines = test_module.LogLines.expected_lines + + class C6090554_ForceRegion_PointForceNegative(EditorSharedTest): + from . import C6090554_ForceRegion_PointForceNegative as test_module + + class C6090550_ForceRegion_WorldSpaceForceNegative(EditorSharedTest): + from . import C6090550_ForceRegion_WorldSpaceForceNegative as test_module + + class C6090552_ForceRegion_LinearDampingNegative(EditorSharedTest): + from . import C6090552_ForceRegion_LinearDampingNegative as test_module + + class C5968760_ForceRegion_CheckNetForceChange(EditorSharedTest): + from . import C5968760_ForceRegion_CheckNetForceChange as test_module + + class C12712452_ScriptCanvas_CollisionEvents(EditorSharedTest): + from . import C12712452_ScriptCanvas_CollisionEvents as test_module + + class C12868578_ForceRegion_DirectionHasNoAffectOnMagnitude(EditorSharedTest): + from . import C12868578_ForceRegion_DirectionHasNoAffectOnMagnitude as test_module + + class C4976204_Verify_Start_Asleep_Condition(EditorSharedTest): + from . import C4976204_Verify_Start_Asleep_Condition as test_module + + class C6090546_ForceRegion_SliceFileInstantiates(EditorSharedTest): + from . import C6090546_ForceRegion_SliceFileInstantiates as test_module + + class C6090551_ForceRegion_LocalSpaceForceNegative(EditorSharedTest): + from . import C6090551_ForceRegion_LocalSpaceForceNegative as test_module + + class C6090553_ForceRegion_SimpleDragForceOnRigidBodies(EditorSharedTest): + from . import C6090553_ForceRegion_SimpleDragForceOnRigidBodies as test_module + + class C4976209_RigidBody_ComputesCOM(EditorSharedTest): + from . import C4976209_RigidBody_ComputesCOM as test_module + + class C4976201_RigidBody_MassIsAssigned(EditorSharedTest): + from . import C4976201_RigidBody_MassIsAssigned as test_module + + class C12868580_ForceRegion_SplineModifiedTransform(EditorSharedTest): + from . import C12868580_ForceRegion_SplineModifiedTransform as test_module + + class C12712455_ScriptCanvas_ShapeCastVerification(EditorSharedTest): + from . import C12712455_ScriptCanvas_ShapeCastVerification as test_module + + class C4976197_RigidBodies_InitialAngularVelocity(EditorSharedTest): + from . import C4976197_RigidBodies_InitialAngularVelocity as test_module + + class C6090555_ForceRegion_SplineFollowOnRigidBodies(EditorSharedTest): + from . import C6090555_ForceRegion_SplineFollowOnRigidBodies as test_module + + class C6131473_StaticSlice_OnDynamicSliceSpawn(EditorSharedTest): + from . import C6131473_StaticSlice_OnDynamicSliceSpawn as test_module + + class C5959808_ForceRegion_PositionOffset(EditorSharedTest): + from . import C5959808_ForceRegion_PositionOffset as test_module + + @pytest.mark.xfail(reason="Something with the CryRenderer disabling is causing this test to fail now.") + class C13895144_Ragdoll_ChangeLevel(EditorSharedTest): + from . import C13895144_Ragdoll_ChangeLevel as test_module + + class C5968759_ForceRegion_ExertsSeveralForcesOnRigidBody(EditorSharedTest): + from . import C5968759_ForceRegion_ExertsSeveralForcesOnRigidBody as test_module + + @pytest.mark.xfail(reason="This test will sometimes fail as the ball will continue to roll before the timeout is reached.") + class C4976202_RigidBody_StopsWhenBelowKineticThreshold(EditorSharedTest): + from . import C4976202_RigidBody_StopsWhenBelowKineticThreshold as test_module + + class C13351703_COM_NotIncludeTriggerShapes(EditorSharedTest): + from . import C13351703_COM_NotIncludeTriggerShapes as test_module + + class C5296614_PhysXMaterial_ColliderShape(EditorSharedTest): + from . import C5296614_PhysXMaterial_ColliderShape as test_module + + class C4982595_Collider_TriggerDisablesCollision(EditorSharedTest): + from . import C4982595_Collider_TriggerDisablesCollision as test_module + + class C14976307_Gravity_SetGravityWorks(EditorSharedTest): + from . import C14976307_Gravity_SetGravityWorks as test_module + + class C4044694_Material_EmptyLibraryUsesDefault(EditorSharedTest): + from . import C4044694_Material_EmptyLibraryUsesDefault as test_module + + class C15845879_ForceRegion_HighLinearDampingForce(EditorSharedTest): + from . import C15845879_ForceRegion_HighLinearDampingForce as test_module + + class C4976218_RigidBodies_InertiaObjectsNotComputed(EditorSharedTest): + from . import C4976218_RigidBodies_InertiaObjectsNotComputed as test_module + + class C14902098_ScriptCanvas_PostPhysicsUpdate(EditorSharedTest): + from . import C14902098_ScriptCanvas_PostPhysicsUpdate as test_module + # Note: Test needs to be updated to log for unexpected lines + # unexpected_lines = ["Assert"] + test_module.Lines.unexpected + + class C5959761_ForceRegion_PhysAssetExertsPointForce(EditorSharedTest): + from . import C5959761_ForceRegion_PhysAssetExertsPointForce as test_module + + # Marking the Test as expected to fail using the xfail decorator due to sporadic failure on Automated Review: SPEC-3146 + # The test still runs, but a failure of the test doesn't result in the test run failing + @pytest.mark.xfail(reason="Test Sporadically fails with message [ NOT FOUND ] Success: Bar1 : Expected angular velocity") + class C13352089_RigidBodies_MaxAngularVelocity(EditorSharedTest): + from . import C13352089_RigidBodies_MaxAngularVelocity as test_module + + class C18243584_Joints_HingeSoftLimitsConstrained(EditorSharedTest): + from . import C18243584_Joints_HingeSoftLimitsConstrained as test_module + + class C18243589_Joints_BallSoftLimitsConstrained(EditorSharedTest): + from . import C18243589_Joints_BallSoftLimitsConstrained as test_module + + class C18243591_Joints_BallLeadFollowerCollide(EditorSharedTest): + from . import C18243591_Joints_BallLeadFollowerCollide as test_module + + class C19578018_ShapeColliderWithNoShapeComponent(EditorSharedTest): + from . import C19578018_ShapeColliderWithNoShapeComponent as test_module + + class C14861500_DefaultSetting_ColliderShape(EditorSharedTest): + from . import C14861500_DefaultSetting_ColliderShape as test_module + + class C19723164_ShapeCollider_WontCrashEditor(EditorSharedTest): + from . import C19723164_ShapeColliders_WontCrashEditor as test_module + + class C4982800_PhysXColliderShape_CanBeSelected(EditorSharedTest): + from . import C4982800_PhysXColliderShape_CanBeSelected as test_module + + class C4982801_PhysXColliderShape_CanBeSelected(EditorSharedTest): + from . import C4982801_PhysXColliderShape_CanBeSelected as test_module + + class C4982802_PhysXColliderShape_CanBeSelected(EditorSharedTest): + from . import C4982802_PhysXColliderShape_CanBeSelected as test_module + + class C12905528_ForceRegion_WithNonTriggerCollider(EditorSharedTest): + from . import C12905528_ForceRegion_WithNonTriggerCollider as test_module + # Fixme: expected_lines = ["[Warning] (PhysX Force Region) - Please ensure collider component marked as trigger exists in entity"] + + class C5932040_ForceRegion_CubeExertsWorldForce(EditorSharedTest): + from . import C5932040_ForceRegion_CubeExertsWorldForce as test_module + + class C5932044_ForceRegion_PointForceOnRigidBody(EditorSharedTest): + from . import C5932044_ForceRegion_PointForceOnRigidBody as test_module + + class C5959759_RigidBody_ForceRegionSpherePointForce(EditorSharedTest): + from . import C5959759_RigidBody_ForceRegionSpherePointForce as test_module + + class C5959809_ForceRegion_RotationalOffset(EditorSharedTest): + from . import C5959809_ForceRegion_RotationalOffset as test_module + + class C15096740_Material_LibraryUpdatedCorrectly(EditorSharedTest): + from . import C15096740_Material_LibraryUpdatedCorrectly as test_module + + class C4976236_AddPhysxColliderComponent(EditorSharedTest): + from . import C4976236_AddPhysxColliderComponent as test_module + + + @pytest.mark.xfail(reason="This will fail due to this issue ATOM-15487.") + class C14861502_PhysXCollider_AssetAutoAssigned(EditorSharedTest): + from . import C14861502_PhysXCollider_AssetAutoAssigned as test_module + + class C14861501_PhysXCollider_RenderMeshAutoAssigned(EditorSharedTest): + from . import C14861501_PhysXCollider_RenderMeshAutoAssigned as test_module + + class C4044695_PhysXCollider_AddMultipleSurfaceFbx(EditorSharedTest): + from . import C4044695_PhysXCollider_AddMultipleSurfaceFbx as test_module + + class C14861504_RenderMeshAsset_WithNoPxAsset(EditorSharedTest): + from . import C14861504_RenderMeshAsset_WithNoPxAsset as test_module + + class C100000_RigidBody_EnablingGravityWorksPoC(EditorSharedTest): + from . import C100000_RigidBody_EnablingGravityWorksPoC as test_module + + class C4982798_Collider_ColliderRotationOffset(EditorSharedTest): + from . import C4982798_Collider_ColliderRotationOffset as test_module + + class C15308217_NoCrash_LevelSwitch(EditorSharedTest): + from . import C15308217_NoCrash_LevelSwitch as test_module + + class C6090547_ForceRegion_ParentChildForceRegions(EditorSharedTest): + from . import C6090547_ForceRegion_ParentChildForceRegions as test_module + + class C19578021_ShapeCollider_CanBeAdded(EditorSharedTest): + from . import C19578021_ShapeCollider_CanBeAdded as test_module + + class C15425929_Undo_Redo(EditorSharedTest): + from . import C15425929_Undo_Redo as test_module diff --git a/AutomatedTesting/Gem/PythonTests/physics/TestSuite_Main_Test.py b/AutomatedTesting/Gem/PythonTests/physics/TestSuite_Main_Test.py deleted file mode 100644 index 6b940f93d1..0000000000 --- a/AutomatedTesting/Gem/PythonTests/physics/TestSuite_Main_Test.py +++ /dev/null @@ -1,105 +0,0 @@ -""" -Copyright (c) Contributors to the Open 3D Engine Project. -For complete copyright and license terms please see the LICENSE at the root of this distribution. - -SPDX-License-Identifier: Apache-2.0 OR MIT -""" - -import pytest -import os -import sys -import inspect - -from ly_test_tools import LAUNCHERS -from ly_test_tools.o3de.editor_test import EditorSingleTest, EditorSharedTest, EditorParallelTest, EditorTestSuite -from .FileManagement import FileManagement as fm - -# Custom test spec, it provides functionality to override files -class EditorSingleTest_WithFileOverrides(EditorSingleTest): - # Specify here what files to override, [(original, override), ...] - files_to_override = [()] - # Base directory of the files (Default path is {ProjectName}) - base_dir = None - # True will will search sub-directories for the files in base - search_subdirs = False - - @classmethod - def wrap_run(cls, instance, request, workspace, editor, editor_test_results, launcher_platform): - root_path = cls.base_dir - if root_path is not None: - root_path = os.path.join(workspace.paths.engine_root(), root_path) - else: - # Default to project folder - root_path = workspace.paths.project() - - # Try to locate both target and source files - original_file_list, override_file_list = zip(*cls.files_to_override) - try: - file_list = fm._find_files(original_file_list + override_file_list, root_path, cls.search_subdirs) - except RuntimeWarning as w: - assert False, ( - w.message - + " Please check use of search_subdirs; make sure you are using the correct parent directory." - ) - - for f in original_file_list: - fm._restore_file(f, file_list[f]) - fm._backup_file(f, file_list[f]) - - for original, override in cls.files_to_override: - fm._copy_file(override, file_list[override], original, file_list[override]) - - yield # Run Test - for f in original_file_list: - fm._restore_file(f, file_list[f]) - - -@pytest.mark.SUITE_main -@pytest.mark.parametrize("launcher_platform", ['windows_editor']) -@pytest.mark.parametrize("project", ["AutomatedTesting"]) -class TestAutomation(EditorTestSuite): - - class C4044459_Material_DynamicFriction(EditorSingleTest_WithFileOverrides): - from . import C4044459_Material_DynamicFriction as test_module - files_to_override = [ - ('physxsystemconfiguration.setreg', 'C4044459_Material_DynamicFriction.setreg_override') - ] - base_dir = "AutomatedTesting/Registry" - - class C4982593_PhysXCollider_CollisionLayerTest(EditorSingleTest_WithFileOverrides): - from . import C4982593_PhysXCollider_CollisionLayerTest as test_module - files_to_override = [ - ('physxsystemconfiguration.setreg', 'C4982593_PhysXCollider_CollisionLayer.setreg_override') - ] - base_dir = "AutomatedTesting/Registry" - - class C111111_RigidBody_EnablingGravityWorksUsingNotificationsPoC(EditorSharedTest): - from . import C111111_RigidBody_EnablingGravityWorksUsingNotificationsPoC as test_module - - class C5932041_PhysXForceRegion_LocalSpaceForceOnRigidBodies(EditorSharedTest): - from . import C5932041_PhysXForceRegion_LocalSpaceForceOnRigidBodies as test_module - - class C15425929_Undo_Redo(EditorSharedTest): - from . import C15425929_Undo_Redo as test_module - - class C4976243_Collision_SameCollisionGroupDiffCollisionLayers(EditorSharedTest): - from . import C4976243_Collision_SameCollisionGroupDiffCollisionLayers as test_module - - class C14654881_CharacterController_SwitchLevels(EditorSharedTest): - from . import C14654881_CharacterController_SwitchLevels as test_module - - class C17411467_AddPhysxRagdollComponent(EditorSharedTest): - from . import C17411467_AddPhysxRagdollComponent as test_module - - class C12712453_ScriptCanvas_MultipleRaycastNode(EditorSharedTest): - from . import C12712453_ScriptCanvas_MultipleRaycastNode as test_module - - class C18243586_Joints_HingeLeadFollowerCollide(EditorSharedTest): - from . import C18243586_Joints_HingeLeadFollowerCollide as test_module - - class C4982803_Enable_PxMesh_Option(EditorSharedTest): - from . import C4982803_Enable_PxMesh_Option as test_module - - class C24308873_CylinderShapeCollider_CollidesWithPhysXTerrain(EditorSharedTest): - from . import C24308873_CylinderShapeCollider_CollidesWithPhysXTerrain as test_module - diff --git a/Tools/LyTestTools/ly_test_tools/_internal/pytest_plugin/editor_test.py b/Tools/LyTestTools/ly_test_tools/_internal/pytest_plugin/editor_test.py index 7c60dd20d2..80a562977b 100644 --- a/Tools/LyTestTools/ly_test_tools/_internal/pytest_plugin/editor_test.py +++ b/Tools/LyTestTools/ly_test_tools/_internal/pytest_plugin/editor_test.py @@ -17,7 +17,7 @@ __test__ = False def pytest_addoption(parser): parser.addoption("--no-editor-batch", action="store_true", help="Don't batch multiple tests in single editor") parser.addoption("--no-editor-parallel", action="store_true", help="Don't run multiple editors in parallel") - parser.addoption("--parallel-editors", type=int, action="store", help="Override the number editors to run at the same time") + parser.addoption("--editors-parallel", type=int, action="store", help="Override the number editors to run at the same time") # Create a custom custom item collection if the class defines pytest_custom_makeitem function # This is used for automtically generating test functions with a custom collector diff --git a/Tools/LyTestTools/ly_test_tools/o3de/editor_test.py b/Tools/LyTestTools/ly_test_tools/o3de/editor_test.py index 0851c63681..0f456cee1a 100644 --- a/Tools/LyTestTools/ly_test_tools/o3de/editor_test.py +++ b/Tools/LyTestTools/ly_test_tools/o3de/editor_test.py @@ -246,14 +246,24 @@ class EditorTestSuite(): ## Internal ## _TIMEOUT_CRASH_LOG = 20 # Maximum time (seconds) for waiting for a crash file, in secondss - _TEST_FAIL_RETCODE = 0xF # Return code for test failure - _asset_processor = None - _results = {} + _TEST_FAIL_RETCODE = 0xF # Return code for test failure @pytest.fixture(scope="class") - def editor_test_results(self, request): - results = {} - return results + def editor_test_data(self, request): + class TestData(): + def __init__(self): + self.results = {} + self.asset_processor = None + + test_data = TestData() + yield test_data + if test_data.asset_processor: + test_data.asset_processor.stop(1) + test_data.asset_processor.teardown() + test_data.asset_processor = None + editor_utils.kill_all_ly_processes(include_asset_processor=True) + else: + editor_utils.kill_all_ly_processes(include_asset_processor=False) class Runner(): def __init__(self, name, func, tests): @@ -315,26 +325,25 @@ class EditorTestSuite(): name = test_spec.__name__ def make_test_func(name, test_spec): @set_marks({"run_type" : "run_single"}) - def single_run(self, request, workspace, editor, editor_test_results, launcher_platform): + def single_run(self, request, workspace, editor, editor_test_data, launcher_platform): # only single tests are allowed to have setup/teardown, however we can have shared tests that # were explicitly set as single, for example via cmdline argument override is_single_test = issubclass(test_spec, EditorSingleTest) if is_single_test: # Setup step for wrap_run - wrap = test_spec.wrap_run(self, request, workspace, editor, editor_test_results, launcher_platform) + wrap = test_spec.wrap_run(self, request, workspace, editor, editor_test_data, launcher_platform) assert isinstance(wrap, types.GeneratorType), "wrap_run must return a generator, did you forget 'yield'?" next(wrap, None) # Setup step - test_spec.setup(self, request, workspace, editor, editor_test_results, launcher_platform) + test_spec.setup(self, request, workspace, editor, editor_test_data, launcher_platform) # Run - self._run_single_test(request, workspace, editor, editor_test_results, test_spec) + self._run_single_test(request, workspace, editor, editor_test_data, test_spec) if is_single_test: # Teardown - test_spec.teardown(self, request, workspace, editor, editor_test_results, launcher_platform) + test_spec.teardown(self, request, workspace, editor, editor_test_data, launcher_platform) # Teardown step for wrap_run next(wrap, None) return single_run - setattr(self.obj, name, make_test_func(name, test_spec)) f = make_test_func(name, test_spec) if hasattr(test_spec, "pytestmark"): f.pytestmark = test_spec.pytestmark @@ -348,8 +357,8 @@ class EditorTestSuite(): runner = EditorTestSuite.Runner(name, function, tests) def make_func(): @set_marks({"runner" : runner, "run_type" : "run_shared"}) - def shared_run(self, request, workspace, editor, editor_test_results, launcher_platform): - getattr(self, function.__name__)(request, workspace, editor, editor_test_results, runner.tests) + def shared_run(self, request, workspace, editor, editor_test_data, launcher_platform): + getattr(self, function.__name__)(request, workspace, editor, editor_test_data, runner.tests) return shared_run setattr(self.obj, name, make_func()) @@ -357,11 +366,11 @@ class EditorTestSuite(): for test_spec in tests: def make_func(test_spec): @set_marks({"runner" : runner, "test_spec" : test_spec, "run_type" : "result"}) - def result(self, request, workspace, editor, editor_test_results, launcher_platform): - # The runner must have filled the editor_test_results dict fixture for this test. + def result(self, request, workspace, editor, editor_test_data, launcher_platform): + # The runner must have filled the editor_test_data.results dict fixture for this test. # Hitting this assert could mean if there was an error executing the runner - assert test_spec.__name__ in editor_test_results, f"No run data for test: {test_spec.__name__}." - cls._report_result(test_spec.__name__, editor_test_results[test_spec.__name__]) + assert test_spec.__name__ in editor_test_data.results, f"No run data for test: {test_spec.__name__}." + cls._report_result(test_spec.__name__, editor_test_data.results[test_spec.__name__]) return result result_func = make_func(test_spec) @@ -481,41 +490,29 @@ class EditorTestSuite(): ) ] - def setup_class(cls): - cls._asset_processor = None - - def teardown_class(cls): - if cls._asset_processor: - cls._asset_processor.stop(1) - cls._asset_processor.teardown() - cls._asset_processor = None - editor_utils.kill_all_ly_processes(include_asset_processor=True) - else: - editor_utils.kill_all_ly_processes(include_asset_processor=False) - ### Utils ### # Prepares the asset processor for the test - def _prepare_asset_processor(self, workspace): + def _prepare_asset_processor(self, workspace, editor_test_data): try: # Start-up an asset processor if we are not running one # If another AP process exist, don't kill it, as we don't own it - if self._asset_processor is None: + if editor_test_data.asset_processor is None: if not process_utils.process_exists("AssetProcessor", ignore_extensions=True): - editor_utils.kill_all_ly_processes() - self._asset_processor = AssetProcessor(workspace) - self._asset_processor.start() + editor_utils.kill_all_ly_processes(include_asset_processor=True) + editor_test_data.asset_processor = AssetProcessor(workspace) + editor_test_data.asset_processor.start() else: editor_utils.kill_all_ly_processes(include_asset_processor=False) else: # Make sure the asset processor from before wasn't closed by accident - self._asset_processor.start() + editor_test_data.asset_processor.start() except Exception as ex: - self._asset_processor = None + editor_test_data.asset_processor = None raise ex - def _setup_editor_test(self, editor, workspace): - self._prepare_asset_processor(workspace) + def _setup_editor_test(self, editor, workspace, editor_test_data): + self._prepare_asset_processor(workspace, editor_test_data) editor_utils.kill_all_ly_processes(include_asset_processor=False) editor.configure_settings() @@ -555,8 +552,11 @@ class EditorTestSuite(): json_output = json_result["output"] # Cut the editor log so it only has the output for this run - m = json_result["log_match"] - end = m.end() if test_spec != test_spec_list[-1] else -1 + if "log_match" in json_result: + m = json_result["log_match"] + end = m.end() if test_spec != test_spec_list[-1] else -1 + else: + end = -1 cur_log = editor_log_content[log_start : end] log_start = end @@ -681,40 +681,38 @@ class EditorTestSuite(): for key, result in results.items(): if isinstance(result, Result.Unknown): results[key] = Result.Timeout.create(result.output, total_timeout, result.editor_log) + # FIX-ME return results # Runs a single test with the given specs, used by the collector to register the test - def _run_single_test(self, request, workspace, editor, editor_test_results, test_spec : EditorTestBase): - self._setup_editor_test(editor, workspace) + def _run_single_test(self, request, workspace, editor, editor_test_data, test_spec : EditorTestBase): + self._setup_editor_test(editor, workspace, editor_test_data) extra_cmdline_args = [] if hasattr(test_spec, "extra_cmdline_args"): extra_cmdline_args = test_spec.extra_cmdline_args results = self._exec_editor_test(request, workspace, editor, 1, "editor_test.log", test_spec, extra_cmdline_args) - if not hasattr(self.__class__, "_results"): - self.__class__._results = {} - - editor_test_results.update(results) + editor_test_data.results.update(results) test_name, test_result = next(iter(results.items())) self._report_result(test_name, test_result) # Runs a batch of tests in one single editor with the given spec list - def _run_batched_tests(self, request, workspace, editor, editor_test_results, test_spec_list : List[EditorTestBase], extra_cmdline_args=[]): + def _run_batched_tests(self, request, workspace, editor, editor_test_data, test_spec_list : List[EditorTestBase], extra_cmdline_args=[]): if not test_spec_list: return - self._setup_editor_test(editor, workspace) + self._setup_editor_test(editor, workspace, editor_test_data) results = self._exec_editor_multitest(request, workspace, editor, 1, "editor_test.log", test_spec_list, extra_cmdline_args) assert results is not None - editor_test_results.update(results) + editor_test_data.results.update(results) # Runs multiple editors with one test on each editor - def _run_parallel_tests(self, request, workspace, editor, editor_test_results, test_spec_list : List[EditorTestBase], extra_cmdline_args=[]): + def _run_parallel_tests(self, request, workspace, editor, editor_test_data, test_spec_list : List[EditorTestBase], extra_cmdline_args=[]): if not test_spec_list: return - self._setup_editor_test(editor, workspace) + self._setup_editor_test(editor, workspace, editor_test_data) parallel_editors = self._get_number_parallel_editors(request) assert parallel_editors > 0, "Must have at least one editor" @@ -744,14 +742,14 @@ class EditorTestSuite(): t.join() for result in results_per_thread: - editor_test_results.update(result) + editor_test_data.results.update(result) # Runs multiple editors with a batch of tests for each editor - def _run_parallel_batched_tests(self, request, workspace, editor, editor_test_results, test_spec_list : List[EditorTestBase], extra_cmdline_args=[]): + def _run_parallel_batched_tests(self, request, workspace, editor, editor_test_data, test_spec_list : List[EditorTestBase], extra_cmdline_args=[]): if not test_spec_list: return - self._setup_editor_test(editor, workspace) + self._setup_editor_test(editor, workspace, editor_test_data) total_threads = self._get_number_parallel_editors(request) assert total_threads > 0, "Must have at least one editor" threads = [] @@ -781,11 +779,11 @@ class EditorTestSuite(): t.join() for result in results_per_thread: - editor_test_results.update(result) + editor_test_data.results.update(result) # Retrieves the number of parallel preference cmdline overrides def _get_number_parallel_editors(self, request): - parallel_editors_value = request.config.getoption("parallel_editors", None) + parallel_editors_value = request.config.getoption("--editors-parallel", None) if parallel_editors_value: return int(parallel_editors_value)