birkeh
/
o3de
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Added conversion of physics tests using new optimizations. Fixed issue on editor test framework to close the AP correctly (#3145)

Browse Source 
* Commit

Signed-off-by: Garcia Ruiz <aljanru@amazon.co.uk>

* Removed xfail

Signed-off-by: Garcia Ruiz <aljanru@amazon.co.uk>

* Fix for ap not closing for optimized tests

Signed-off-by: Garcia Ruiz <aljanru@amazon.co.uk>

* renamed tests to _Optimized

Signed-off-by: Garcia Ruiz <aljanru@amazon.co.uk>

* Added xfail so the test will run in jenkins without affecting outcomes

Signed-off-by: Garcia Ruiz <aljanru@amazon.co.uk>

Co-authored-by: Garcia Ruiz <aljanru@amazon.co.uk>
monroegm-disable-blank-issue-2
AMZN-AlexOteiza 4 years ago committed by GitHub
parent 4e0399b283
commit 426ba2f4a6
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
5 changed files with 418 additions and 163 deletions
Show Stats Download Patch File Download Diff File

13
AutomatedTesting/Gem/PythonTests/physics/CMakeLists.txt
Unescape Escape View File

@ -19,6 +19,19 @@ if(PAL_TRAIT_BUILD_TESTS_SUPPORTED AND PAL_TRAIT_BUILD_HOST_TOOLS)
COMPONENT
Physics
)
ly_add_pytest(
NAME AutomatedTesting::PhysicsTests_Main_Optimized
TEST_SUITE main
TEST_SERIAL
PATH ${CMAKE_CURRENT_LIST_DIR}/TestSuite_Main_Optimized.py
TIMEOUT 1500
RUNTIME_DEPENDENCIES
Legacy::Editor
AZ::AssetProcessor
AutomatedTesting.Assets
COMPONENT
Physics
)
ly_add_pytest(
NAME AutomatedTesting::PhysicsTests_Periodic
TEST_SUITE periodic

349
AutomatedTesting/Gem/PythonTests/physics/TestSuite_Main_Optimized.py
Unescape Escape View File

@ -0,0 +1,349 @@
"""
Copyright (c) Contributors to the Open 3D Engine Project.
For complete copyright and license terms please see the LICENSE at the root of this distribution.
SPDX-License-Identifier: Apache-2.0 OR MIT
"""
import pytest
import os
import sys
import inspect
from ly_test_tools import LAUNCHERS
from ly_test_tools.o3de.editor_test import EditorSingleTest, EditorSharedTest, EditorParallelTest, EditorTestSuite
from .FileManagement import FileManagement as fm
# Custom test spec, it provides functionality to override files
class EditorSingleTest_WithFileOverrides(EditorSingleTest):
# Specify here what files to override, [(original, override), ...]
files_to_override = [()]
# Base directory of the files (Default path is {ProjectName})
base_dir = None
# True will will search sub-directories for the files in base
search_subdirs = False
@classmethod
def wrap_run(cls, instance, request, workspace, editor, editor_test_results, launcher_platform):
root_path = cls.base_dir
if root_path is not None:
root_path = os.path.join(workspace.paths.engine_root(), root_path)
else:
# Default to project folder
root_path = workspace.paths.project()
# Try to locate both target and source files
original_file_list, override_file_list = zip(*cls.files_to_override)
try:
file_list = fm._find_files(original_file_list + override_file_list, root_path, cls.search_subdirs)
except RuntimeWarning as w:
assert False, (
w.message
+ " Please check use of search_subdirs; make sure you are using the correct parent directory."
)
for f in original_file_list:
fm._restore_file(f, file_list[f])
fm._backup_file(f, file_list[f])
for original, override in cls.files_to_override:
fm._copy_file(override, file_list[override], original, file_list[override])
yield # Run Test
for f in original_file_list:
fm._restore_file(f, file_list[f])
@pytest.mark.xfail(reason="Optimized tests are experimental, we will enable xfail and monitor them temporarly.")
@pytest.mark.SUITE_main
@pytest.mark.parametrize("launcher_platform", ['windows_editor'])
@pytest.mark.parametrize("project", ["AutomatedTesting"])
class TestAutomation(EditorTestSuite):
@staticmethod
def get_number_parallel_editors():
return 16
#########################################
# Non-atomic tests: These need to be run in a single editor because they have custom setup and teardown
class C4044459_Material_DynamicFriction(EditorSingleTest_WithFileOverrides):
from . import C4044459_Material_DynamicFriction as test_module
files_to_override = [
('physxsystemconfiguration.setreg', 'C4044459_Material_DynamicFriction.setreg_override')
]
base_dir = "AutomatedTesting/Registry"
class C4982593_PhysXCollider_CollisionLayerTest(EditorSingleTest_WithFileOverrides):
from . import C4982593_PhysXCollider_CollisionLayerTest as test_module
files_to_override = [
('physxsystemconfiguration.setreg', 'C4982593_PhysXCollider_CollisionLayer.setreg_override')
]
base_dir = "AutomatedTesting/Registry"
#########################################
class C111111_RigidBody_EnablingGravityWorksUsingNotificationsPoC(EditorSharedTest):
from . import C111111_RigidBody_EnablingGravityWorksUsingNotificationsPoC as test_module
class C5932041_PhysXForceRegion_LocalSpaceForceOnRigidBodies(EditorSharedTest):
from . import C5932041_PhysXForceRegion_LocalSpaceForceOnRigidBodies as test_module
class C15425929_Undo_Redo(EditorSharedTest):
from . import C15425929_Undo_Redo as test_module
class C4976243_Collision_SameCollisionGroupDiffCollisionLayers(EditorSharedTest):
from . import C4976243_Collision_SameCollisionGroupDiffCollisionLayers as test_module
class C14654881_CharacterController_SwitchLevels(EditorSharedTest):
from . import C14654881_CharacterController_SwitchLevels as test_module
class C17411467_AddPhysxRagdollComponent(EditorSharedTest):
from . import C17411467_AddPhysxRagdollComponent as test_module
class C12712453_ScriptCanvas_MultipleRaycastNode(EditorSharedTest):
from . import C12712453_ScriptCanvas_MultipleRaycastNode as test_module
class C18243586_Joints_HingeLeadFollowerCollide(EditorSharedTest):
from . import C18243586_Joints_HingeLeadFollowerCollide as test_module
class C4982803_Enable_PxMesh_Option(EditorSharedTest):
from . import C4982803_Enable_PxMesh_Option as test_module
class C24308873_CylinderShapeCollider_CollidesWithPhysXTerrain(EditorSharedTest):
from . import C24308873_CylinderShapeCollider_CollidesWithPhysXTerrain as test_module
class C3510642_Terrain_NotCollideWithTerrain(EditorSharedTest):
from . import C3510642_Terrain_NotCollideWithTerrain as test_module
class C4976195_RigidBodies_InitialLinearVelocity(EditorSharedTest):
from . import C4976195_RigidBodies_InitialLinearVelocity as test_module
class C4976206_RigidBodies_GravityEnabledActive(EditorSharedTest):
from . import C4976206_RigidBodies_GravityEnabledActive as test_module
class C4976207_PhysXRigidBodies_KinematicBehavior(EditorSharedTest):
from . import C4976207_PhysXRigidBodies_KinematicBehavior as test_module
class C5932042_PhysXForceRegion_LinearDamping(EditorSharedTest):
from . import C5932042_PhysXForceRegion_LinearDamping as test_module
class C5932043_ForceRegion_SimpleDragOnRigidBodies(EditorSharedTest):
from . import C5932043_ForceRegion_SimpleDragOnRigidBodies as test_module
class C5959760_PhysXForceRegion_PointForceExertion(EditorSharedTest):
from . import C5959760_PhysXForceRegion_PointForceExertion as test_module
class C5959764_ForceRegion_ForceRegionImpulsesCapsule(EditorSharedTest):
from . import C5959764_ForceRegion_ForceRegionImpulsesCapsule as test_module
class C5340400_RigidBody_ManualMomentOfInertia(EditorSharedTest):
from . import C5340400_RigidBody_ManualMomentOfInertia as test_module
class C4976210_COM_ManualSetting(EditorSharedTest):
from . import C4976210_COM_ManualSetting as test_module
class C4976194_RigidBody_PhysXComponentIsValid(EditorSharedTest):
from . import C4976194_RigidBody_PhysXComponentIsValid as test_module
class C5932045_ForceRegion_Spline(EditorSharedTest):
from . import C5932045_ForceRegion_Spline as test_module
class C4982797_Collider_ColliderOffset(EditorSharedTest):
from . import C4982797_Collider_ColliderOffset as test_module
class C4976200_RigidBody_AngularDampingObjectRotation(EditorSharedTest):
from . import C4976200_RigidBody_AngularDampingObjectRotation as test_module
class C5689529_Verify_Terrain_RigidBody_Collider_Mesh(EditorSharedTest):
from . import C5689529_Verify_Terrain_RigidBody_Collider_Mesh as test_module
class C5959810_ForceRegion_ForceRegionCombinesForces(EditorSharedTest):
from . import C5959810_ForceRegion_ForceRegionCombinesForces as test_module
class C5959765_ForceRegion_AssetGetsImpulsed(EditorSharedTest):
from . import C5959765_ForceRegion_AssetGetsImpulsed as test_module
class C6274125_ScriptCanvas_TriggerEvents(EditorSharedTest):
from . import C6274125_ScriptCanvas_TriggerEvents as test_module
# needs to be updated to log for unexpected lines
# expected_lines = test_module.LogLines.expected_lines
class C6090554_ForceRegion_PointForceNegative(EditorSharedTest):
from . import C6090554_ForceRegion_PointForceNegative as test_module
class C6090550_ForceRegion_WorldSpaceForceNegative(EditorSharedTest):
from . import C6090550_ForceRegion_WorldSpaceForceNegative as test_module
class C6090552_ForceRegion_LinearDampingNegative(EditorSharedTest):
from . import C6090552_ForceRegion_LinearDampingNegative as test_module
class C5968760_ForceRegion_CheckNetForceChange(EditorSharedTest):
from . import C5968760_ForceRegion_CheckNetForceChange as test_module
class C12712452_ScriptCanvas_CollisionEvents(EditorSharedTest):
from . import C12712452_ScriptCanvas_CollisionEvents as test_module
class C12868578_ForceRegion_DirectionHasNoAffectOnMagnitude(EditorSharedTest):
from . import C12868578_ForceRegion_DirectionHasNoAffectOnMagnitude as test_module
class C4976204_Verify_Start_Asleep_Condition(EditorSharedTest):
from . import C4976204_Verify_Start_Asleep_Condition as test_module
class C6090546_ForceRegion_SliceFileInstantiates(EditorSharedTest):
from . import C6090546_ForceRegion_SliceFileInstantiates as test_module
class C6090551_ForceRegion_LocalSpaceForceNegative(EditorSharedTest):
from . import C6090551_ForceRegion_LocalSpaceForceNegative as test_module
class C6090553_ForceRegion_SimpleDragForceOnRigidBodies(EditorSharedTest):
from . import C6090553_ForceRegion_SimpleDragForceOnRigidBodies as test_module
class C4976209_RigidBody_ComputesCOM(EditorSharedTest):
from . import C4976209_RigidBody_ComputesCOM as test_module
class C4976201_RigidBody_MassIsAssigned(EditorSharedTest):
from . import C4976201_RigidBody_MassIsAssigned as test_module
class C12868580_ForceRegion_SplineModifiedTransform(EditorSharedTest):
from . import C12868580_ForceRegion_SplineModifiedTransform as test_module
class C12712455_ScriptCanvas_ShapeCastVerification(EditorSharedTest):
from . import C12712455_ScriptCanvas_ShapeCastVerification as test_module
class C4976197_RigidBodies_InitialAngularVelocity(EditorSharedTest):
from . import C4976197_RigidBodies_InitialAngularVelocity as test_module
class C6090555_ForceRegion_SplineFollowOnRigidBodies(EditorSharedTest):
from . import C6090555_ForceRegion_SplineFollowOnRigidBodies as test_module
class C6131473_StaticSlice_OnDynamicSliceSpawn(EditorSharedTest):
from . import C6131473_StaticSlice_OnDynamicSliceSpawn as test_module
class C5959808_ForceRegion_PositionOffset(EditorSharedTest):
from . import C5959808_ForceRegion_PositionOffset as test_module
@pytest.mark.xfail(reason="Something with the CryRenderer disabling is causing this test to fail now.")
class C13895144_Ragdoll_ChangeLevel(EditorSharedTest):
from . import C13895144_Ragdoll_ChangeLevel as test_module
class C5968759_ForceRegion_ExertsSeveralForcesOnRigidBody(EditorSharedTest):
from . import C5968759_ForceRegion_ExertsSeveralForcesOnRigidBody as test_module
@pytest.mark.xfail(reason="This test will sometimes fail as the ball will continue to roll before the timeout is reached.")
class C4976202_RigidBody_StopsWhenBelowKineticThreshold(EditorSharedTest):
from . import C4976202_RigidBody_StopsWhenBelowKineticThreshold as test_module
class C13351703_COM_NotIncludeTriggerShapes(EditorSharedTest):
from . import C13351703_COM_NotIncludeTriggerShapes as test_module
class C5296614_PhysXMaterial_ColliderShape(EditorSharedTest):
from . import C5296614_PhysXMaterial_ColliderShape as test_module
class C4982595_Collider_TriggerDisablesCollision(EditorSharedTest):
from . import C4982595_Collider_TriggerDisablesCollision as test_module
class C14976307_Gravity_SetGravityWorks(EditorSharedTest):
from . import C14976307_Gravity_SetGravityWorks as test_module
class C4044694_Material_EmptyLibraryUsesDefault(EditorSharedTest):
from . import C4044694_Material_EmptyLibraryUsesDefault as test_module
class C15845879_ForceRegion_HighLinearDampingForce(EditorSharedTest):
from . import C15845879_ForceRegion_HighLinearDampingForce as test_module
class C4976218_RigidBodies_InertiaObjectsNotComputed(EditorSharedTest):
from . import C4976218_RigidBodies_InertiaObjectsNotComputed as test_module
class C14902098_ScriptCanvas_PostPhysicsUpdate(EditorSharedTest):
from . import C14902098_ScriptCanvas_PostPhysicsUpdate as test_module
# Note: Test needs to be updated to log for unexpected lines
# unexpected_lines = ["Assert"] + test_module.Lines.unexpected
class C5959761_ForceRegion_PhysAssetExertsPointForce(EditorSharedTest):
from . import C5959761_ForceRegion_PhysAssetExertsPointForce as test_module
# Marking the Test as expected to fail using the xfail decorator due to sporadic failure on Automated Review: SPEC-3146
# The test still runs, but a failure of the test doesn't result in the test run failing
@pytest.mark.xfail(reason="Test Sporadically fails with message [ NOT FOUND ] Success: Bar1 : Expected angular velocity")
class C13352089_RigidBodies_MaxAngularVelocity(EditorSharedTest):
from . import C13352089_RigidBodies_MaxAngularVelocity as test_module
class C18243584_Joints_HingeSoftLimitsConstrained(EditorSharedTest):
from . import C18243584_Joints_HingeSoftLimitsConstrained as test_module
class C18243589_Joints_BallSoftLimitsConstrained(EditorSharedTest):
from . import C18243589_Joints_BallSoftLimitsConstrained as test_module
class C18243591_Joints_BallLeadFollowerCollide(EditorSharedTest):
from . import C18243591_Joints_BallLeadFollowerCollide as test_module
class C19578018_ShapeColliderWithNoShapeComponent(EditorSharedTest):
from . import C19578018_ShapeColliderWithNoShapeComponent as test_module
class C14861500_DefaultSetting_ColliderShape(EditorSharedTest):
from . import C14861500_DefaultSetting_ColliderShape as test_module
class C19723164_ShapeCollider_WontCrashEditor(EditorSharedTest):
from . import C19723164_ShapeColliders_WontCrashEditor as test_module
class C4982800_PhysXColliderShape_CanBeSelected(EditorSharedTest):
from . import C4982800_PhysXColliderShape_CanBeSelected as test_module
class C4982801_PhysXColliderShape_CanBeSelected(EditorSharedTest):
from . import C4982801_PhysXColliderShape_CanBeSelected as test_module
class C4982802_PhysXColliderShape_CanBeSelected(EditorSharedTest):
from . import C4982802_PhysXColliderShape_CanBeSelected as test_module
class C12905528_ForceRegion_WithNonTriggerCollider(EditorSharedTest):
from . import C12905528_ForceRegion_WithNonTriggerCollider as test_module
# Fixme: expected_lines = ["[Warning] (PhysX Force Region) - Please ensure collider component marked as trigger exists in entity"]
class C5932040_ForceRegion_CubeExertsWorldForce(EditorSharedTest):
from . import C5932040_ForceRegion_CubeExertsWorldForce as test_module
class C5932044_ForceRegion_PointForceOnRigidBody(EditorSharedTest):
from . import C5932044_ForceRegion_PointForceOnRigidBody as test_module
class C5959759_RigidBody_ForceRegionSpherePointForce(EditorSharedTest):
from . import C5959759_RigidBody_ForceRegionSpherePointForce as test_module
class C5959809_ForceRegion_RotationalOffset(EditorSharedTest):
from . import C5959809_ForceRegion_RotationalOffset as test_module
class C15096740_Material_LibraryUpdatedCorrectly(EditorSharedTest):
from . import C15096740_Material_LibraryUpdatedCorrectly as test_module
class C4976236_AddPhysxColliderComponent(EditorSharedTest):
from . import C4976236_AddPhysxColliderComponent as test_module
@pytest.mark.xfail(reason="This will fail due to this issue ATOM-15487.")
class C14861502_PhysXCollider_AssetAutoAssigned(EditorSharedTest):
from . import C14861502_PhysXCollider_AssetAutoAssigned as test_module
class C14861501_PhysXCollider_RenderMeshAutoAssigned(EditorSharedTest):
from . import C14861501_PhysXCollider_RenderMeshAutoAssigned as test_module
class C4044695_PhysXCollider_AddMultipleSurfaceFbx(EditorSharedTest):
from . import C4044695_PhysXCollider_AddMultipleSurfaceFbx as test_module
class C14861504_RenderMeshAsset_WithNoPxAsset(EditorSharedTest):
from . import C14861504_RenderMeshAsset_WithNoPxAsset as test_module
class C100000_RigidBody_EnablingGravityWorksPoC(EditorSharedTest):
from . import C100000_RigidBody_EnablingGravityWorksPoC as test_module
class C4982798_Collider_ColliderRotationOffset(EditorSharedTest):
from . import C4982798_Collider_ColliderRotationOffset as test_module
class C15308217_NoCrash_LevelSwitch(EditorSharedTest):
from . import C15308217_NoCrash_LevelSwitch as test_module
class C6090547_ForceRegion_ParentChildForceRegions(EditorSharedTest):
from . import C6090547_ForceRegion_ParentChildForceRegions as test_module
class C19578021_ShapeCollider_CanBeAdded(EditorSharedTest):
from . import C19578021_ShapeCollider_CanBeAdded as test_module
class C15425929_Undo_Redo(EditorSharedTest):
from . import C15425929_Undo_Redo as test_module

105
AutomatedTesting/Gem/PythonTests/physics/TestSuite_Main_Test.py
Unescape Escape View File

@ -1,105 +0,0 @@
"""
Copyright (c) Contributors to the Open 3D Engine Project.
For complete copyright and license terms please see the LICENSE at the root of this distribution.
SPDX-License-Identifier: Apache-2.0 OR MIT
"""
import pytest
import os
import sys
import inspect
from ly_test_tools import LAUNCHERS
from ly_test_tools.o3de.editor_test import EditorSingleTest, EditorSharedTest, EditorParallelTest, EditorTestSuite
from .FileManagement import FileManagement as fm
# Custom test spec, it provides functionality to override files
class EditorSingleTest_WithFileOverrides(EditorSingleTest):
# Specify here what files to override, [(original, override), ...]
files_to_override = [()]
# Base directory of the files (Default path is {ProjectName})
base_dir = None
# True will will search sub-directories for the files in base
search_subdirs = False
@classmethod
def wrap_run(cls, instance, request, workspace, editor, editor_test_results, launcher_platform):
root_path = cls.base_dir
if root_path is not None:
root_path = os.path.join(workspace.paths.engine_root(), root_path)
else:
# Default to project folder
root_path = workspace.paths.project()
# Try to locate both target and source files
original_file_list, override_file_list = zip(*cls.files_to_override)
try:
file_list = fm._find_files(original_file_list + override_file_list, root_path, cls.search_subdirs)
except RuntimeWarning as w:
assert False, (
w.message
+ " Please check use of search_subdirs; make sure you are using the correct parent directory."
)
for f in original_file_list:
fm._restore_file(f, file_list[f])
fm._backup_file(f, file_list[f])
for original, override in cls.files_to_override:
fm._copy_file(override, file_list[override], original, file_list[override])
yield # Run Test
for f in original_file_list:
fm._restore_file(f, file_list[f])
@pytest.mark.SUITE_main
@pytest.mark.parametrize("launcher_platform", ['windows_editor'])
@pytest.mark.parametrize("project", ["AutomatedTesting"])
class TestAutomation(EditorTestSuite):
class C4044459_Material_DynamicFriction(EditorSingleTest_WithFileOverrides):
from . import C4044459_Material_DynamicFriction as test_module
files_to_override = [
('physxsystemconfiguration.setreg', 'C4044459_Material_DynamicFriction.setreg_override')
]
base_dir = "AutomatedTesting/Registry"
class C4982593_PhysXCollider_CollisionLayerTest(EditorSingleTest_WithFileOverrides):
from . import C4982593_PhysXCollider_CollisionLayerTest as test_module
files_to_override = [
('physxsystemconfiguration.setreg', 'C4982593_PhysXCollider_CollisionLayer.setreg_override')
]
base_dir = "AutomatedTesting/Registry"
class C111111_RigidBody_EnablingGravityWorksUsingNotificationsPoC(EditorSharedTest):
from . import C111111_RigidBody_EnablingGravityWorksUsingNotificationsPoC as test_module
class C5932041_PhysXForceRegion_LocalSpaceForceOnRigidBodies(EditorSharedTest):
from . import C5932041_PhysXForceRegion_LocalSpaceForceOnRigidBodies as test_module
class C15425929_Undo_Redo(EditorSharedTest):
from . import C15425929_Undo_Redo as test_module
class C4976243_Collision_SameCollisionGroupDiffCollisionLayers(EditorSharedTest):
from . import C4976243_Collision_SameCollisionGroupDiffCollisionLayers as test_module
class C14654881_CharacterController_SwitchLevels(EditorSharedTest):
from . import C14654881_CharacterController_SwitchLevels as test_module
class C17411467_AddPhysxRagdollComponent(EditorSharedTest):
from . import C17411467_AddPhysxRagdollComponent as test_module
class C12712453_ScriptCanvas_MultipleRaycastNode(EditorSharedTest):
from . import C12712453_ScriptCanvas_MultipleRaycastNode as test_module
class C18243586_Joints_HingeLeadFollowerCollide(EditorSharedTest):
from . import C18243586_Joints_HingeLeadFollowerCollide as test_module
class C4982803_Enable_PxMesh_Option(EditorSharedTest):
from . import C4982803_Enable_PxMesh_Option as test_module
class C24308873_CylinderShapeCollider_CollidesWithPhysXTerrain(EditorSharedTest):
from . import C24308873_CylinderShapeCollider_CollidesWithPhysXTerrain as test_module

2
Tools/LyTestTools/ly_test_tools/_internal/pytest_plugin/editor_test.py
Unescape Escape View File

@ -17,7 +17,7 @@ __test__ = False
def pytest_addoption(parser):
parser.addoption("--no-editor-batch", action="store_true", help="Don't batch multiple tests in single editor")
parser.addoption("--no-editor-parallel", action="store_true", help="Don't run multiple editors in parallel")
parser.addoption("--parallel-editors", type=int, action="store", help="Override the number editors to run at the same time")
parser.addoption("--editors-parallel", type=int, action="store", help="Override the number editors to run at the same time")
# Create a custom custom item collection if the class defines pytest_custom_makeitem function
# This is used for automtically generating test functions with a custom collector

112
Tools/LyTestTools/ly_test_tools/o3de/editor_test.py
Unescape Escape View File

@ -246,14 +246,24 @@ class EditorTestSuite():
## Internal ##
_TIMEOUT_CRASH_LOG = 20 # Maximum time (seconds) for waiting for a crash file, in secondss
_TEST_FAIL_RETCODE = 0xF # Return code for test failure
_asset_processor = None
_results = {}
_TEST_FAIL_RETCODE = 0xF # Return code for test failure
@pytest.fixture(scope="class")
def editor_test_results(self, request):
results = {}
return results
def editor_test_data(self, request):
class TestData():
def __init__(self):
self.results = {}
self.asset_processor = None
test_data = TestData()
yield test_data
if test_data.asset_processor:
test_data.asset_processor.stop(1)
test_data.asset_processor.teardown()
test_data.asset_processor = None
editor_utils.kill_all_ly_processes(include_asset_processor=True)
else:
editor_utils.kill_all_ly_processes(include_asset_processor=False)
class Runner():
def __init__(self, name, func, tests):
@ -315,26 +325,25 @@ class EditorTestSuite():
name = test_spec.__name__
def make_test_func(name, test_spec):
@set_marks({"run_type" : "run_single"})
def single_run(self, request, workspace, editor, editor_test_results, launcher_platform):
def single_run(self, request, workspace, editor, editor_test_data, launcher_platform):
# only single tests are allowed to have setup/teardown, however we can have shared tests that
# were explicitly set as single, for example via cmdline argument override
is_single_test = issubclass(test_spec, EditorSingleTest)
if is_single_test:
# Setup step for wrap_run
wrap = test_spec.wrap_run(self, request, workspace, editor, editor_test_results, launcher_platform)
wrap = test_spec.wrap_run(self, request, workspace, editor, editor_test_data, launcher_platform)
assert isinstance(wrap, types.GeneratorType), "wrap_run must return a generator, did you forget 'yield'?"
next(wrap, None)
# Setup step
test_spec.setup(self, request, workspace, editor, editor_test_results, launcher_platform)
test_spec.setup(self, request, workspace, editor, editor_test_data, launcher_platform)
# Run
self._run_single_test(request, workspace, editor, editor_test_results, test_spec)
self._run_single_test(request, workspace, editor, editor_test_data, test_spec)
if is_single_test:
# Teardown
test_spec.teardown(self, request, workspace, editor, editor_test_results, launcher_platform)
test_spec.teardown(self, request, workspace, editor, editor_test_data, launcher_platform)
# Teardown step for wrap_run
next(wrap, None)
return single_run
setattr(self.obj, name, make_test_func(name, test_spec))
f = make_test_func(name, test_spec)
if hasattr(test_spec, "pytestmark"):
f.pytestmark = test_spec.pytestmark
@ -348,8 +357,8 @@ class EditorTestSuite():
runner = EditorTestSuite.Runner(name, function, tests)
def make_func():
@set_marks({"runner" : runner, "run_type" : "run_shared"})
def shared_run(self, request, workspace, editor, editor_test_results, launcher_platform):
getattr(self, function.__name__)(request, workspace, editor, editor_test_results, runner.tests)
def shared_run(self, request, workspace, editor, editor_test_data, launcher_platform):
getattr(self, function.__name__)(request, workspace, editor, editor_test_data, runner.tests)
return shared_run
setattr(self.obj, name, make_func())
@ -357,11 +366,11 @@ class EditorTestSuite():
for test_spec in tests:
def make_func(test_spec):
@set_marks({"runner" : runner, "test_spec" : test_spec, "run_type" : "result"})
def result(self, request, workspace, editor, editor_test_results, launcher_platform):
# The runner must have filled the editor_test_results dict fixture for this test.
def result(self, request, workspace, editor, editor_test_data, launcher_platform):
# The runner must have filled the editor_test_data.results dict fixture for this test.
# Hitting this assert could mean if there was an error executing the runner
assert test_spec.__name__ in editor_test_results, f"No run data for test: {test_spec.__name__}."
cls._report_result(test_spec.__name__, editor_test_results[test_spec.__name__])
assert test_spec.__name__ in editor_test_data.results, f"No run data for test: {test_spec.__name__}."
cls._report_result(test_spec.__name__, editor_test_data.results[test_spec.__name__])
return result
result_func = make_func(test_spec)
@ -481,41 +490,29 @@ class EditorTestSuite():
)
]
def setup_class(cls):
cls._asset_processor = None
def teardown_class(cls):
if cls._asset_processor:
cls._asset_processor.stop(1)
cls._asset_processor.teardown()
cls._asset_processor = None
editor_utils.kill_all_ly_processes(include_asset_processor=True)
else:
editor_utils.kill_all_ly_processes(include_asset_processor=False)
### Utils ###
# Prepares the asset processor for the test
def _prepare_asset_processor(self, workspace):
def _prepare_asset_processor(self, workspace, editor_test_data):
try:
# Start-up an asset processor if we are not running one
# If another AP process exist, don't kill it, as we don't own it
if self._asset_processor is None:
if editor_test_data.asset_processor is None:
if not process_utils.process_exists("AssetProcessor", ignore_extensions=True):
editor_utils.kill_all_ly_processes()
self._asset_processor = AssetProcessor(workspace)
self._asset_processor.start()
editor_utils.kill_all_ly_processes(include_asset_processor=True)
editor_test_data.asset_processor = AssetProcessor(workspace)
editor_test_data.asset_processor.start()
else:
editor_utils.kill_all_ly_processes(include_asset_processor=False)
else:
# Make sure the asset processor from before wasn't closed by accident
self._asset_processor.start()
editor_test_data.asset_processor.start()
except Exception as ex:
self._asset_processor = None
editor_test_data.asset_processor = None
raise ex
def _setup_editor_test(self, editor, workspace):
self._prepare_asset_processor(workspace)
def _setup_editor_test(self, editor, workspace, editor_test_data):
self._prepare_asset_processor(workspace, editor_test_data)
editor_utils.kill_all_ly_processes(include_asset_processor=False)
editor.configure_settings()
@ -555,8 +552,11 @@ class EditorTestSuite():
json_output = json_result["output"]
# Cut the editor log so it only has the output for this run
m = json_result["log_match"]
end = m.end() if test_spec != test_spec_list[-1] else -1
if "log_match" in json_result:
m = json_result["log_match"]
end = m.end() if test_spec != test_spec_list[-1] else -1
else:
end = -1
cur_log = editor_log_content[log_start : end]
log_start = end
@ -681,40 +681,38 @@ class EditorTestSuite():
for key, result in results.items():
if isinstance(result, Result.Unknown):
results[key] = Result.Timeout.create(result.output, total_timeout, result.editor_log)
# FIX-ME
return results
# Runs a single test with the given specs, used by the collector to register the test
def _run_single_test(self, request, workspace, editor, editor_test_results, test_spec : EditorTestBase):
self._setup_editor_test(editor, workspace)
def _run_single_test(self, request, workspace, editor, editor_test_data, test_spec : EditorTestBase):
self._setup_editor_test(editor, workspace, editor_test_data)
extra_cmdline_args = []
if hasattr(test_spec, "extra_cmdline_args"):
extra_cmdline_args = test_spec.extra_cmdline_args
results = self._exec_editor_test(request, workspace, editor, 1, "editor_test.log", test_spec, extra_cmdline_args)
if not hasattr(self.__class__, "_results"):
self.__class__._results = {}
editor_test_results.update(results)
editor_test_data.results.update(results)
test_name, test_result = next(iter(results.items()))
self._report_result(test_name, test_result)
# Runs a batch of tests in one single editor with the given spec list
def _run_batched_tests(self, request, workspace, editor, editor_test_results, test_spec_list : List[EditorTestBase], extra_cmdline_args=[]):
def _run_batched_tests(self, request, workspace, editor, editor_test_data, test_spec_list : List[EditorTestBase], extra_cmdline_args=[]):
if not test_spec_list:
return
self._setup_editor_test(editor, workspace)
self._setup_editor_test(editor, workspace, editor_test_data)
results = self._exec_editor_multitest(request, workspace, editor, 1, "editor_test.log", test_spec_list, extra_cmdline_args)
assert results is not None
editor_test_results.update(results)
editor_test_data.results.update(results)
# Runs multiple editors with one test on each editor
def _run_parallel_tests(self, request, workspace, editor, editor_test_results, test_spec_list : List[EditorTestBase], extra_cmdline_args=[]):
def _run_parallel_tests(self, request, workspace, editor, editor_test_data, test_spec_list : List[EditorTestBase], extra_cmdline_args=[]):
if not test_spec_list:
return
self._setup_editor_test(editor, workspace)
self._setup_editor_test(editor, workspace, editor_test_data)
parallel_editors = self._get_number_parallel_editors(request)
assert parallel_editors > 0, "Must have at least one editor"
@ -744,14 +742,14 @@ class EditorTestSuite():
t.join()
for result in results_per_thread:
editor_test_results.update(result)
editor_test_data.results.update(result)
# Runs multiple editors with a batch of tests for each editor
def _run_parallel_batched_tests(self, request, workspace, editor, editor_test_results, test_spec_list : List[EditorTestBase], extra_cmdline_args=[]):
def _run_parallel_batched_tests(self, request, workspace, editor, editor_test_data, test_spec_list : List[EditorTestBase], extra_cmdline_args=[]):
if not test_spec_list:
return
self._setup_editor_test(editor, workspace)
self._setup_editor_test(editor, workspace, editor_test_data)
total_threads = self._get_number_parallel_editors(request)
assert total_threads > 0, "Must have at least one editor"
threads = []
@ -781,11 +779,11 @@ class EditorTestSuite():
t.join()
for result in results_per_thread:
editor_test_results.update(result)
editor_test_data.results.update(result)
# Retrieves the number of parallel preference cmdline overrides
def _get_number_parallel_editors(self, request):
parallel_editors_value = request.config.getoption("parallel_editors", None)
parallel_editors_value = request.config.getoption("--editors-parallel", None)
if parallel_editors_value:
return int(parallel_editors_value)

Write Preview
Loading…
Cancel
Save