birkeh/o3de
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Fixed All Physics automated tests (#129)

Browse Source 
* Fixed all Tests.
* Fixed tests stdout redirection
* Changed return code for failed tests to be 0xF
* Small improvements on automated testing code
* Created Periodic test suite and moved tests
* Made physics main to only have one test for now
* Renamed all tests to have leading AutomatedTesting::
This commit is contained in:
AMZN-AlexOteiza
2021-04-26 15:25:34 +01:00
committed by GitHub
parent b885e902d0
commit 3f32cc929c
18 changed files with 374 additions and 313 deletions
Download Patch File Download Diff File Expand all files Collapse all files
AutomatedTesting/Gem/PythonTests/CMakeLists.txt
+52 -39
View File
@@ -19,37 +19,50 @@ add_subdirectory(assetpipeline)
add_subdirectory(atom_renderer)
## Physics ##
# DISABLED - see LYN-2536
#if(PAL_TRAIT_BUILD_TESTS_SUPPORTED AND PAL_TRAIT_BUILD_HOST_TOOLS)
# ly_add_pytest(
# NAME AutomatedTesting::PhysicsTests
# TEST_SUITE main
# TEST_SERIAL
# PATH ${CMAKE_CURRENT_LIST_DIR}/physics/TestSuite_Active.py
# TIMEOUT 3600
# RUNTIME_DEPENDENCIES
# Legacy::Editor
# Legacy::CryRenderNULL
# AZ::AssetProcessor
# AutomatedTesting.Assets
# COMPONENT
# Physics
# )
# ly_add_pytest(
# NAME AutomatedTesting::PhysicsTests_Sandbox
# TEST_SUITE sandbox
# TEST_SERIAL
# PATH ${CMAKE_CURRENT_LIST_DIR}/physics/TestSuite_Sandbox.py
# TIMEOUT 3600
# RUNTIME_DEPENDENCIES
# Legacy::Editor
# Legacy::CryRenderNULL
# AZ::AssetProcessor
# AutomatedTesting.Assets
# COMPONENT
# Physics
# )
#endif()
if(PAL_TRAIT_BUILD_TESTS_SUPPORTED AND PAL_TRAIT_BUILD_HOST_TOOLS)
ly_add_pytest(
NAME AutomatedTesting::PhysicsTests_Main
TEST_SUITE main
TEST_SERIAL
PATH ${CMAKE_CURRENT_LIST_DIR}/physics/TestSuite_Main.py
TIMEOUT 3600
RUNTIME_DEPENDENCIES
Legacy::Editor
Legacy::CryRenderNULL
AZ::AssetProcessor
AutomatedTesting.Assets
COMPONENT
Physics
)
ly_add_pytest(
NAME AutomatedTesting::PhysicsTests_Periodic
TEST_SUITE periodic
TEST_SERIAL
PATH ${CMAKE_CURRENT_LIST_DIR}/physics/TestSuite_Periodic.py
TIMEOUT 3600
RUNTIME_DEPENDENCIES
Legacy::Editor
Legacy::CryRenderNULL
AZ::AssetProcessor
AutomatedTesting.Assets
COMPONENT
Physics
)
ly_add_pytest(
NAME AutomatedTesting::PhysicsTests_Sandbox
TEST_SUITE sandbox
TEST_SERIAL
PATH ${CMAKE_CURRENT_LIST_DIR}/physics/TestSuite_Sandbox.py
TIMEOUT 3600
RUNTIME_DEPENDENCIES
Legacy::Editor
Legacy::CryRenderNULL
AZ::AssetProcessor
AutomatedTesting.Assets
COMPONENT
Physics
)
endif()
## ScriptCanvas ##
if(PAL_TRAIT_BUILD_TESTS_SUPPORTED AND PAL_TRAIT_BUILD_HOST_TOOLS)
@@ -178,7 +191,7 @@ if(PAL_TRAIT_BUILD_TESTS_SUPPORTED AND PAL_TRAIT_BUILD_HOST_TOOLS AND PAL_TRAIT_
## DynVeg ##
ly_add_pytest(
NAME DynamicVegetationTests_Main_GPU
NAME AutomatedTesting::DynamicVegetationTests_Main_GPU
TEST_REQUIRES gpu
TEST_SERIAL
TEST_SUITE main
@@ -195,7 +208,7 @@ if(PAL_TRAIT_BUILD_TESTS_SUPPORTED AND PAL_TRAIT_BUILD_HOST_TOOLS AND PAL_TRAIT_
)
ly_add_pytest(
NAME DynamicVegetationTests_Sandbox_GPU
NAME AutomatedTesting::DynamicVegetationTests_Sandbox_GPU
TEST_REQUIRES gpu
TEST_SERIAL
TEST_SUITE sandbox
@@ -212,7 +225,7 @@ if(PAL_TRAIT_BUILD_TESTS_SUPPORTED AND PAL_TRAIT_BUILD_HOST_TOOLS AND PAL_TRAIT_
)
ly_add_pytest(
NAME DynamicVegetationTests_Periodic_GPU
NAME AutomatedTesting::DynamicVegetationTests_Periodic_GPU
TEST_REQUIRES gpu
TEST_SERIAL
TEST_SUITE periodic
@@ -229,7 +242,7 @@ if(PAL_TRAIT_BUILD_TESTS_SUPPORTED AND PAL_TRAIT_BUILD_HOST_TOOLS AND PAL_TRAIT_
## LandscapeCanvas ##
ly_add_pytest(
NAME LandscapeCanvasTests_Main
NAME AutomatedTesting::LandscapeCanvasTests_Main
TEST_REQUIRES gpu
TEST_SERIAL
TEST_SUITE main
@@ -245,7 +258,7 @@ if(PAL_TRAIT_BUILD_TESTS_SUPPORTED AND PAL_TRAIT_BUILD_HOST_TOOLS AND PAL_TRAIT_
)
ly_add_pytest(
NAME LandscapeCanvasTests_Periodic
NAME AutomatedTesting::LandscapeCanvasTests_Periodic
TEST_REQUIRES gpu
TEST_SERIAL
TEST_SUITE periodic
@@ -262,7 +275,7 @@ if(PAL_TRAIT_BUILD_TESTS_SUPPORTED AND PAL_TRAIT_BUILD_HOST_TOOLS AND PAL_TRAIT_
## GradientSignal ##
ly_add_pytest(
NAME GradientSignalTests_Periodic
NAME AutomatedTesting::GradientSignalTests_Periodic
TEST_REQUIRES gpu
TEST_SERIAL
TEST_SUITE periodic
@@ -281,7 +294,7 @@ endif()
## Editor ##
if(PAL_TRAIT_BUILD_TESTS_SUPPORTED AND PAL_TRAIT_BUILD_HOST_TOOLS AND PAL_TRAIT_FOUNDATION_TEST_SUPPORTED)
ly_add_pytest(
NAME EditorTests_Periodic
NAME AutomatedTesting::EditorTests_Periodic
TEST_SUITE periodic
TEST_SERIAL
PATH ${CMAKE_CURRENT_LIST_DIR}/editor
@@ -299,7 +312,7 @@ endif()
if(PAL_TRAIT_BUILD_TESTS_SUPPORTED AND PAL_TRAIT_BUILD_HOST_TOOLS)
# Unstable, SPEC-3838 will restore
#ly_add_pytest(
# NAME asset_load_benchmark_test
# NAME AutomatedTesting::asset_load_benchmark_test
# TEST_SERIAL
# TEST_SUITE benchmark
# PATH ${CMAKE_CURRENT_LIST_DIR}/streaming/benchmark/asset_load_benchmark_test.py
AutomatedTesting/Gem/PythonTests/EditorPythonTestTools/editor_python_test_tools/utils.py
+18
View File
@@ -278,6 +278,12 @@ class Tracer:
self.function = args[3]
self.message = args[4]
def __str__(self):
return f"Warning: [{self.filename}:{self.function}:{self.line}]: [{self.window}] {self.message}"
def __repr__(self):
return f"[Warning: {self.message}]"
class ErrorInfo:
def __init__(self, args):
self.window = args[0]
@@ -285,6 +291,12 @@ class Tracer:
self.line = args[2]
self.function = args[3]
self.message = args[4]
def __str__(self):
return f"Error: [{self.filename}:{self.function}:{self.line}]: [{self.window}] {self.message}"
def __repr__(self):
return f"[Error: {self.message}]"
class AssertInfo:
def __init__(self, args):
@@ -292,6 +304,12 @@ class Tracer:
self.line = args[1]
self.function = args[2]
self.message = args[3]
def __str__(self):
return f"Assert: [{self.filename}:{self.function}:{self.line}]: {self.message}"
def __repr__(self):
return f"[Assert: {self.message}]"
def _on_warning(self, args):
warningInfo = Tracer.WarningInfo(args)
AutomatedTesting/Gem/PythonTests/automatedtesting_shared/base.py
+6 -6
View File
@@ -94,13 +94,13 @@ class TestAutomationBase:
editor_starttime = time.time()
self.logger.debug("Running automated test")
testcase_module_filepath = self._get_testcase_module_filepath(testcase_module)
pycmd = ["--runpythontest", testcase_module_filepath, "-BatchMode", "-autotest_mode", "-NullRenderer"] + extra_cmdline_args
pycmd = ["--runpythontest", testcase_module_filepath, "-BatchMode", "-autotest_mode", "-rhi=null"] + extra_cmdline_args
editor.args.extend(pycmd) # args are added to the WinLauncher start command
editor.start(backupFiles = False, launch_ap = False)
try:
editor.wait(TestAutomationBase.MAX_TIMEOUT)
except WaitTimeoutError:
errors.append(TestRunError("TIMEOUT", "Editor did not close after {TestAutomationBase.MAX_TIMEOUT} seconds, verify the test is ending and the application didn't freeze"))
errors.append(TestRunError("TIMEOUT", f"Editor did not close after {TestAutomationBase.MAX_TIMEOUT} seconds, verify the test is ending and the application didn't freeze"))
editor.kill()
output = editor.get_output()
@@ -118,16 +118,16 @@ class TestAutomationBase:
else:
error_str = "Test failed, no output available..\n"
errors.append(TestRunError("FAILED TEST", error_str))
if return_code != TestAutomationBase.TEST_FAIL_RETCODE: # Crashed
if return_code and return_code != TestAutomationBase.TEST_FAIL_RETCODE: # Crashed
crash_info = "-- No crash log available --"
error_log = os.path.join(workspace.paths.project_log(), 'error.log')
crash_log = os.path.join(workspace.paths.project_log(), 'error.log')
try:
waiter.wait_for(lambda: os.path.exists(error_log), timeout=TestAutomationBase.WAIT_FOR_CRASH_LOG)
waiter.wait_for(lambda: os.path.exists(crash_log), timeout=TestAutomationBase.WAIT_FOR_CRASH_LOG)
except AssertionError:
pass
try:
with open(error_log) as f:
with open(crash_log) as f:
crash_info = f.read()
except Exception as ex:
crash_info += f"\n{str(ex)}"
AutomatedTesting/Gem/PythonTests/physics/C14861501_PhysXCollider_RenderMeshAutoAssigned.py
+9 -5
View File
@@ -24,7 +24,7 @@ class Tests():
# fmt: on
def run():
def C14861501_PhysXCollider_RenderMeshAutoAssigned():
"""
Summary:
Create entity with Mesh component and assign a render mesh to the Mesh component. Add Physics Collider component
@@ -61,7 +61,7 @@ def run():
from asset_utils import Asset
# Asset paths
STATIC_MESH = os.path.join("assets", "c14861501_physxcollider_rendermeshautoassigned", "spherebot", "r0-b_body.cgf")
STATIC_MESH = os.path.join("assets", "c14861501_physxcollider_rendermeshautoassigned", "spherebot", "r0-b_body.azmodel")
PHYSX_MESH = os.path.join(
"assets", "c14861501_physxcollider_rendermeshautoassigned", "spherebot", "r0-b_body.pxmesh"
)
@@ -80,8 +80,8 @@ def run():
# 4) Assign a render mesh asset to Mesh component (the fbx mesh having both Static mesh and PhysX collision Mesh)
mesh_asset = Asset.find_asset_by_path(STATIC_MESH)
mesh_component.set_component_property_value("MeshComponentRenderNode|Mesh asset", mesh_asset.id)
mesh_asset.id = mesh_component.get_component_property_value("MeshComponentRenderNode|Mesh asset")
mesh_component.set_component_property_value("Controller|Configuration|Mesh Asset", mesh_asset.id)
mesh_asset.id = mesh_component.get_component_property_value("Controller|Configuration|Mesh Asset")
Report.result(Tests.assign_mesh_asset, mesh_asset.get_path() == STATIC_MESH.replace(os.sep, "/"))
# 5) Add PhysX Collider component
@@ -95,4 +95,8 @@ def run():
if __name__ == "__main__":
run()
import ImportPathHelper as imports
imports.init()
from utils import Report
Report.start_test(C14861501_PhysXCollider_RenderMeshAutoAssigned)
AutomatedTesting/Gem/PythonTests/physics/C14861502_PhysXCollider_AssetAutoAssigned.py
+2 -2
View File
@@ -62,8 +62,8 @@ def C14861502_PhysXCollider_AssetAutoAssigned():
# Open 3D Engine Imports
import azlmbr.legacy.general as general
MESH_ASSET_PATH = os.path.join("Objects", "SphereBot", "r0-b_body.cgf")
MESH_PROPERTY_PATH = "MeshComponentRenderNode|Mesh asset"
MESH_ASSET_PATH = os.path.join("Objects", "SphereBot", "r0-b_body.azmodel")
MESH_PROPERTY_PATH = "Controller|Configuration|Mesh Asset"
TESTED_PROPERTY_PATH = "Shape Configuration|Asset|PhysX Mesh"
helper.init_idle()
AutomatedTesting/Gem/PythonTests/physics/C14861504_RenderMeshAsset_WithNoPxAsset.py
+3 -3
View File
@@ -69,7 +69,7 @@ def run():
import azlmbr.asset as azasset
# Asset paths
STATIC_MESH = os.path.join("assets", "c14861504_rendermeshasset_withnopxasset", "test_asset.cgf")
STATIC_MESH = os.path.join("assets", "c14861504_rendermeshasset_withnopxasset", "test_asset.azmodel")
helper.init_idle()
# 1) Load the empty level
@@ -85,8 +85,8 @@ def run():
# 4) Assign a render mesh asset to Mesh component (the fbx mesh having both Static mesh and PhysX collision Mesh)
mesh_asset = Asset.find_asset_by_path(STATIC_MESH)
mesh_component.set_component_property_value("MeshComponentRenderNode|Mesh asset", mesh_asset.id)
mesh_asset.id = mesh_component.get_component_property_value("MeshComponentRenderNode|Mesh asset")
mesh_component.set_component_property_value("Controller|Configuration|Mesh Asset", mesh_asset.id)
mesh_asset.id = mesh_component.get_component_property_value("Controller|Configuration|Mesh Asset")
Report.result(Tests.assign_mesh_asset, mesh_asset.get_path() == STATIC_MESH.replace(os.sep, "/"))
# 5) Add PhysX Collider component
AutomatedTesting/Gem/PythonTests/physics/C4044695_PhysXCollider_AddMultipleSurfaceFbx.py
+10 -11
View File
@@ -27,7 +27,7 @@ class Tests():
# fmt: on
def run():
def C4044695_PhysXCollider_AddMultipleSurfaceFbx():
"""
Summary:
Create entity with Mesh and PhysX Collider components and assign a fbx file in both the components.
@@ -45,12 +45,7 @@ def run():
4) Select the PhysicsAsset shape in the PhysX Collider component
5) Assign the fbx file in PhysX Mesh and Mesh component
6) Check if multiple material slots show up under Materials section in the PhysX Collider component
Note:
- This test file must be called from the Open 3D Engine Editor command terminal
- Any passed and failed tests are written to the Editor.log file.
Parsing the file or running a log_monitor are required to observe the test results.
:return: None
"""
# Builtins
@@ -70,7 +65,7 @@ def run():
SURFACE_TAG_COUNT = 4 # Number of surface tags included in used asset
# Asset paths
STATIC_MESH = os.path.join("assets", "c4044695_physxcollider_addmultiplesurfacefbx", "test.cgf")
STATIC_MESH = os.path.join("assets", "c4044695_physxcollider_addmultiplesurfacefbx", "test.azmodel")
PHYSX_MESH = os.path.join("assets", "c4044695_physxcollider_addmultiplesurfacefbx", "test.pxmesh")
helper.init_idle()
@@ -100,8 +95,8 @@ def run():
Report.result(Tests.assign_px_mesh_asset, px_asset.get_path() == PHYSX_MESH.replace(os.sep, "/"))
mesh_asset = Asset.find_asset_by_path(STATIC_MESH)
mesh_component.set_component_property_value("MeshComponentRenderNode|Mesh asset", mesh_asset.id)
mesh_asset.id = mesh_component.get_component_property_value("MeshComponentRenderNode|Mesh asset")
mesh_component.set_component_property_value("Controller|Configuration|Mesh Asset", mesh_asset.id)
mesh_asset.id = mesh_component.get_component_property_value("Controller|Configuration|Mesh Asset")
Report.result(Tests.assign_mesh_asset, mesh_asset.get_path() == STATIC_MESH.replace(os.sep, "/"))
# 6) Check if multiple material slots show up under Materials section in the PhysX Collider component
@@ -116,4 +111,8 @@ def run():
if __name__ == "__main__":
run()
import ImportPathHelper as imports
imports.init()
from utils import Report
Report.start_test(C4044695_PhysXCollider_AddMultipleSurfaceFbx)
AutomatedTesting/Gem/PythonTests/physics/C4976236_AddPhysxColliderComponent.py
+10 -22
View File
@@ -27,7 +27,7 @@ class Tests():
def C4976236_AddPhysxColliderComponent():
"""
Summary:
Load level with Entity having PhysX Collider component. Verify that editor remains stable in Game mode.
Opens an empty level and creates an Entity with PhysX Collider. Verify that editor remains stable in Game mode.
Expected Behavior:
The Editor is stable there are no warnings or errors.
@@ -37,16 +37,10 @@ def C4976236_AddPhysxColliderComponent():
2) Create test entity
3) Start the Tracer to catch any errors and warnings
4) Add the PhysX Collider component and change shape to box
5) Add Mesh component and an asset
6) Enter game mode
7) Verify there are no errors and warnings in the logs
8) Exit game mode
9) Close the editor
Note:
- This test file must be called from the Open 3D Engine Editor command terminal
- Any passed and failed tests are written to the Editor.log file.
Parsing the file or running a log_monitor are required to observe the test results.
5) Enter game mode
6) Verify there are no errors and warnings in the logs
7) Exit game mode
8) Close the editor
:return: None
"""
@@ -60,7 +54,7 @@ def C4976236_AddPhysxColliderComponent():
from editor_python_test_tools.utils import TestHelper as helper
from editor_python_test_tools.utils import Tracer
from asset_utils import Asset
helper.init_idle()
# 1) Load the level
helper.open_level("Physics", "Base")
@@ -74,17 +68,12 @@ def C4976236_AddPhysxColliderComponent():
# 4) Add the PhysX Collider component and change shape to box
collider_component = test_entity.add_component("PhysX Collider")
Report.result(Tests.add_physx_collider, test_entity.has_component("PhysX Collider"))
collider_component.set_component_property_value('Shape Configuration|Shape', 1)
collider_component.set_component_property_value('Shape Configuration|Shape', azlmbr.physics.ShapeType_Box)
# 5) Add Mesh component and an asset
mesh_component = test_entity.add_component("Mesh")
asset = Asset.find_asset_by_path(r"Objects\default\primitive_cube.cgf")
mesh_component.set_component_property_value('MeshComponentRenderNode|Mesh asset', asset.id)
# 6) Enter game mode
# 5) Enter game mode
helper.enter_game_mode(Tests.enter_game_mode)
# 7) Verify there are no errors and warnings in the logs
# 6) Verify there are no errors and warnings in the logs
success_condition = not (section_tracer.has_errors or section_tracer.has_warnings)
Report.result(Tests.no_errors_and_warnings_found, success_condition)
if not success_condition:
@@ -92,9 +81,8 @@ def C4976236_AddPhysxColliderComponent():
Report.info(f"Warnings found: {section_tracer.warnings}")
if section_tracer.has_errors:
Report.info(f"Errors found: {section_tracer.errors}")
Report.failure(Tests.no_errors_and_warnings_found)
# 8) Exit game mode
# 7) Exit game mode
helper.exit_game_mode(Tests.exit_game_mode)
AutomatedTesting/Gem/PythonTests/physics/TestSuite_Main.py
+37
View File
@@ -0,0 +1,37 @@
"""
All or portions of this file Copyright (c) Amazon.com, Inc. or its affiliates or
its licensors.
For complete copyright and license terms please see the LICENSE at the root of this
distribution (the "License"). All use of this software is governed by the License,
or, if provided, by the license below or the license accompanying this file. Do not
remove or modify any license notices. This file is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
"""
# This suite consists of all test cases that are passing and have been verified.
import pytest
import os
import sys
from .FileManagement import FileManagement as fm
from ly_test_tools import LAUNCHERS
sys.path.append(os.path.dirname(os.path.abspath(__file__)) + '/../automatedtesting_shared')
from base import TestAutomationBase
revert_physics_config = fm.file_revert_list(['physxdebugconfiguration.setreg', 'physxdefaultsceneconfiguration.setreg', 'physxsystemconfiguration.setreg'], 'AutomatedTesting/Registry')
@pytest.mark.SUITE_main
@pytest.mark.parametrize("launcher_platform", ['windows_editor'])
@pytest.mark.parametrize("project", ["AutomatedTesting"])
class TestAutomation(TestAutomationBase):
def test_C111111_RigidBody_EnablingGravityWorksUsingNotificationsPoC(self, request, workspace, editor, launcher_platform):
from . import C111111_RigidBody_EnablingGravityWorksUsingNotificationsPoC as test_module
self._run_test(request, workspace, editor, test_module)
AutomatedTesting/Gem/PythonTests/physics/TestSuite_Active.py → AutomatedTesting/Gem/PythonTests/physics/TestSuite_Periodic.py
+3 -17
View File
@@ -27,26 +27,16 @@ from base import TestAutomationBase
revert_physics_config = fm.file_revert_list(['physxdebugconfiguration.setreg', 'physxdefaultsceneconfiguration.setreg', 'physxsystemconfiguration.setreg'], 'AutomatedTesting/Registry')
@pytest.mark.SUITE_main
@pytest.mark.SUITE_periodic
@pytest.mark.parametrize("launcher_platform", ['windows_editor'])
@pytest.mark.parametrize("project", ["AutomatedTesting"])
class TestAutomation(TestAutomationBase):
# Marking the test as an expected failure due to sporadic failure on Automated Review: SPEC-3146
# The test still runs, but a failure of the test doesn't result in the test run failing
@pytest.mark.xfail(
reason="This test seems to fail sometimes due to it being the first test in the testsuite, we'll duplicate it temporarly."
"Need to figure out the reason why this is the case")
@revert_physics_config
def test_C000000_RigidBody_EnablingGravityWorksPoC_DUPLICATE(self, request, workspace, editor, launcher_platform):
from . import C100000_RigidBody_EnablingGravityWorksPoC as test_module
self._run_test(request, workspace, editor, test_module)
@revert_physics_config
def test_C3510642_Terrain_NotCollideWithTerrain(self, request, workspace, editor, launcher_platform):
from . import C3510642_Terrain_NotCollideWithTerrain as test_module
self._run_test(request, workspace, editor, test_module)
@revert_physics_config
def test_C4976195_RigidBodies_InitialLinearVelocity(self, request, workspace, editor, launcher_platform):
from . import C4976195_RigidBodies_InitialLinearVelocity as test_module
@@ -530,8 +520,4 @@ class TestAutomation(TestAutomationBase):
def test_C100000_RigidBody_EnablingGravityWorksPoC(self, request, workspace, editor, launcher_platform):
from . import C100000_RigidBody_EnablingGravityWorksPoC as test_module
self._run_test(request, workspace, editor, test_module)
def test_C111111_RigidBody_EnablingGravityWorksUsingNotificationsPoC(self, request, workspace, editor, launcher_platform):
from . import C111111_RigidBody_EnablingGravityWorksUsingNotificationsPoC as test_module
self._run_test(request, workspace, editor, test_module)
self._run_test(request, workspace, editor, test_module)