(Continuation) Implemented automation paralellization & standarization (#1718)
Engine improvements/fixes Fixed behavior that made the editor automated test to be sometimes stuck if lost the focus is lost. Added support for specifying multiple tests to in batch to the editor, this is achieved by passing --runpythontest with the tests separated by ';' Added new cmdline argument --project-user-path for overriding the user path. This allows to have multiple editors running writing logs and crash logs in different locations. Moved responsability of exiting after a test finishes/passes out of ExecuteByFilenameAsTest, callers will use the bool return to know if the test passed. Editor test batch and parallelization implementation: Now the external python portion of the editor tests will be specified via test specs which will generate the test. Requiring no code. This is almost a data-driven approach. Tests can be specified as single tests, parallel, batchable or batchable+parallel Command line arguments for pytest to override the maximum number of editors, disable parallelization or batching. Automated tests for testing this new editor testing utility Signed-off-by: Garcia Ruiz <aljanru@amazon.co.uk> Co-authored-by: Garcia Ruiz <aljanru@amazon.co.uk>
This commit is contained in:
AMZN-AlexOteiza
GitHub
+83
-35
@@ -11,14 +11,16 @@ import math
|
||||
import azlmbr
|
||||
import azlmbr.legacy.general as general
|
||||
import azlmbr.debug
|
||||
import json
|
||||
|
||||
import traceback
|
||||
|
||||
from typing import Callable, Tuple
|
||||
|
||||
class FailFast(Exception):
|
||||
"""
|
||||
Raise to stop proceeding through test steps.
|
||||
"""
|
||||
|
||||
pass
|
||||
|
||||
|
||||
@@ -30,8 +32,8 @@ class TestHelper:
|
||||
# general.idle_wait_frames(1)
|
||||
|
||||
@staticmethod
|
||||
def open_level(directory, level):
|
||||
# type: (str, ) -> None
|
||||
def open_level(directory : str, level : str):
|
||||
# type: (str, str) -> None
|
||||
"""
|
||||
:param level: the name of the level folder in AutomatedTesting\\Physics\\
|
||||
|
||||
@@ -51,7 +53,7 @@ class TestHelper:
|
||||
general.idle_wait_frames(200)
|
||||
|
||||
@staticmethod
|
||||
def enter_game_mode(msgtuple_success_fail):
|
||||
def enter_game_mode(msgtuple_success_fail : Tuple[str, str]):
|
||||
# type: (tuple) -> None
|
||||
"""
|
||||
:param msgtuple_success_fail: The tuple with the expected/unexpected messages for entering game mode.
|
||||
@@ -65,7 +67,7 @@ class TestHelper:
|
||||
Report.critical_result(msgtuple_success_fail, general.is_in_game_mode())
|
||||
|
||||
@staticmethod
|
||||
def exit_game_mode(msgtuple_success_fail):
|
||||
def exit_game_mode(msgtuple_success_fail : Tuple[str, str]):
|
||||
# type: (tuple) -> None
|
||||
"""
|
||||
:param msgtuple_success_fail: The tuple with the expected/unexpected messages for exiting game mode.
|
||||
@@ -147,84 +149,130 @@ class Timeout:
|
||||
def timed_out(self):
|
||||
return time.time() > self.die_after
|
||||
|
||||
|
||||
class Report:
|
||||
_results = []
|
||||
_exception = None
|
||||
|
||||
@staticmethod
|
||||
def start_test(test_function):
|
||||
def start_test(test_function : Callable):
|
||||
"""
|
||||
Runs the test, outputs the report and asserts in case of failure.
|
||||
@param: The test function to execute
|
||||
"""
|
||||
Report._results = []
|
||||
Report._exception = None
|
||||
general.test_output(f"Starting test {test_function.__name__}...\n")
|
||||
try:
|
||||
test_function()
|
||||
except Exception as ex:
|
||||
Report._exception = traceback.format_exc()
|
||||
Report.report_results(test_function)
|
||||
|
||||
success, report_str = Report.get_report(test_function)
|
||||
# Print on the o3de console, for debugging purpuses
|
||||
print(report_str)
|
||||
# Print the report on the piped stdout of the application
|
||||
general.test_output(report_str)
|
||||
assert success, f"Test {test_function.__name__} failed"
|
||||
|
||||
@staticmethod
|
||||
def report_results(test_function):
|
||||
success = True
|
||||
report = f"Report for {test_function.__name__}:\n"
|
||||
def get_report(test_function : Callable) -> (bool, str):
|
||||
"""
|
||||
Outputs infomation on the editor console for the test
|
||||
@param msg: message to output
|
||||
@return: (success, report_information) tuple
|
||||
"""
|
||||
success = True
|
||||
report = f"Test {test_function.__name__} finished.\nReport:\n"
|
||||
# report_dict is a JSON that can be used to parse test run information from a external runner
|
||||
# The regular report string is intended to be used for manual debugging
|
||||
filename = os.path.splitext(os.path.basename(test_function.__code__.co_filename))[0]
|
||||
report_dict = {'name' : filename, 'success' : True, 'exception' : None}
|
||||
for result in Report._results:
|
||||
passed, info = result
|
||||
success = success and passed
|
||||
test_result_info = ""
|
||||
if passed:
|
||||
report += f"[SUCCESS] {info}\n"
|
||||
test_result_info = f"[SUCCESS] {info}"
|
||||
else:
|
||||
report += f"[FAILED ] {info}\n"
|
||||
test_result_info = f"[FAILED ] {info}"
|
||||
report += f"{test_result_info}\n"
|
||||
if Report._exception:
|
||||
report += "EXCEPTION raised:\n %s\n" % Report._exception[:-1].replace("\n", "\n ")
|
||||
exception_str = Report._exception[:-1].replace("\n", "\n ")
|
||||
report += "EXCEPTION raised:\n %s\n" % exception_str
|
||||
report_dict['exception'] = exception_str
|
||||
success = False
|
||||
report += "Test result: "
|
||||
report += "SUCCESS" if success else "FAILURE"
|
||||
print(report)
|
||||
general.report_test_result(success, report)
|
||||
report += "Test result: " + ("SUCCESS" if success else "FAILURE")
|
||||
report_dict['success'] = success
|
||||
report_dict['output'] = report
|
||||
report_json_str = json.dumps(report_dict)
|
||||
# For helping parsing, the json will be always contained between JSON_START JSON_END
|
||||
report += f"\nJSON_START({report_json_str})JSON_END\n"
|
||||
return success, report
|
||||
|
||||
@staticmethod
|
||||
def info(msg):
|
||||
def info(msg : str):
|
||||
"""
|
||||
Outputs infomation on the editor console for the test
|
||||
@param msg: message to output
|
||||
"""
|
||||
print("Info: {}".format(msg))
|
||||
|
||||
@staticmethod
|
||||
def success(msgtuple_success_fail):
|
||||
def success(msgtuple_success_fail : Tuple[str, str]):
|
||||
"""
|
||||
Given a test string tuple (success_string, failure_string), registers the test result as success
|
||||
@param msgtuple_success_fail: Two element tuple of success and failure strings
|
||||
"""
|
||||
outcome = "Success: {}".format(msgtuple_success_fail[0])
|
||||
print(outcome)
|
||||
Report._results.append((True, outcome))
|
||||
|
||||
@staticmethod
|
||||
def failure(msgtuple_success_fail):
|
||||
def failure(msgtuple_success_fail : Tuple[str, str]):
|
||||
"""
|
||||
Given a test string tuple (success_string, failure_string), registers the test result as failed
|
||||
@param msgtuple_success_fail: Two element tuple of success and failure strings
|
||||
"""
|
||||
outcome = "Failure: {}".format(msgtuple_success_fail[1])
|
||||
print(outcome)
|
||||
Report._results.append((False, outcome))
|
||||
|
||||
@staticmethod
|
||||
def result(msgtuple_success_fail, condition):
|
||||
if not isinstance(condition, bool):
|
||||
raise TypeError("condition argument must be a bool")
|
||||
def result(msgtuple_success_fail : Tuple[str, str], outcome : bool):
|
||||
"""
|
||||
Given a test string tuple (success_string, failure_string), registers the test result based on the
|
||||
given outcome
|
||||
@param msgtuple_success_fail: Two element tuple of success and failure strings
|
||||
@param outcome: True or False if the result has been a sucess or failure
|
||||
"""
|
||||
if not isinstance(outcome, bool):
|
||||
raise TypeError("outcome argument must be a bool")
|
||||
|
||||
if condition:
|
||||
if outcome:
|
||||
Report.success(msgtuple_success_fail)
|
||||
else:
|
||||
Report.failure(msgtuple_success_fail)
|
||||
return condition
|
||||
return outcome
|
||||
|
||||
@staticmethod
|
||||
def critical_result(msgtuple_success_fail, condition, fast_fail_message=None):
|
||||
def critical_result(msgtuple_success_fail : Tuple[str, str], outcome : bool, fast_fail_message : str = None):
|
||||
# type: (tuple, bool, str) -> None
|
||||
"""
|
||||
if condition is False we will fail fast
|
||||
if outcome is False we will fail fast
|
||||
|
||||
:param msgtuple_success_fail: messages to print based on the condition
|
||||
:param condition: success (True) or failure (False)
|
||||
:param msgtuple_success_fail: messages to print based on the outcome
|
||||
:param outcome: success (True) or failure (False)
|
||||
:param fast_fail_message: [optional] message to include on fast fail
|
||||
"""
|
||||
if not isinstance(condition, bool):
|
||||
raise TypeError("condition argument must be a bool")
|
||||
if not isinstance(outcome, bool):
|
||||
raise TypeError("outcome argument must be a bool")
|
||||
|
||||
if not Report.result(msgtuple_success_fail, condition):
|
||||
if not Report.result(msgtuple_success_fail, outcome):
|
||||
TestHelper.fail_fast(fast_fail_message)
|
||||
|
||||
# DEPRECATED: Use vector3_str()
|
||||
@staticmethod
|
||||
def info_vector3(vector3, label="", magnitude=None):
|
||||
def info_vector3(vector3 : azlmbr.math.Vector3, label : str ="", magnitude : float =None):
|
||||
# type: (azlmbr.math.Vector3, str, float) -> None
|
||||
"""
|
||||
prints the vector to the Report.info log. If applied, label will print first,
|
||||
@@ -390,4 +438,4 @@ def vector3_str(vector3):
|
||||
return "(x: {:.2f}, y: {:.2f}, z: {:.2f})".format(vector3.x, vector3.y, vector3.z)
|
||||
|
||||
def aabb_str(aabb):
|
||||
return "[Min: %s, Max: %s]" % (vector3_str(aabb.min), vector3_str(aabb.max))
|
||||
return "[Min: %s, Max: %s]" % (vector3_str(aabb.min), vector3_str(aabb.max))
|
||||
|
||||
Reference in New Issue
Block a user