You cannot select more than 25 topics
Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.
214 lines
8.7 KiB
Python
214 lines
8.7 KiB
Python
#
|
|
# Copyright (c) Contributors to the Open 3D Engine Project.
|
|
# For complete copyright and license terms please see the LICENSE at the root of this distribution.
|
|
#
|
|
# SPDX-License-Identifier: Apache-2.0 OR MIT
|
|
#
|
|
#
|
|
|
|
import argparse
|
|
import glob
|
|
import json
|
|
import logging
|
|
import os
|
|
import pathlib
|
|
import plistlib
|
|
import re
|
|
import sys
|
|
import time
|
|
|
|
TEST_TARGET_NAME = 'TestLauncherTarget'
|
|
TEST_STARTED_STRING = 'TEST STARTED'
|
|
TEST_SUCCESS_STRING = 'TEST SUCCEEDED'
|
|
TEST_FAILURE_STRING = 'TEST FAILED'
|
|
|
|
TEST_RUN_SEARCH_PATTERN=re.compile(r'^\[\s*([a-zA-Z0-9]*\s*)\]\s*([a-zA-Z0-9\.\s]*)(\(.*\))')
|
|
|
|
# Resolve the common python module
|
|
ROOT_DEV_PATH = os.path.realpath(os.path.join(os.path.dirname(__file__), '..', '..', '..', '..'))
|
|
if ROOT_DEV_PATH not in sys.path:
|
|
sys.path.append(ROOT_DEV_PATH)
|
|
|
|
from cmake.Tools import common
|
|
|
|
def launch_ios_test(build_dir, target_dev_name, test_target, timeout_secs, test_filter, xctestrun_file, test_report_json_file):
|
|
|
|
build_path = pathlib.Path(build_dir) if os.path.isabs(build_dir) else pathlib.Path(ROOT_DEV_PATH) / build_dir
|
|
if not build_path.is_dir():
|
|
raise common.LmbrCmdError(f"Invalid build directory '{str(build_path)}'")
|
|
|
|
if xctestrun_file:
|
|
xctestrun_file_path = build_path / xctestrun_file
|
|
if not os.path.exists(xctestrun_file_path):
|
|
raise common.LmbrCmdError(f"'{str(xctestrun_file_path)}' not found in '{str(build_path)}'.")
|
|
else:
|
|
test_run_file = xctestrun_file_path
|
|
else:
|
|
# By default, xcodebuild will place the xctestrun file at the root of the build directory. There is only one file.
|
|
# The xctestrun filename has the format <scheme_name>_iphoneos<sdk_version>-<arch>.xctestrun
|
|
# Our scheme name is always "AzTestRunner" and the only iOS architecture we support is arm64.
|
|
# The SDK version is the only variable. But, Xcode only allows the installation of the latest SDK(based on Xcode version).
|
|
glob_pattern = str(build_path) + '/AzTestRunner_iphoneos*-arm64.xctestrun'
|
|
test_run_files = glob.glob(glob_pattern)
|
|
|
|
if not test_run_files:
|
|
raise common.LmbrCmdError(f"No xctestrun file found in '{str(build_path)}'. Run build_ios_test.py first.")
|
|
|
|
test_run_file = test_run_files[0]
|
|
|
|
test_targets = common.get_validated_test_modules(test_modules=test_target, build_dir_path=build_path)
|
|
|
|
test_run_contents = []
|
|
test_case_successes = []
|
|
test_case_fails = []
|
|
crashed_test_modules = []
|
|
test_case_count = 0
|
|
|
|
with open(test_run_file, 'rb') as fp:
|
|
test_run_contents = plistlib.load(fp)
|
|
|
|
xcode_build = common.CommandLineExec('/usr/bin/xcodebuild')
|
|
|
|
for target in test_targets:
|
|
with open(test_run_file, 'wb') as fp:
|
|
fp.truncate(0)
|
|
|
|
command_line_arguments = [target, 'AzRunUnitTests']
|
|
if test_filter:
|
|
command_line_arguments.extend(['gtest_filter', test_filter])
|
|
test_run_contents[TEST_TARGET_NAME]['CommandLineArguments'] = command_line_arguments
|
|
|
|
with open(test_run_file, 'wb') as fp:
|
|
plistlib.dump(test_run_contents, fp, sort_keys=False)
|
|
|
|
xcode_args = ['test-without-building', '-xctestrun', test_run_file, '-destination', f'platform=iOS,name={target_dev_name}', '-allowProvisioningUpdates', '-allowProvisioningDeviceRegistration']
|
|
if timeout_secs < 0:
|
|
xcode_args.extend(['-test-timeouts-enabled', 'NO'])
|
|
else:
|
|
xcode_args.extend(['-test-timeouts-enabled', 'YES'])
|
|
xcode_args.extend(['-maximum-test-execution-time-allowance', f'{timeout_secs}'])
|
|
|
|
xcode_out = xcode_build.popen(xcode_args, cwd=build_path, shell=False)
|
|
|
|
# Log XCTest's output to debug.
|
|
# Use test start and end markers to separate XCTest output from AzTestRunner's output
|
|
test_success = False
|
|
test_output = False
|
|
while xcode_out.poll() is None:
|
|
line = xcode_out.stdout.readline()
|
|
if line.startswith('** TEST EXECUTE FAILED **'):
|
|
# The test run crashed, so we need to track the failed test module
|
|
crashed_test_modules.append(target)
|
|
else:
|
|
matched = TEST_RUN_SEARCH_PATTERN.search(line)
|
|
if matched:
|
|
test_case_action = matched.group(1).strip()
|
|
test_case_name = matched.group(2).strip()
|
|
test_case_elapsed = matched.group(3).strip() if len(matched.groups()) > 2 else ''
|
|
if test_case_action == 'OK':
|
|
test_case_successes.append(f'{test_case_name} {test_case_elapsed}')
|
|
test_case_count += 1
|
|
elif test_case_action == 'FAILED' and 'listed below:' not in test_case_name:
|
|
test_case_fails.append(f'{test_case_name} {test_case_elapsed}')
|
|
test_case_count += 1
|
|
|
|
if TEST_STARTED_STRING in line:
|
|
test_output = True
|
|
|
|
if TEST_SUCCESS_STRING in line:
|
|
test_success = True
|
|
test_output = False
|
|
elif TEST_FAILURE_STRING in line:
|
|
test_output = False
|
|
|
|
if test_output:
|
|
print(line, end='')
|
|
else:
|
|
logging.debug(line)
|
|
|
|
logging.info(f'{target} Succeeded') if test_success else print(f'{target} Failed')
|
|
|
|
if test_report_json_file:
|
|
test_report_json_path = pathlib.Path(test_report_json_file)
|
|
test_timestamp = time.strftime("%Y-%m-%dT%H:%M:%SZ", time.localtime())
|
|
result_dict = {
|
|
'index': 'ly_platforms.test',
|
|
'payload': {
|
|
'Git Success': True,
|
|
'Build Success': True,
|
|
'Passed': test_case_successes,
|
|
'Failed': test_case_fails,
|
|
'CrashedModules': crashed_test_modules,
|
|
'Count': test_case_count
|
|
},
|
|
'pipeline': 'filebeat',
|
|
'timestamp': test_timestamp
|
|
}
|
|
result_json = json.dumps(result_dict, indent=4)
|
|
test_report_json_path.write_text(result_json, encoding='UTF-8', errors='ignore')
|
|
logging.info(f'MARS report saved to {test_report_json_path}')
|
|
logging.info(f"MARS report:\n{result_json}")
|
|
|
|
|
|
def main(args):
|
|
|
|
parser = argparse.ArgumentParser(description="Launch a test module on a target iOS device.")
|
|
|
|
parser.add_argument('-b', '--build-dir',
|
|
help='The relative build directory to deploy from.',
|
|
required=True)
|
|
|
|
parser.add_argument('test_module',
|
|
nargs='*',
|
|
help='The test module(s) to launch on the target device. Defaults to all registered test modules',
|
|
default=[])
|
|
|
|
parser.add_argument('--device-name',
|
|
help='The name of the iOS device on which to launch.',
|
|
required=True)
|
|
|
|
parser.add_argument('--timeout',
|
|
help='The timeout in secs for each test module to prevent deadlocked tests',
|
|
type=int,
|
|
default=-1)
|
|
|
|
parser.add_argument('--test-filter',
|
|
help='Optional gtest filter to pass along to the unit test launcher',
|
|
default=None)
|
|
|
|
parser.add_argument('--xctestrun-file',
|
|
help='Optional parameter to specify custom xctestrun file (path relative to build directory)',
|
|
default=None)
|
|
|
|
parser.add_argument('--debug',
|
|
help='Enable debug logging',
|
|
action='store_true')
|
|
|
|
parser.add_argument('--test-report-json',
|
|
help='The optional path to the test report json file that will be generated for MARS reporting',
|
|
default=None)
|
|
|
|
parsed_args = parser.parse_args(args)
|
|
|
|
logging.basicConfig(format='%(levelname)s: %(message)s',
|
|
level=logging.DEBUG if parsed_args.debug else logging.INFO)
|
|
|
|
result = launch_ios_test(build_dir=parsed_args.build_dir,
|
|
target_dev_name=parsed_args.device_name,
|
|
test_target=parsed_args.test_module,
|
|
timeout_secs=int(parsed_args.timeout),
|
|
test_filter=parsed_args.test_filter,
|
|
xctestrun_file=parsed_args.xctestrun_file,
|
|
test_report_json_file=parsed_args.test_report_json)
|
|
return 0 if result else 1
|
|
|
|
if __name__ == '__main__':
|
|
|
|
try:
|
|
result_code = main(sys.argv[1:])
|
|
exit(result_code)
|
|
|
|
except common.LmbrCmdError as err:
|
|
logging.error(str(err))
|
|
exit(err.code)
|