From 965fced771299d5a65f7264828dc9db2d326a186 Mon Sep 17 00:00:00 2001 From: Steve Pham <82231385+spham-amzn@users.noreply.github.com> Date: Tue, 8 Jun 2021 15:01:44 -0700 Subject: [PATCH] SPEC-4524: Script updates to support iOS Unit Test Jobs (#1180) --- cmake/Tools/Platform/iOS/build_ios_test.py | 5 +- cmake/Tools/Platform/iOS/launch_ios_test.py | 71 +++++++++++++++--- scripts/build/Platform/iOS/build_config.json | 19 +++++ scripts/build/Platform/iOS/build_ios_test.sh | 76 ++++++++++++++++++++ 4 files changed, 159 insertions(+), 12 deletions(-) create mode 100755 scripts/build/Platform/iOS/build_ios_test.sh diff --git a/cmake/Tools/Platform/iOS/build_ios_test.py b/cmake/Tools/Platform/iOS/build_ios_test.py index 1de24a79f5..c02f1efd1b 100755 --- a/cmake/Tools/Platform/iOS/build_ios_test.py +++ b/cmake/Tools/Platform/iOS/build_ios_test.py @@ -15,6 +15,7 @@ import pathlib import sys SCHEME_NAME = 'AzTestRunner' +XCODE_PROJECT_NAME = 'O3DE' # Resolve the common python module ROOT_DEV_PATH = os.path.realpath(os.path.join(os.path.dirname(__file__), '..', '..', '..', '..')) @@ -30,7 +31,7 @@ def build_ios_test(build_dir, configuration): xcode_build = common.CommandLineExec('/usr/bin/xcodebuild') command_line_arguments = ['build-for-testing', - '-project', 'Lumberyard.xcodeproj', + '-project', f'{XCODE_PROJECT_NAME}.xcodeproj', '-scheme', SCHEME_NAME, '-configuration', configuration, '-allowProvisioningUpdates', @@ -55,7 +56,7 @@ def main(args): parsed_args = parser.parse_args(args) build_ios_test(build_dir=parsed_args.build_dir, - configuration=parsed_args.configuration) + configuration=parsed_args.configuration) return 0 diff --git a/cmake/Tools/Platform/iOS/launch_ios_test.py b/cmake/Tools/Platform/iOS/launch_ios_test.py index f34b6f68b1..19cdd8b91b 100755 --- a/cmake/Tools/Platform/iOS/launch_ios_test.py +++ b/cmake/Tools/Platform/iOS/launch_ios_test.py @@ -11,17 +11,22 @@ import argparse import glob +import json import logging import os import pathlib import plistlib +import re import sys +import time TEST_TARGET_NAME = 'TestLauncherTarget' TEST_STARTED_STRING = 'TEST STARTED' TEST_SUCCESS_STRING = 'TEST SUCCEEDED' TEST_FAILURE_STRING = 'TEST FAILED' +TEST_RUN_SEARCH_PATTERN=re.compile(r'^\[\s*([a-zA-Z0-9]*\s*)\]\s*([a-zA-Z0-9\.\s]*)(\(.*\))') + # Resolve the common python module ROOT_DEV_PATH = os.path.realpath(os.path.join(os.path.dirname(__file__), '..', '..', '..', '..')) if ROOT_DEV_PATH not in sys.path: @@ -29,7 +34,7 @@ if ROOT_DEV_PATH not in sys.path: from cmake.Tools import common -def launch_ios_test(build_dir, target_dev_name, test_target, timeout_secs, test_filter, xctestrun_file): +def launch_ios_test(build_dir, target_dev_name, test_target, timeout_secs, test_filter, xctestrun_file, test_report_json_file): build_path = pathlib.Path(build_dir) if os.path.isabs(build_dir) else pathlib.Path(ROOT_DEV_PATH) / build_dir if not build_path.is_dir(): @@ -57,6 +62,11 @@ def launch_ios_test(build_dir, target_dev_name, test_target, timeout_secs, test_ test_targets = common.get_validated_test_modules(test_modules=test_target, build_dir_path=build_path) test_run_contents = [] + test_case_successes = [] + test_case_fails = [] + crashed_test_modules = [] + test_case_count = 0 + with open(test_run_file, 'rb') as fp: test_run_contents = plistlib.load(fp) @@ -74,7 +84,7 @@ def launch_ios_test(build_dir, target_dev_name, test_target, timeout_secs, test_ with open(test_run_file, 'wb') as fp: plistlib.dump(test_run_contents, fp, sort_keys=False) - xcode_args = ['test-without-building', '-xctestrun', test_run_file, '-destination', f'platform=iOS,name={target_dev_name}'] + xcode_args = ['test-without-building', '-xctestrun', test_run_file, '-destination', f'platform=iOS,name={target_dev_name}', '-allowProvisioningUpdates', '-allowProvisioningDeviceRegistration'] if timeout_secs < 0: xcode_args.extend(['-test-timeouts-enabled', 'NO']) else: @@ -89,6 +99,22 @@ def launch_ios_test(build_dir, target_dev_name, test_target, timeout_secs, test_ test_output = False while xcode_out.poll() is None: line = xcode_out.stdout.readline() + if line.startswith('** TEST EXECUTE FAILED **'): + # The test run crashed, so we need to track the failed test module + crashed_test_modules.append(target) + else: + matched = TEST_RUN_SEARCH_PATTERN.search(line) + if matched: + test_case_action = matched.group(1).strip() + test_case_name = matched.group(2).strip() + test_case_elapsed = matched.group(3).strip() if len(matched.groups()) > 2 else '' + if test_case_action == 'OK': + test_case_successes.append(f'{test_case_name} {test_case_elapsed}') + test_case_count += 1 + elif test_case_action == 'FAILED' and 'listed below:' not in test_case_name: + test_case_fails.append(f'{test_case_name} {test_case_elapsed}') + test_case_count += 1 + if TEST_STARTED_STRING in line: test_output = True @@ -99,11 +125,32 @@ def launch_ios_test(build_dir, target_dev_name, test_target, timeout_secs, test_ test_output = False if test_output: - print(line) + print(line, end='') else: logging.debug(line) - print(f'{target} Succeeded') if test_success else print(f'{target} Failed') + logging.info(f'{target} Succeeded') if test_success else print(f'{target} Failed') + + if test_report_json_file: + test_report_json_path = pathlib.Path(test_report_json_file) + test_timestamp = time.strftime("%Y-%m-%dT%H:%M:%SZ", time.localtime()) + result_dict = { + 'index': 'ly_platforms.test', + 'payload': { + 'Git Success': True, + 'Build Success': True, + 'Passed': test_case_successes, + 'Failed': test_case_fails, + 'CrashedModules': crashed_test_modules, + 'Count': test_case_count + }, + 'pipeline': 'filebeat', + 'timestamp': test_timestamp + } + result_json = json.dumps(result_dict, indent=4) + test_report_json_path.write_text(result_json, encoding='UTF-8', errors='ignore') + logging.info(f'MARS report saved to {test_report_json_path}') + logging.info(f"MARS report:\n{result_json}") def main(args): @@ -140,20 +187,24 @@ def main(args): help='Enable debug logging', action='store_true') + parser.add_argument('--test-report-json', + help='The optional path to the test report json file that will be generated for MARS reporting', + default=None) + parsed_args = parser.parse_args(args) logging.basicConfig(format='%(levelname)s: %(message)s', level=logging.DEBUG if parsed_args.debug else logging.INFO) result = launch_ios_test(build_dir=parsed_args.build_dir, - target_dev_name=parsed_args.device_name, - test_target=parsed_args.test_module, - timeout_secs=int(parsed_args.timeout), - test_filter=parsed_args.test_filter, - xctestrun_file=parsed_args.xctestrun_file) + target_dev_name=parsed_args.device_name, + test_target=parsed_args.test_module, + timeout_secs=int(parsed_args.timeout), + test_filter=parsed_args.test_filter, + xctestrun_file=parsed_args.xctestrun_file, + test_report_json_file=parsed_args.test_report_json) return 0 if result else 1 - if __name__ == '__main__': try: diff --git a/scripts/build/Platform/iOS/build_config.json b/scripts/build/Platform/iOS/build_config.json index 9fd0f8c7fc..e0f59a72be 100644 --- a/scripts/build/Platform/iOS/build_config.json +++ b/scripts/build/Platform/iOS/build_config.json @@ -99,5 +99,24 @@ "CMAKE_TARGET": "ALL_BUILD", "CMAKE_NATIVE_BUILD_ARGS": "-destination generic/platform=iOS" } + }, + "device_test_profile": { + "TAGS": [ + ], + "PIPELINE_ENV":{ + "NODE_LABEL":"Catalina_DeviceFarm_1", + "LY_3RDPARTY_PATH": "/Volumes/Data/jenkins/3rdParty", + "WORKSPACE": "/Volumes/Data/jenkins/workspace" + }, + "COMMAND": "./build_ios_test.sh", + "PARAMETERS": { + "CONFIGURATION": "profile", + "OUTPUT_DIRECTORY": "build/ios_test", + "CMAKE_OPTIONS": "-G Xcode -DCMAKE_TOOLCHAIN_FILE=cmake/Platform/iOS/Toolchain_ios.cmake -DLY_MONOLITHIC_GAME=FALSE -DCMAKE_XCODE_ATTRIBUTE_CODE_SIGNING_REQUIRED=TRUE -DLY_IOS_CODE_SIGNING_IDENTITY=\"\" -DCMAKE_XCODE_ATTRIBUTE_CODE_SIGN_ENTITLEMENTS=\"\" -DCMAKE_XCODE_ATTRIBUTE_CODE_SIGNING_ALLOWED=TRUE -DLY_UNITY_BUILD=TRUE -DO3DE_HOME_PATH=\"${WORKSPACE}/home\" -DO3DE_REGISTER_ENGINE_PATH=\"${WORKSPACE}/o3de\" -DO3DE_REGISTER_THIS_ENGINE=TRUE", + "CMAKE_TARGET": "ALL_BUILD", + "CMAKE_NATIVE_BUILD_ARGS": "", + "TARGET_DEVICE_NAME": "Lumberyard", + "TEST_MODULE_TIMEOUT": "600" + } } } diff --git a/scripts/build/Platform/iOS/build_ios_test.sh b/scripts/build/Platform/iOS/build_ios_test.sh new file mode 100755 index 0000000000..11733fb922 --- /dev/null +++ b/scripts/build/Platform/iOS/build_ios_test.sh @@ -0,0 +1,76 @@ +#!/usr/bin/env bash +# +# All or portions of this file Copyright (c) Amazon.com, Inc. or its affiliates or +# its licensors. +# +# For complete copyright and license terms please see the LICENSE at the root of this +# distribution (the "License"). All use of this software is governed by the License, +# or, if provided, by the license below or the license accompanying this file. Do not +# remove or modify any license notices. This file is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# + +set -o errexit # exit on the first failure encountered + +BASEDIR=$(dirname "$0") +source $BASEDIR/../Mac/env_mac.sh + +mkdir -p ${OUTPUT_DIRECTORY} +SOURCE_DIRECTORY=${PWD} + + +ECHO Configuring for iOS Testing + +LAST_CONFIGURE_CMD_FILE=ci_last_configure_cmd.txt +CONFIGURE_CMD="cmake -B ${OUTPUT_DIRECTORY} ${SOURCE_DIRECTORY} ${CMAKE_OPTIONS} ${EXTRA_CMAKE_OPTIONS} -DLY_3RDPARTY_PATH=${LY_3RDPARTY_PATH}" +if [[ ! -e "CMakeCache.txt" ]]; then + echo [ci_build] First run, generating + RUN_CONFIGURE=1 +elif [[ ! -e ${LAST_CONFIGURE_CMD_FILE} ]]; then + echo [ci_build] Last run command not found, generating + RUN_CONFIGURE=1 +else + # Detect if the input has changed + LAST_CMD=$(<${LAST_CONFIGURE_CMD_FILE}) + if [[ "${LAST_CMD}" != "${CONFIGURE_CMD}" ]]; then + echo [ci_build] Last run command different, generating + RUN_CONFIGURE=1 + fi +fi +if [[ ! -z "$RUN_CONFIGURE" ]]; then + # have to use eval since $CMAKE_OPTIONS (${EXTRA_CMAKE_OPTIONS}) contains quotes that need to be processed + echo [ci_build] ${CONFIGURE_CMD} + eval ${CONFIGURE_CMD} + # Save the run only if success + echo "${CONFIGURE_CMD}" > ${LAST_CONFIGURE_CMD_FILE} +fi + +if [ $? -ne 0 ] +then + echo "CMake configuration failed" + exit 1 +fi + +echo Building for iOS Testing +echo +echo [ci+build] ${SOURCE_DIRECTORY}/python/python.sh ${SOURCE_DIRECTORY}/cmake/Tools/Platform/iOS/build_ios_test.py -b ${OUTPUT_DIRECTORY} -c ${CONFIGURATION} +${SOURCE_DIRECTORY}/python/python.sh ${SOURCE_DIRECTORY}/cmake/Tools/Platform/iOS/build_ios_test.py -b ${OUTPUT_DIRECTORY} -c ${CONFIGURATION} +if [ $? -ne 0 ] +then + echo "iOS Test Build failed" + exit 1 +fi + +ECHO Launching iOS Test +echo [ci+build] ${SOURCE_DIRECTORY}/python/python.sh ${SOURCE_DIRECTORY}/cmake/Tools/Platform/iOS/launch_ios_test.py -b ${OUTPUT_DIRECTORY} --device-name ${TARGET_DEVICE_NAME} --timeout ${TEST_MODULE_TIMEOUT} +${SOURCE_DIRECTORY}/python/python.sh ${SOURCE_DIRECTORY}/cmake/Tools/Platform/iOS/launch_ios_test.py -b ${OUTPUT_DIRECTORY} --device-name ${TARGET_DEVICE_NAME} --timeout ${TEST_MODULE_TIMEOUT} + +if [ $? -ne 0 ] +then + echo "iOS Test failed" + exit 1 +fi + +echo "iOS Tests passed" +exit 0 +