birkeh/o3de
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

SPEC-4524: Script updates to support iOS Unit Test Jobs (#1180)

Browse Source
This commit is contained in:
Steve Pham
2021-06-08 15:01:44 -07:00
committed by GitHub
parent ef2d89a843
commit 965fced771
4 changed files with 159 additions and 12 deletions
Download Patch File Download Diff File Expand all files Collapse all files
cmake/Tools/Platform/iOS/build_ios_test.py
+3 -2
View File
@@ -15,6 +15,7 @@ import pathlib
import sys
SCHEME_NAME = 'AzTestRunner'
XCODE_PROJECT_NAME = 'O3DE'
# Resolve the common python module
ROOT_DEV_PATH = os.path.realpath(os.path.join(os.path.dirname(__file__), '..', '..', '..', '..'))
@@ -30,7 +31,7 @@ def build_ios_test(build_dir, configuration):
xcode_build = common.CommandLineExec('/usr/bin/xcodebuild')
command_line_arguments = ['build-for-testing',
'-project', 'Lumberyard.xcodeproj',
'-project', f'{XCODE_PROJECT_NAME}.xcodeproj',
'-scheme', SCHEME_NAME,
'-configuration', configuration,
'-allowProvisioningUpdates',
@@ -55,7 +56,7 @@ def main(args):
parsed_args = parser.parse_args(args)
build_ios_test(build_dir=parsed_args.build_dir,
configuration=parsed_args.configuration)
configuration=parsed_args.configuration)
return 0
cmake/Tools/Platform/iOS/launch_ios_test.py
+61 -10
View File
@@ -11,17 +11,22 @@
import argparse
import glob
import json
import logging
import os
import pathlib
import plistlib
import re
import sys
import time
TEST_TARGET_NAME = 'TestLauncherTarget'
TEST_STARTED_STRING = 'TEST STARTED'
TEST_SUCCESS_STRING = 'TEST SUCCEEDED'
TEST_FAILURE_STRING = 'TEST FAILED'
TEST_RUN_SEARCH_PATTERN=re.compile(r'^\[\s*([a-zA-Z0-9]*\s*)\]\s*([a-zA-Z0-9\.\s]*)(\(.*\))')
# Resolve the common python module
ROOT_DEV_PATH = os.path.realpath(os.path.join(os.path.dirname(__file__), '..', '..', '..', '..'))
if ROOT_DEV_PATH not in sys.path:
@@ -29,7 +34,7 @@ if ROOT_DEV_PATH not in sys.path:
from cmake.Tools import common
def launch_ios_test(build_dir, target_dev_name, test_target, timeout_secs, test_filter, xctestrun_file):
def launch_ios_test(build_dir, target_dev_name, test_target, timeout_secs, test_filter, xctestrun_file, test_report_json_file):
build_path = pathlib.Path(build_dir) if os.path.isabs(build_dir) else pathlib.Path(ROOT_DEV_PATH) / build_dir
if not build_path.is_dir():
@@ -57,6 +62,11 @@ def launch_ios_test(build_dir, target_dev_name, test_target, timeout_secs, test_
test_targets = common.get_validated_test_modules(test_modules=test_target, build_dir_path=build_path)
test_run_contents = []
test_case_successes = []
test_case_fails = []
crashed_test_modules = []
test_case_count = 0
with open(test_run_file, 'rb') as fp:
test_run_contents = plistlib.load(fp)
@@ -74,7 +84,7 @@ def launch_ios_test(build_dir, target_dev_name, test_target, timeout_secs, test_
with open(test_run_file, 'wb') as fp:
plistlib.dump(test_run_contents, fp, sort_keys=False)
xcode_args = ['test-without-building', '-xctestrun', test_run_file, '-destination', f'platform=iOS,name={target_dev_name}']
xcode_args = ['test-without-building', '-xctestrun', test_run_file, '-destination', f'platform=iOS,name={target_dev_name}', '-allowProvisioningUpdates', '-allowProvisioningDeviceRegistration']
if timeout_secs < 0:
xcode_args.extend(['-test-timeouts-enabled', 'NO'])
else:
@@ -89,6 +99,22 @@ def launch_ios_test(build_dir, target_dev_name, test_target, timeout_secs, test_
test_output = False
while xcode_out.poll() is None:
line = xcode_out.stdout.readline()
if line.startswith('** TEST EXECUTE FAILED **'):
# The test run crashed, so we need to track the failed test module
crashed_test_modules.append(target)
else:
matched = TEST_RUN_SEARCH_PATTERN.search(line)
if matched:
test_case_action = matched.group(1).strip()
test_case_name = matched.group(2).strip()
test_case_elapsed = matched.group(3).strip() if len(matched.groups()) > 2 else ''
if test_case_action == 'OK':
test_case_successes.append(f'{test_case_name} {test_case_elapsed}')
test_case_count += 1
elif test_case_action == 'FAILED' and 'listed below:' not in test_case_name:
test_case_fails.append(f'{test_case_name} {test_case_elapsed}')
test_case_count += 1
if TEST_STARTED_STRING in line:
test_output = True
@@ -99,11 +125,32 @@ def launch_ios_test(build_dir, target_dev_name, test_target, timeout_secs, test_
test_output = False
if test_output:
print(line)
print(line, end='')
else:
logging.debug(line)
print(f'{target} Succeeded') if test_success else print(f'{target} Failed')
logging.info(f'{target} Succeeded') if test_success else print(f'{target} Failed')
if test_report_json_file:
test_report_json_path = pathlib.Path(test_report_json_file)
test_timestamp = time.strftime("%Y-%m-%dT%H:%M:%SZ", time.localtime())
result_dict = {
'index': 'ly_platforms.test',
'payload': {
'Git Success': True,
'Build Success': True,
'Passed': test_case_successes,
'Failed': test_case_fails,
'CrashedModules': crashed_test_modules,
'Count': test_case_count
},
'pipeline': 'filebeat',
'timestamp': test_timestamp
}
result_json = json.dumps(result_dict, indent=4)
test_report_json_path.write_text(result_json, encoding='UTF-8', errors='ignore')
logging.info(f'MARS report saved to {test_report_json_path}')
logging.info(f"MARS report:\n{result_json}")
def main(args):
@@ -140,20 +187,24 @@ def main(args):
help='Enable debug logging',
action='store_true')
parser.add_argument('--test-report-json',
help='The optional path to the test report json file that will be generated for MARS reporting',
default=None)
parsed_args = parser.parse_args(args)
logging.basicConfig(format='%(levelname)s: %(message)s',
level=logging.DEBUG if parsed_args.debug else logging.INFO)
result = launch_ios_test(build_dir=parsed_args.build_dir,
target_dev_name=parsed_args.device_name,
test_target=parsed_args.test_module,
timeout_secs=int(parsed_args.timeout),
test_filter=parsed_args.test_filter,
xctestrun_file=parsed_args.xctestrun_file)
target_dev_name=parsed_args.device_name,
test_target=parsed_args.test_module,
timeout_secs=int(parsed_args.timeout),
test_filter=parsed_args.test_filter,
xctestrun_file=parsed_args.xctestrun_file,
test_report_json_file=parsed_args.test_report_json)
return 0 if result else 1
if __name__ == '__main__':
try:
scripts/build/Platform/iOS/build_config.json
+19
View File
@@ -99,5 +99,24 @@
"CMAKE_TARGET": "ALL_BUILD",
"CMAKE_NATIVE_BUILD_ARGS": "-destination generic/platform=iOS"
}
},
"device_test_profile": {
"TAGS": [
],
"PIPELINE_ENV":{
"NODE_LABEL":"Catalina_DeviceFarm_1",
"LY_3RDPARTY_PATH": "/Volumes/Data/jenkins/3rdParty",
"WORKSPACE": "/Volumes/Data/jenkins/workspace"
},
"COMMAND": "./build_ios_test.sh",
"PARAMETERS": {
"CONFIGURATION": "profile",
"OUTPUT_DIRECTORY": "build/ios_test",
"CMAKE_OPTIONS": "-G Xcode -DCMAKE_TOOLCHAIN_FILE=cmake/Platform/iOS/Toolchain_ios.cmake -DLY_MONOLITHIC_GAME=FALSE -DCMAKE_XCODE_ATTRIBUTE_CODE_SIGNING_REQUIRED=TRUE -DLY_IOS_CODE_SIGNING_IDENTITY=\"\" -DCMAKE_XCODE_ATTRIBUTE_CODE_SIGN_ENTITLEMENTS=\"\" -DCMAKE_XCODE_ATTRIBUTE_CODE_SIGNING_ALLOWED=TRUE -DLY_UNITY_BUILD=TRUE -DO3DE_HOME_PATH=\"${WORKSPACE}/home\" -DO3DE_REGISTER_ENGINE_PATH=\"${WORKSPACE}/o3de\" -DO3DE_REGISTER_THIS_ENGINE=TRUE",
"CMAKE_TARGET": "ALL_BUILD",
"CMAKE_NATIVE_BUILD_ARGS": "",
"TARGET_DEVICE_NAME": "Lumberyard",
"TEST_MODULE_TIMEOUT": "600"
}
}
}
scripts/build/Platform/iOS/build_ios_test.sh
Executable
+76
View File
@@ -0,0 +1,76 @@
#!/usr/bin/env bash
#
# All or portions of this file Copyright (c) Amazon.com, Inc. or its affiliates or
# its licensors.
#
# For complete copyright and license terms please see the LICENSE at the root of this
# distribution (the "License"). All use of this software is governed by the License,
# or, if provided, by the license below or the license accompanying this file. Do not
# remove or modify any license notices. This file is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
#
set -o errexit # exit on the first failure encountered
BASEDIR=$(dirname "$0")
source $BASEDIR/../Mac/env_mac.sh
mkdir -p ${OUTPUT_DIRECTORY}
SOURCE_DIRECTORY=${PWD}
ECHO Configuring for iOS Testing
LAST_CONFIGURE_CMD_FILE=ci_last_configure_cmd.txt
CONFIGURE_CMD="cmake -B ${OUTPUT_DIRECTORY} ${SOURCE_DIRECTORY} ${CMAKE_OPTIONS} ${EXTRA_CMAKE_OPTIONS} -DLY_3RDPARTY_PATH=${LY_3RDPARTY_PATH}"
if [[ ! -e "CMakeCache.txt" ]]; then
echo [ci_build] First run, generating
RUN_CONFIGURE=1
elif [[ ! -e ${LAST_CONFIGURE_CMD_FILE} ]]; then
echo [ci_build] Last run command not found, generating
RUN_CONFIGURE=1
else
# Detect if the input has changed
LAST_CMD=$(<${LAST_CONFIGURE_CMD_FILE})
if [[ "${LAST_CMD}" != "${CONFIGURE_CMD}" ]]; then
echo [ci_build] Last run command different, generating
RUN_CONFIGURE=1
fi
fi
if [[ ! -z "$RUN_CONFIGURE" ]]; then
# have to use eval since $CMAKE_OPTIONS (${EXTRA_CMAKE_OPTIONS}) contains quotes that need to be processed
echo [ci_build] ${CONFIGURE_CMD}
eval ${CONFIGURE_CMD}
# Save the run only if success
echo "${CONFIGURE_CMD}" > ${LAST_CONFIGURE_CMD_FILE}
fi
if [ $? -ne 0 ]
then
echo "CMake configuration failed"
exit 1
fi
echo Building for iOS Testing
echo
echo [ci+build] ${SOURCE_DIRECTORY}/python/python.sh ${SOURCE_DIRECTORY}/cmake/Tools/Platform/iOS/build_ios_test.py -b ${OUTPUT_DIRECTORY} -c ${CONFIGURATION}
${SOURCE_DIRECTORY}/python/python.sh ${SOURCE_DIRECTORY}/cmake/Tools/Platform/iOS/build_ios_test.py -b ${OUTPUT_DIRECTORY} -c ${CONFIGURATION}
if [ $? -ne 0 ]
then
echo "iOS Test Build failed"
exit 1
fi
ECHO Launching iOS Test
echo [ci+build] ${SOURCE_DIRECTORY}/python/python.sh ${SOURCE_DIRECTORY}/cmake/Tools/Platform/iOS/launch_ios_test.py -b ${OUTPUT_DIRECTORY} --device-name ${TARGET_DEVICE_NAME} --timeout ${TEST_MODULE_TIMEOUT}
${SOURCE_DIRECTORY}/python/python.sh ${SOURCE_DIRECTORY}/cmake/Tools/Platform/iOS/launch_ios_test.py -b ${OUTPUT_DIRECTORY} --device-name ${TARGET_DEVICE_NAME} --timeout ${TEST_MODULE_TIMEOUT}
if [ $? -ne 0 ]
then
echo "iOS Test failed"
exit 1
fi
echo "iOS Tests passed"
exit 0