You cannot select more than 25 topics
Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.
251 lines
11 KiB
Python
251 lines
11 KiB
Python
"""
|
|
Copyright (c) Contributors to the Open 3D Engine Project.
|
|
For complete copyright and license terms please see the LICENSE at the root of this distribution.
|
|
|
|
SPDX-License-Identifier: Apache-2.0 OR MIT
|
|
|
|
A wrapper to simplify invoking CTest with common parameters and sub-filter to specific suites
|
|
"""
|
|
import argparse
|
|
import multiprocessing
|
|
import os
|
|
import result_processing.result_processing as rp
|
|
import subprocess
|
|
import sys
|
|
import shutil
|
|
|
|
SUITES_AND_DESCRIPTIONS = {
|
|
"smoke": "Quick across-the-board set of tests designed to check if something is fundamentally broken",
|
|
"main": "The default set of tests, covers most of all testing.",
|
|
"periodic": "Tests which can take a long time and should be done periodially instead of every commit - these should not block code submission",
|
|
"benchmark": "Benchmarks - instead of pass/fail, these collect data for comparison against historic data",
|
|
"sandbox": "Flaky/Intermittent failing tests, this is used as a temporary spot to hold flaky tests, this will not block code submission. Ideally, this suite should always be empty"
|
|
}
|
|
|
|
BUILD_CONFIGURATIONS = [
|
|
"profile",
|
|
"debug",
|
|
"release",
|
|
]
|
|
|
|
|
|
def _regex_matching_any(words):
|
|
"""
|
|
:param words: iterable of strings to match
|
|
:return: a regex with groups to match each string
|
|
"""
|
|
return "^(" + "|".join(words) + ")$"
|
|
|
|
|
|
def run_single_test_suite(suite, ctest_path, cmake_build_path, build_config, disable_gpu, only_gpu, generate_xml, repeat, extra_args):
|
|
"""
|
|
Starts CTest to filter down to a specific suite
|
|
:param suite: subset of tests to run, see SUITES_AND_DESCRIPTIONS
|
|
:param ctest_path: path to ctest.exe
|
|
:param cmake_build_path: path to build output
|
|
:param build_config: cmake build variant to select
|
|
:param disable_gpu: optional, run only non-gpu tests
|
|
:param only_gpu: optional, run only gpu-required tests
|
|
:param generate_xml: optional, enable to produce the CTest xml file
|
|
:param repeat: optional, number of times to run the tests in the suite
|
|
:param extrargs: optional, forward args to ctest
|
|
:return: CTest exit code
|
|
"""
|
|
ctest_command = [
|
|
ctest_path,
|
|
"--build-config", build_config,
|
|
"--output-on-failure",
|
|
"--parallel", str(multiprocessing.cpu_count()), # leave serial vs parallel scheduling to CTest via set_tests_properties()
|
|
"--no-tests=error",
|
|
]
|
|
|
|
label_excludes = []
|
|
label_includes = []
|
|
|
|
# ctest can't actually do "AND" queries in label include and name-include, if any
|
|
# match, it will accept them. In addition, the regex language it uses does
|
|
# not include positive lookahead to use workarounds...
|
|
# So if someone is asking for "main" AND "requires_gpu"
|
|
# the only way to do this is to exclude all OTHER suites
|
|
# to solve this problem generally, we will always exclude all other suites than
|
|
# the one being tested.
|
|
|
|
for label_name in SUITES_AND_DESCRIPTIONS.keys():
|
|
if label_name != suite:
|
|
label_excludes.append(f"SUITE_{label_name}")
|
|
|
|
# only one of these can be true, or neither. If neither, we apply no REQUIRES_* filter.
|
|
if only_gpu:
|
|
label_includes.append("REQUIRES_gpu")
|
|
elif disable_gpu:
|
|
label_excludes.append("REQUIRES_gpu")
|
|
|
|
union_regex = _regex_matching_any(label_includes) if label_includes else None
|
|
difference_regex = _regex_matching_any(label_excludes) if label_excludes else None
|
|
|
|
if union_regex:
|
|
ctest_command.append("--label-regex")
|
|
ctest_command.append(union_regex)
|
|
if difference_regex:
|
|
ctest_command.append("--label-exclude")
|
|
ctest_command.append(difference_regex)
|
|
if generate_xml:
|
|
ctest_command.append('-T')
|
|
ctest_command.append('Test')
|
|
|
|
for extra_arg in extra_args:
|
|
ctest_command.append(extra_arg)
|
|
|
|
ctest_command_string = ' '.join(ctest_command) # ONLY used for display
|
|
print(f"Executing CTest {repeat} time(s) with command:\n"
|
|
f" {ctest_command_string}\n"
|
|
"in working directory:\n"
|
|
f" {cmake_build_path}\n")
|
|
|
|
error_code = 0
|
|
|
|
if repeat:
|
|
# Run the tests multiple times. Previous test results are deleted, new test results are combined in a file per
|
|
# test runner.
|
|
|
|
test_result_prefix = 'Repeat'
|
|
repeat = int(repeat)
|
|
if generate_xml:
|
|
rp.clean_test_results(cmake_build_path)
|
|
|
|
for iteration in range(repeat):
|
|
print(f"Executing CTest iteration {iteration + 1}/{repeat}")
|
|
result = subprocess.run(ctest_command, shell=False, cwd=cmake_build_path, stdout=sys.stdout, stderr=sys.stderr)
|
|
|
|
if generate_xml:
|
|
rp.rename_test_results(cmake_build_path, test_result_prefix, iteration + 1, repeat)
|
|
if result.returncode:
|
|
error_code = result.returncode
|
|
|
|
if generate_xml:
|
|
rp.collect_test_results(cmake_build_path, test_result_prefix)
|
|
summary = rp.summarize_test_results(cmake_build_path, repeat)
|
|
|
|
print() # empty line
|
|
print('Test stability summary:')
|
|
if summary:
|
|
print('The following test(s) failed:')
|
|
for line in summary: print(line)
|
|
else:
|
|
print(f'All tests were executed {repeat} times and passed 100% of the time.')
|
|
|
|
else:
|
|
# Run the tests one time. Previous test results are not deleted but
|
|
# might be overwritten.
|
|
result = subprocess.run(ctest_command, shell=False, cwd=cmake_build_path, stdout=sys.stdout, stderr=sys.stderr)
|
|
error_code = result.returncode
|
|
|
|
return error_code
|
|
|
|
|
|
def main():
|
|
# establish defaults
|
|
ctest_version = "3.17.0"
|
|
if sys.platform == "win32":
|
|
ctest_build = "Windows"
|
|
ctest_relpath = "bin"
|
|
ctest_exe = "ctest.exe"
|
|
elif sys.platform.startswith("linux"):
|
|
ctest_build = "Linux"
|
|
ctest_relpath = "bin"
|
|
ctest_exe = "ctest"
|
|
elif sys.platform.startswith('darwin'):
|
|
ctest_build = "Mac"
|
|
ctest_relpath = "CMake.app/Contents/bin"
|
|
ctest_exe = "ctest"
|
|
else:
|
|
raise NotImplementedError(f"CTest is not currently configured for platform '{sys.platform}'")
|
|
|
|
current_script_path = os.path.dirname(__file__)
|
|
dev_default = os.path.dirname(current_script_path)
|
|
thirdparty_default = os.path.join(os.path.dirname(dev_default), "3rdParty")
|
|
|
|
# if a specific known location contains cmake, we'll use it
|
|
ctest_default = os.path.join(thirdparty_default, "CMake", ctest_version, ctest_build, ctest_relpath, ctest_exe)
|
|
|
|
# parse args, with defaults
|
|
parser = argparse.ArgumentParser(
|
|
description="CTest CLI driver: simplifies providing common arguments to CTest",
|
|
# extra wide help messages to avoid newlines appearing in path defaults, which break copy-paste of paths
|
|
formatter_class=lambda prog: argparse.ArgumentDefaultsHelpFormatter(prog, width=4096),
|
|
epilog="(Unrecognised parameters will be sent to ctest directly)"
|
|
)
|
|
|
|
parser.add_argument('-x', '--ctest-executable',
|
|
help="Override path to the CTest executable (will use PATH env otherwise)")
|
|
parser.add_argument('-B', '--build-path', # -B to match cmake's syntax for same thing.
|
|
help="Path to a CMake build folder (generated by running cmake).",
|
|
required=True)
|
|
parser.add_argument('--config', choices=BUILD_CONFIGURATIONS, default="debug", # --config to match cmake
|
|
help="CMake variant build configuration to target (debug/profile/release)")
|
|
parser.add_argument('-s', '--suite', choices=SUITES_AND_DESCRIPTIONS.keys(),
|
|
default="main",
|
|
help="Which subset of tests to execute")
|
|
parser.add_argument('--generate-xml', action='store_true',
|
|
help='Enable this option to produce the CTest xml file.')
|
|
parser.add_argument('-r', '--repeat', help="Run the tests the specified number times to identify intermittent "
|
|
"failures (e.g. --repeat 3 for running the test three times). When used"
|
|
"with --generate-xml, the resulting test reports will be combined, "
|
|
"aggregated and summarized.", type=int)
|
|
group = parser.add_mutually_exclusive_group()
|
|
group.add_argument('--no-gpu', action='store_true',
|
|
help="Disable tests that require a GPU")
|
|
group.add_argument('--only-gpu', action='store_true',
|
|
help="Run only tests that require a GPU")
|
|
|
|
args, unknown_args = parser.parse_known_args()
|
|
|
|
# handle the CTEST executable.
|
|
# we always obey command line, and its an error if the command line has
|
|
# a bad executable
|
|
# if no command line is specified, it will fallback to a known good location
|
|
# and then finally, use the PATH.
|
|
if args.ctest_executable and not os.path.exists(args.ctest_executable):
|
|
print(f"Error: Invalid ctest executable specified - not found: {args.ctest_executable}")
|
|
return 1
|
|
if not args.ctest_executable:
|
|
# try the default
|
|
if os.path.exists(ctest_default):
|
|
print(f"Using default CTest executable: {ctest_default}")
|
|
args.ctest_executable = ctest_default
|
|
else: # try the PATH env var:
|
|
found_ctest = shutil.which(ctest_exe)
|
|
if found_ctest:
|
|
print(f"Using CTest executable from PATH: {found_ctest}")
|
|
args.ctest_executable = found_ctest
|
|
else:
|
|
print(f"Could not find CTest Executable ('{ctest_exe}')on PATH or in a pre-set location.")
|
|
return 1
|
|
|
|
# handle the build path. You must specify a build path, and it must contain CTestTestfile.cmake
|
|
if not os.path.exists(args.build_path):
|
|
print(f"Error: specified folder does not exist: {args.build_path}")
|
|
return 1
|
|
ctest_testfile = os.path.join(args.build_path, "CTestTestfile.cmake")
|
|
if not os.path.exists(ctest_testfile):
|
|
print(f"Error: '{ctest_testfile}' missing, run CMake configure+generate on the folder first.")
|
|
return 1
|
|
|
|
print(f"Starting '{args.suite}' suite: {SUITES_AND_DESCRIPTIONS[args.suite]}")
|
|
|
|
# execute
|
|
return run_single_test_suite(
|
|
suite=args.suite,
|
|
ctest_path=args.ctest_executable,
|
|
cmake_build_path=args.build_path,
|
|
build_config=args.config,
|
|
disable_gpu=args.no_gpu,
|
|
only_gpu=args.only_gpu,
|
|
generate_xml=args.generate_xml,
|
|
repeat=args.repeat,
|
|
extra_args=unknown_args)
|
|
|
|
|
|
if __name__ == "__main__":
|
|
sys.exit(main())
|