SPEC-5070 Move ctest_scripts to scripts/ctest
* removing unused function and moving ctest_scripts to scripts/ctest * Re-adding ebp-test * Fixing typo that is making this test run in parallel with other tests * Fixing hang when parameters are passed * passing absolute path as a project * small tweak to not print out during Python execution * Moving the timeout to be in the build step * Disable ebo_sanity_smoke_no_gpu Co-authored-by: jackalbe <23512001+jackalbe@users.noreply.github.com>
This commit is contained in:
Esteban Papp
GitHub
@@ -12,3 +12,4 @@
|
||||
add_subdirectory(detect_file_changes)
|
||||
add_subdirectory(commit_validation)
|
||||
add_subdirectory(project_manager)
|
||||
add_subdirectory(ctest)
|
||||
|
||||
@@ -0,0 +1,89 @@
|
||||
#
|
||||
# All or portions of this file Copyright (c) Amazon.com, Inc. or its affiliates or
|
||||
# its licensors.
|
||||
#
|
||||
# For complete copyright and license terms please see the LICENSE at the root of this
|
||||
# distribution (the "License"). All use of this software is governed by the License,
|
||||
# or, if provided, by the license below or the license accompanying this file. Do not
|
||||
# remove or modify any license notices. This file is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
#
|
||||
|
||||
# Currently a sanity test is being registered here to validate that the ly_add_pytest function works
|
||||
# don't change the names of the tests, they are used in a self test.
|
||||
|
||||
if(NOT PAL_TRAIT_BUILD_TESTS_SUPPORTED)
|
||||
return()
|
||||
endif()
|
||||
|
||||
################################################################################
|
||||
# Asset Processing Target
|
||||
# i.e. Tests depend on AutomatedTesting.Assets
|
||||
################################################################################
|
||||
|
||||
if(PAL_TRAIT_BUILD_TESTS_SUPPORTED AND PAL_TRAIT_BUILD_HOST_TOOLS)
|
||||
get_property(LY_PROJECTS_TARGET_NAME GLOBAL PROPERTY LY_PROJECTS_TARGET_NAME)
|
||||
foreach(project_target_name project_path IN ZIP_LISTS LY_PROJECTS_TARGET_NAME LY_PROJECTS)
|
||||
file(REAL_PATH ${project_path} project_real_path BASE_DIRECTORY ${LY_ROOT_FOLDER})
|
||||
# With the lock file, asset processing jobs are serialized to avoid race conditions
|
||||
# on files that are created temporarily in source folders during shader processing.
|
||||
add_custom_target(${project_target_name}.Assets
|
||||
COMMENT "Processing ${project_target_name} assets..."
|
||||
COMMAND "${CMAKE_COMMAND}"
|
||||
-DLY_LOCK_FILE=$<TARGET_FILE_DIR:AZ::AssetProcessorBatch>/project_assets.lock
|
||||
-P ${LY_ROOT_FOLDER}/cmake/CommandExecution.cmake
|
||||
EXEC_COMMAND $<TARGET_FILE:AZ::AssetProcessorBatch>
|
||||
--zeroAnalysisMode
|
||||
--project-path=${project_real_path}
|
||||
--platforms=${LY_ASSET_DEPLOY_ASSET_TYPE}
|
||||
)
|
||||
set_target_properties(${project_target_name}.Assets
|
||||
PROPERTIES
|
||||
EXCLUDE_FROM_ALL TRUE
|
||||
FOLDER ${project_target_name}
|
||||
)
|
||||
endforeach()
|
||||
endif()
|
||||
|
||||
################################################################################
|
||||
# Tests
|
||||
################################################################################
|
||||
|
||||
foreach(suite_name ${LY_TEST_GLOBAL_KNOWN_SUITE_NAMES})
|
||||
ly_add_pytest(
|
||||
NAME pytest_sanity_${suite_name}_no_gpu
|
||||
PATH ${CMAKE_CURRENT_LIST_DIR}/sanity_test.py
|
||||
TEST_SUITE ${suite_name}
|
||||
)
|
||||
|
||||
ly_add_pytest(
|
||||
NAME pytest_sanity_${suite_name}_requires_gpu
|
||||
PATH ${CMAKE_CURRENT_LIST_DIR}/sanity_test.py
|
||||
TEST_SUITE ${suite_name}
|
||||
TEST_REQUIRES gpu
|
||||
)
|
||||
endforeach()
|
||||
|
||||
# EPB Sanity test is being registered here to validate that the ly_add_editor_python_test function works.
|
||||
#if(PAL_TRAIT_BUILD_HOST_TOOLS AND PAL_TRAIT_BUILD_TESTS_SUPPORTED AND AutomatedTesting IN_LIST LY_PROJECTS_TARGET_NAME)
|
||||
# ly_add_editor_python_test(
|
||||
# NAME epb_sanity_smoke_no_gpu
|
||||
# TEST_PROJECT AutomatedTesting
|
||||
# PATH ${CMAKE_CURRENT_LIST_DIR}/epb_sanity_test.py
|
||||
# TEST_SUITE smoke
|
||||
# TEST_SERIAL TRUE
|
||||
# RUNTIME_DEPENDENCIES
|
||||
# AutomatedTesting.Assets
|
||||
# )
|
||||
#endif()
|
||||
|
||||
# add a custom test which makes sure that the test filtering works!
|
||||
|
||||
ly_add_test(
|
||||
NAME cli_test_driver
|
||||
EXCLUDE_TEST_RUN_TARGET_FROM_IDE
|
||||
TEST_COMMAND ${LY_PYTHON_CMD} ${CMAKE_CURRENT_LIST_DIR}/ctest_driver_test.py
|
||||
-x ${CMAKE_CTEST_COMMAND}
|
||||
--build-path ${CMAKE_BINARY_DIR}
|
||||
)
|
||||
|
||||
Executable
+254
@@ -0,0 +1,254 @@
|
||||
"""
|
||||
All or portions of this file Copyright (c) Amazon.com, Inc. or its affiliates or
|
||||
its licensors.
|
||||
|
||||
For complete copyright and license terms please see the LICENSE at the root of this
|
||||
distribution (the "License"). All use of this software is governed by the License,
|
||||
or, if provided, by the license below or the license accompanying this file. Do not
|
||||
remove or modify any license notices. This file is distributed on an "AS IS" BASIS,
|
||||
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
|
||||
A wrapper to simplify invoking CTest with common parameters and sub-filter to specific suites
|
||||
"""
|
||||
import argparse
|
||||
import multiprocessing
|
||||
import os
|
||||
import result_processing.result_processing as rp
|
||||
import subprocess
|
||||
import sys
|
||||
import shutil
|
||||
|
||||
SUITES_AND_DESCRIPTIONS = {
|
||||
"smoke": "Quick across-the-board set of tests designed to check if something is fundamentally broken",
|
||||
"main": "The default set of tests, covers most of all testing.",
|
||||
"periodic": "Tests which can take a long time and should be done periodially instead of every commit - these should not block code submission",
|
||||
"benchmark": "Benchmarks - instead of pass/fail, these collect data for comparison against historic data",
|
||||
"sandbox": "Flaky/Intermittent failing tests, this is used as a temporary spot to hold flaky tests, this will not block code submission. Ideally, this suite should always be empty"
|
||||
}
|
||||
|
||||
BUILD_CONFIGURATIONS = [
|
||||
"profile",
|
||||
"debug",
|
||||
"release",
|
||||
]
|
||||
|
||||
|
||||
def _regex_matching_any(words):
|
||||
"""
|
||||
:param words: iterable of strings to match
|
||||
:return: a regex with groups to match each string
|
||||
"""
|
||||
return "^(" + "|".join(words) + ")$"
|
||||
|
||||
|
||||
def run_single_test_suite(suite, ctest_path, cmake_build_path, build_config, disable_gpu, only_gpu, generate_xml, repeat, extra_args):
|
||||
"""
|
||||
Starts CTest to filter down to a specific suite
|
||||
:param suite: subset of tests to run, see SUITES_AND_DESCRIPTIONS
|
||||
:param ctest_path: path to ctest.exe
|
||||
:param cmake_build_path: path to build output
|
||||
:param build_config: cmake build variant to select
|
||||
:param disable_gpu: optional, run only non-gpu tests
|
||||
:param only_gpu: optional, run only gpu-required tests
|
||||
:param generate_xml: optional, enable to produce the CTest xml file
|
||||
:param repeat: optional, number of times to run the tests in the suite
|
||||
:param extrargs: optional, forward args to ctest
|
||||
:return: CTest exit code
|
||||
"""
|
||||
ctest_command = [
|
||||
ctest_path,
|
||||
"--build-config", build_config,
|
||||
"--output-on-failure",
|
||||
"--parallel", str(multiprocessing.cpu_count()), # leave serial vs parallel scheduling to CTest via set_tests_properties()
|
||||
"--no-tests=error",
|
||||
]
|
||||
|
||||
label_excludes = []
|
||||
label_includes = []
|
||||
|
||||
# ctest can't actually do "AND" queries in label include and name-include, if any
|
||||
# match, it will accept them. In addition, the regex language it uses does
|
||||
# not include positive lookahead to use workarounds...
|
||||
# So if someone is asking for "main" AND "requires_gpu"
|
||||
# the only way to do this is to exclude all OTHER suites
|
||||
# to solve this problem generally, we will always exclude all other suites than
|
||||
# the one being tested.
|
||||
|
||||
for label_name in SUITES_AND_DESCRIPTIONS.keys():
|
||||
if label_name != suite:
|
||||
label_excludes.append(f"SUITE_{label_name}")
|
||||
|
||||
# only one of these can be true, or neither. If neither, we apply no REQUIRES_* filter.
|
||||
if only_gpu:
|
||||
label_includes.append("REQUIRES_gpu")
|
||||
elif disable_gpu:
|
||||
label_excludes.append("REQUIRES_gpu")
|
||||
|
||||
union_regex = _regex_matching_any(label_includes) if label_includes else None
|
||||
difference_regex = _regex_matching_any(label_excludes) if label_excludes else None
|
||||
|
||||
if union_regex:
|
||||
ctest_command.append("--label-regex")
|
||||
ctest_command.append(union_regex)
|
||||
if difference_regex:
|
||||
ctest_command.append("--label-exclude")
|
||||
ctest_command.append(difference_regex)
|
||||
if generate_xml:
|
||||
ctest_command.append('-T')
|
||||
ctest_command.append('Test')
|
||||
|
||||
for extra_arg in extra_args:
|
||||
ctest_command.append(extra_arg)
|
||||
|
||||
ctest_command_string = ' '.join(ctest_command) # ONLY used for display
|
||||
print(f"Executing CTest {repeat} time(s) with command:\n"
|
||||
f" {ctest_command_string}\n"
|
||||
"in working directory:\n"
|
||||
f" {cmake_build_path}\n")
|
||||
|
||||
error_code = 0
|
||||
|
||||
if repeat:
|
||||
# Run the tests multiple times. Previous test results are deleted, new test results are combined in a file per
|
||||
# test runner.
|
||||
|
||||
test_result_prefix = 'Repeat'
|
||||
repeat = int(repeat)
|
||||
if generate_xml:
|
||||
rp.clean_test_results(cmake_build_path)
|
||||
|
||||
for iteration in range(repeat):
|
||||
print(f"Executing CTest iteration {iteration + 1}/{repeat}")
|
||||
result = subprocess.run(ctest_command, shell=False, cwd=cmake_build_path, stdout=sys.stdout, stderr=sys.stderr)
|
||||
|
||||
if generate_xml:
|
||||
rp.rename_test_results(cmake_build_path, test_result_prefix, iteration + 1, repeat)
|
||||
if result.returncode:
|
||||
error_code = result.returncode
|
||||
|
||||
if generate_xml:
|
||||
rp.collect_test_results(cmake_build_path, test_result_prefix)
|
||||
summary = rp.summarize_test_results(cmake_build_path, repeat)
|
||||
|
||||
print() # empty line
|
||||
print('Test stability summary:')
|
||||
if summary:
|
||||
print('The following test(s) failed:')
|
||||
for line in summary: print(line)
|
||||
else:
|
||||
print(f'All tests were executed {repeat} times and passed 100% of the time.')
|
||||
|
||||
else:
|
||||
# Run the tests one time. Previous test results are not deleted but
|
||||
# might be overwritten.
|
||||
result = subprocess.run(ctest_command, shell=False, cwd=cmake_build_path, stdout=sys.stdout, stderr=sys.stderr)
|
||||
error_code = result.returncode
|
||||
|
||||
return error_code
|
||||
|
||||
|
||||
def main():
|
||||
# establish defaults
|
||||
ctest_version = "3.17.0"
|
||||
if sys.platform == "win32":
|
||||
ctest_build = "Windows"
|
||||
ctest_relpath = "bin"
|
||||
ctest_exe = "ctest.exe"
|
||||
elif sys.platform.startswith("linux"):
|
||||
ctest_build = "Linux"
|
||||
ctest_relpath = "bin"
|
||||
ctest_exe = "ctest"
|
||||
elif sys.platform.startswith('darwin'):
|
||||
ctest_build = "Mac"
|
||||
ctest_relpath = "CMake.app/Contents/bin"
|
||||
ctest_exe = "ctest"
|
||||
else:
|
||||
raise NotImplementedError(f"CTest is not currently configured for platform '{sys.platform}'")
|
||||
|
||||
current_script_path = os.path.dirname(__file__)
|
||||
dev_default = os.path.dirname(current_script_path)
|
||||
thirdparty_default = os.path.join(os.path.dirname(dev_default), "3rdParty")
|
||||
|
||||
# if a specific known location contains cmake, we'll use it
|
||||
ctest_default = os.path.join(thirdparty_default, "CMake", ctest_version, ctest_build, ctest_relpath, ctest_exe)
|
||||
|
||||
# parse args, with defaults
|
||||
parser = argparse.ArgumentParser(
|
||||
description="CTest CLI driver: simplifies providing common arguments to CTest",
|
||||
# extra wide help messages to avoid newlines appearing in path defaults, which break copy-paste of paths
|
||||
formatter_class=lambda prog: argparse.ArgumentDefaultsHelpFormatter(prog, width=4096),
|
||||
epilog="(Unrecognised parameters will be sent to ctest directly)"
|
||||
)
|
||||
|
||||
parser.add_argument('-x', '--ctest-executable',
|
||||
help="Override path to the CTest executable (will use PATH env otherwise)")
|
||||
parser.add_argument('-B', '--build-path', # -B to match cmake's syntax for same thing.
|
||||
help="Path to a CMake build folder (generated by running cmake).",
|
||||
required=True)
|
||||
parser.add_argument('--config', choices=BUILD_CONFIGURATIONS, default="debug", # --config to match cmake
|
||||
help="CMake variant build configuration to target (debug/profile/release)")
|
||||
parser.add_argument('-s', '--suite', choices=SUITES_AND_DESCRIPTIONS.keys(),
|
||||
default="main",
|
||||
help="Which subset of tests to execute")
|
||||
parser.add_argument('--generate-xml', action='store_true',
|
||||
help='Enable this option to produce the CTest xml file.')
|
||||
parser.add_argument('-r', '--repeat', help="Run the tests the specified number times to identify intermittent "
|
||||
"failures (e.g. --repeat 3 for running the test three times). When used"
|
||||
"with --generate-xml, the resulting test reports will be combined, "
|
||||
"aggregated and summarized.", type=int)
|
||||
group = parser.add_mutually_exclusive_group()
|
||||
group.add_argument('--no-gpu', action='store_true',
|
||||
help="Disable tests that require a GPU")
|
||||
group.add_argument('--only-gpu', action='store_true',
|
||||
help="Run only tests that require a GPU")
|
||||
|
||||
args, unknown_args = parser.parse_known_args()
|
||||
|
||||
# handle the CTEST executable.
|
||||
# we always obey command line, and its an error if the command line has
|
||||
# a bad executable
|
||||
# if no command line is specified, it will fallback to a known good location
|
||||
# and then finally, use the PATH.
|
||||
if args.ctest_executable and not os.path.exists(args.ctest_executable):
|
||||
print(f"Error: Invalid ctest executable specified - not found: {args.ctest_executable}")
|
||||
return 1
|
||||
if not args.ctest_executable:
|
||||
# try the default
|
||||
if os.path.exists(ctest_default):
|
||||
print(f"Using default CTest executable: {ctest_default}")
|
||||
args.ctest_executable = ctest_default
|
||||
else: # try the PATH env var:
|
||||
found_ctest = shutil.which(ctest_exe)
|
||||
if found_ctest:
|
||||
print(f"Using CTest executable from PATH: {found_ctest}")
|
||||
args.ctest_executable = found_ctest
|
||||
else:
|
||||
print(f"Could not find CTest Executable ('{ctest_exe}')on PATH or in a pre-set location.")
|
||||
return 1
|
||||
|
||||
# handle the build path. You must specify a build path, and it must contain CTestTestfile.cmake
|
||||
if not os.path.exists(args.build_path):
|
||||
print(f"Error: specified folder does not exist: {args.build_path}")
|
||||
return 1
|
||||
ctest_testfile = os.path.join(args.build_path, "CTestTestfile.cmake")
|
||||
if not os.path.exists(ctest_testfile):
|
||||
print(f"Error: '{ctest_testfile}' missing, run CMake configure+generate on the folder first.")
|
||||
return 1
|
||||
|
||||
print(f"Starting '{args.suite}' suite: {SUITES_AND_DESCRIPTIONS[args.suite]}")
|
||||
|
||||
# execute
|
||||
return run_single_test_suite(
|
||||
suite=args.suite,
|
||||
ctest_path=args.ctest_executable,
|
||||
cmake_build_path=args.build_path,
|
||||
build_config=args.config,
|
||||
disable_gpu=args.no_gpu,
|
||||
only_gpu=args.only_gpu,
|
||||
generate_xml=args.generate_xml,
|
||||
repeat=args.repeat,
|
||||
extra_args=unknown_args)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
sys.exit(main())
|
||||
Executable
+87
@@ -0,0 +1,87 @@
|
||||
"""
|
||||
All or portions of this file Copyright (c) Amazon.com, Inc. or its affiliates or
|
||||
its licensors.
|
||||
|
||||
For complete copyright and license terms please see the LICENSE at the root of this
|
||||
distribution (the "License"). All use of this software is governed by the License,
|
||||
or, if provided, by the license below or the license accompanying this file. Do not
|
||||
remove or modify any license notices. This file is distributed on an "AS IS" BASIS,
|
||||
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
|
||||
Self-tests for ctest_driver.py.
|
||||
|
||||
Expects you to have generated a folder using cmake and to pass in that folder name
|
||||
as the only param
|
||||
"""
|
||||
import os
|
||||
import subprocess
|
||||
import sys
|
||||
import argparse
|
||||
from ctest_driver import SUITES_AND_DESCRIPTIONS
|
||||
|
||||
def main(build_path, ctest_executable):
|
||||
script_folder = os.path.dirname(__file__)
|
||||
# -N prevents tests from running, just lists them:
|
||||
base_args = [sys.executable, os.path.join(script_folder,'ctest_driver.py'), "--build-path", build_path, '-N']
|
||||
if ctest_executable:
|
||||
base_args.append("--ctest-executable")
|
||||
base_args.append(ctest_executable)
|
||||
|
||||
base_args.append("-R") # limit it to only the sanity tests, faster
|
||||
base_args.append("pytest_sanity_.*")
|
||||
|
||||
for suite in SUITES_AND_DESCRIPTIONS:
|
||||
for gpu_option in [None, True, False]:
|
||||
args_to_send = base_args.copy()
|
||||
args_to_send.append("--suite")
|
||||
args_to_send.append(suite)
|
||||
print(f"----- Test case: [suite = {suite}, gpu = {gpu_option}] ----")
|
||||
if gpu_option is not None:
|
||||
if gpu_option:
|
||||
args_to_send.append("--only-gpu")
|
||||
else:
|
||||
args_to_send.append("--no-gpu")
|
||||
result = subprocess.check_output(args_to_send, shell=False, cwd=build_path, stderr=sys.stderr)
|
||||
output = result.decode('utf-8')
|
||||
|
||||
# ensure that the appropriate suites are filtered in and out
|
||||
for suite_name in SUITES_AND_DESCRIPTIONS.keys():
|
||||
if (f"pytest_sanity_{suite_name}_" in output and suite != suite_name):
|
||||
print(output)
|
||||
print("Test failed - executed a test not in the currently selected suite.")
|
||||
return -1
|
||||
|
||||
gpu_test_present = "_requires_gpu" in output
|
||||
non_gpu_test_present = "_no_gpu" in output
|
||||
|
||||
if gpu_option is not None:
|
||||
if gpu_option and non_gpu_test_present:
|
||||
print("Test failed - required gpu only, and a non-gpu tes was run")
|
||||
print(output)
|
||||
return -1
|
||||
|
||||
if not gpu_option and gpu_test_present:
|
||||
print("Test failed - executed a gpu test when forbidden")
|
||||
print(output)
|
||||
return -1
|
||||
else:
|
||||
if not (gpu_test_present and non_gpu_test_present):
|
||||
print("Test failed - expected both kinds of tests (gpu and non gpu)")
|
||||
print(output)
|
||||
return -1
|
||||
|
||||
print("All passed.")
|
||||
return 0
|
||||
|
||||
if __name__ == '__main__':
|
||||
parser = argparse.ArgumentParser(
|
||||
description="CTest CLI driver self-tests.")
|
||||
parser.add_argument('-x', '--ctest-executable',
|
||||
help="Override path to the CTest executable (will use PATH env otherwise)")
|
||||
parser.add_argument('-b', '--build-path',
|
||||
required=True,
|
||||
help="Path to a CMake build folder (generated by running cmake)")
|
||||
args = parser.parse_args()
|
||||
sys.exit(main(args.build_path, args.ctest_executable))
|
||||
|
||||
|
||||
@@ -0,0 +1,22 @@
|
||||
@ECHO OFF
|
||||
REM
|
||||
REM All or portions of this file Copyright (c) Amazon.com, Inc. or its affiliates or
|
||||
REM its licensors.
|
||||
REM
|
||||
REM For complete copyright and license terms please see the LICENSE at the root of this
|
||||
REM distribution (the "License"). All use of this software is governed by the License,
|
||||
REM or, if provided, by the license below or the license accompanying this file. Do not
|
||||
REM remove or modify any license notices. This file is distributed on an "AS IS" BASIS,
|
||||
REM WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
REM
|
||||
REM Continuous Integration CLI entrypoint script to start CTest, triggering post-build tests
|
||||
REM
|
||||
SETLOCAL
|
||||
|
||||
SET DEV_DIR=%~dp0\..
|
||||
SET PYTHON=%DEV_DIR%\python\python.cmd
|
||||
SET CTEST_SCRIPT=%~dp0\ctest_driver.py
|
||||
|
||||
REM pass all args to python-based CTest script
|
||||
call %PYTHON% %CTEST_SCRIPT% %*
|
||||
exit /b %ERRORLEVEL%
|
||||
Executable
+23
@@ -0,0 +1,23 @@
|
||||
#!/bin/bash
|
||||
|
||||
#
|
||||
# All or portions of this file Copyright (c) Amazon.com, Inc. or its affiliates or
|
||||
# its licensors.
|
||||
#
|
||||
# For complete copyright and license terms please see the LICENSE at the root of this
|
||||
# distribution (the "License"). All use of this software is governed by the License,
|
||||
# or, if provided, by the license below or the license accompanying this file. Do not
|
||||
# remove or modify any license notices. This file is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
#
|
||||
# Continuous Integration CLI entrypoint script to start CTest, triggering post-build tests
|
||||
#
|
||||
|
||||
CURRENT_SCRIPT_DIR="$( cd "$( dirname "${BASH_SOURCE:-0}" )" >/dev/null 2>&1 && pwd )"
|
||||
DEV_DIR="$( dirname "$CURRENT_SCRIPT_DIR" )"
|
||||
PYTHON=$DEV_DIR/python/python.sh
|
||||
CTEST_SCRIPT=$CURRENT_SCRIPT_DIR/ctest_driver.py
|
||||
|
||||
# pass all args to python-based CTest script
|
||||
echo "Invoking: $PYTHON $CTEST_SCRIPT $*"
|
||||
$PYTHON $CTEST_SCRIPT $*
|
||||
Executable
+21
@@ -0,0 +1,21 @@
|
||||
"""
|
||||
All or portions of this file Copyright (c) Amazon.com, Inc. or its affiliates or
|
||||
its licensors.
|
||||
|
||||
For complete copyright and license terms please see the LICENSE at the root of this
|
||||
distribution (the "License"). All use of this software is governed by the License,
|
||||
or, if provided, by the license below or the license accompanying this file. Do not
|
||||
remove or modify any license notices. This file is distributed on an "AS IS" BASIS,
|
||||
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
"""
|
||||
|
||||
# Sanity test for EditorPythonBindings CTest wrapper
|
||||
|
||||
import azlmbr.framework as framework
|
||||
|
||||
print("EditorPythonBindings CTest Sanity Test")
|
||||
|
||||
# A test should have logic to determine success (zero) or failure (non-zero) and
|
||||
# return it to the caller. In this sanity test, always return success.
|
||||
return_code = 0
|
||||
framework.Terminate(return_code)
|
||||
Executable
+10
@@ -0,0 +1,10 @@
|
||||
"""
|
||||
All or portions of this file Copyright (c) Amazon.com, Inc. or its affiliates or
|
||||
its licensors.
|
||||
|
||||
For complete copyright and license terms please see the LICENSE at the root of this
|
||||
distribution (the "License"). All use of this software is governed by the License,
|
||||
or, if provided, by the license below or the license accompanying this file. Do not
|
||||
remove or modify any license notices. This file is distributed on an "AS IS" BASIS,
|
||||
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
"""
|
||||
+239
@@ -0,0 +1,239 @@
|
||||
"""
|
||||
All or portions of this file Copyright (c) Amazon.com, Inc. or its affiliates or
|
||||
its licensors.
|
||||
|
||||
For complete copyright and license terms please see the LICENSE at the root of this
|
||||
distribution (the "License"). All use of this software is governed by the License,
|
||||
or, if provided, by the license below or the license accompanying this file. Do not
|
||||
remove or modify any license notices. This file is distributed on an "AS IS" BASIS,
|
||||
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
|
||||
Helper functions for test result xml merging and processing.
|
||||
"""
|
||||
import glob
|
||||
import os
|
||||
import xml.etree.ElementTree as xet
|
||||
|
||||
TEST_RESULTS_DIR = 'Testing'
|
||||
|
||||
|
||||
def _get_ctest_tag_content(cmake_build_path):
|
||||
"""
|
||||
Get the content of the CTest TAG file. This file contains the name of the CTest test results directory.
|
||||
:param cmake_build_path: Path to the CMake build directory.
|
||||
:return: First line of the TAG file.
|
||||
"""
|
||||
tag_file_path = os.path.join(cmake_build_path, TEST_RESULTS_DIR, 'TAG')
|
||||
|
||||
if not os.path.exists(tag_file_path):
|
||||
raise FileNotFoundError(f'Could not find CTest TAG file at {tag_file_path}')
|
||||
|
||||
first_line = None
|
||||
|
||||
with open(tag_file_path) as tag_file:
|
||||
first_line = tag_file.readline().strip()
|
||||
|
||||
return first_line
|
||||
|
||||
|
||||
def _build_ctest_test_results_path(cmake_build_path):
|
||||
"""
|
||||
Build the path to the CTest test results directory.
|
||||
:param cmake_build_path: Path to the CMake build directory.
|
||||
:return: Path to the CTest test results directory.
|
||||
"""
|
||||
tag_content = _get_ctest_tag_content(cmake_build_path)
|
||||
if not tag_content:
|
||||
raise Exception('TAG file is empty.')
|
||||
|
||||
ctest_results_path = os.path.join(cmake_build_path, TEST_RESULTS_DIR, tag_content)
|
||||
return ctest_results_path
|
||||
|
||||
|
||||
def _build_gtest_test_results_path(cmake_build_path):
|
||||
"""
|
||||
Build the path to the GTest test results directory.
|
||||
:param cmake_build_path: Path to the CMake build directory.
|
||||
:return: Path to the GTest test results directory.
|
||||
"""
|
||||
gtest_results_path = os.path.join(cmake_build_path, TEST_RESULTS_DIR, 'Gtest')
|
||||
|
||||
return gtest_results_path
|
||||
|
||||
|
||||
def _build_pytest_test_results_path(cmake_build_path):
|
||||
"""
|
||||
Build the path to the Pytest test results directory.
|
||||
:param cmake_build_path: Path to the CMake build directory.
|
||||
:return: Path to the Pytest test results directory.
|
||||
"""
|
||||
pytest_results_path = os.path.join(cmake_build_path, TEST_RESULTS_DIR, 'Pytest')
|
||||
|
||||
return pytest_results_path
|
||||
|
||||
|
||||
def _get_all_test_results_paths(cmake_build_path, ctest_path_error_ok=False):
|
||||
"""
|
||||
Build and return the test result paths for all test harnesses.
|
||||
:param cmake_build_path: Path to the CMake build directory.
|
||||
:param ctest_path_error_ok: Ignore errors that occur while building CTest results path.
|
||||
:return: Test result paths for all test harnesses.
|
||||
"""
|
||||
paths = [
|
||||
_build_gtest_test_results_path(cmake_build_path),
|
||||
_build_pytest_test_results_path(cmake_build_path)
|
||||
]
|
||||
|
||||
try:
|
||||
paths.append(_build_ctest_test_results_path(cmake_build_path))
|
||||
except FileNotFoundError as e:
|
||||
if ctest_path_error_ok:
|
||||
print(e)
|
||||
else:
|
||||
raise
|
||||
|
||||
return paths
|
||||
|
||||
|
||||
def _merge_xml_results(xml_results_path, prefix, merged_xml_name, parent_element_name, child_element_name,
|
||||
attributes_to_aggregate):
|
||||
"""
|
||||
Merge the contents of XML test result files.
|
||||
:param xml_results_path: Path to the directory containing the files to merge.
|
||||
:param prefix: Test result file prefix.
|
||||
:param merged_xml_name: Name for the merged test result file.
|
||||
:param parent_element_name: Name of the XML element that will store the test results.
|
||||
:param child_element_name: Name of the XML element that contains the test results.
|
||||
:param attributes_to_aggregate: List of AttributeInfo items used for test result aggregation.
|
||||
"""
|
||||
xml_files = glob.glob(os.path.join(xml_results_path, f'{prefix}*.xml'))
|
||||
|
||||
if not xml_files:
|
||||
return
|
||||
|
||||
temp_dict = {}
|
||||
for attribute in attributes_to_aggregate:
|
||||
temp_dict[attribute.name] = attribute.func(0)
|
||||
|
||||
def _aggregate_attributes(nodes):
|
||||
for node in nodes:
|
||||
for attribute in attributes_to_aggregate:
|
||||
value = node.attrib[attribute.name]
|
||||
temp_dict[attribute.name] += attribute.func(value)
|
||||
|
||||
base_tree = xet.parse(xml_files[0])
|
||||
base_tree_root = base_tree.getroot()
|
||||
|
||||
if base_tree_root.tag == parent_element_name:
|
||||
parent_element = base_tree_root
|
||||
else:
|
||||
parent_element = base_tree_root.find(parent_element_name)
|
||||
|
||||
_aggregate_attributes(base_tree_root.findall(child_element_name))
|
||||
|
||||
for xml_file in xml_files[1:]:
|
||||
root = xet.parse(xml_file).getroot()
|
||||
child_nodes = root.findall(child_element_name)
|
||||
|
||||
_aggregate_attributes(child_nodes)
|
||||
parent_element.extend(child_nodes)
|
||||
|
||||
for attribute in attributes_to_aggregate:
|
||||
parent_element.attrib[attribute.name] = str(temp_dict[attribute.name])
|
||||
|
||||
base_tree.write(os.path.join(xml_results_path, merged_xml_name), encoding='UTF-8', xml_declaration=True)
|
||||
|
||||
|
||||
def clean_test_results(cmake_build_path):
|
||||
"""
|
||||
Clean the test results directories.
|
||||
:param cmake_build_path: Path to the CMake build directory.
|
||||
"""
|
||||
|
||||
# Using ctest_path_error_ok=True since the CTest path might not exist before tests are run for the first
|
||||
# time in a clean build.
|
||||
|
||||
for path in _get_all_test_results_paths(cmake_build_path, ctest_path_error_ok=True):
|
||||
xml_files = glob.glob(os.path.join(path, '*.xml'))
|
||||
for xml_file in xml_files:
|
||||
os.remove(xml_file)
|
||||
|
||||
|
||||
def rename_test_results(cmake_build_path, prefix, iteration, total):
|
||||
"""
|
||||
Rename the test result files with a prefix to prevent files being overwritten by subsequent test runs.
|
||||
:param cmake_build_path: Path to the CMake build directory.
|
||||
:param prefix: Test result file prefix.
|
||||
:param iteration: Test run number.
|
||||
:param total: Total number of test runs.
|
||||
"""
|
||||
for path in _get_all_test_results_paths(cmake_build_path):
|
||||
xml_files = glob.glob(os.path.join(path, '*.xml'))
|
||||
for xml_file in xml_files:
|
||||
filename = os.path.basename(xml_file)
|
||||
directory = os.path.dirname(xml_file)
|
||||
|
||||
if not filename.startswith(f'{prefix}-'):
|
||||
new_name = os.path.join(directory, f'{prefix}-{iteration}-{total}-{filename}')
|
||||
os.rename(xml_file, new_name)
|
||||
|
||||
|
||||
def collect_test_results(cmake_build_path, prefix):
|
||||
"""
|
||||
Combines and aggregates test results for each test harness.
|
||||
:param cmake_build_path: Path to the CMake build directory.
|
||||
:param prefix: Test result file prefix.
|
||||
"""
|
||||
class AttributeInfo:
|
||||
def __init__(self, name, func):
|
||||
self.name = name
|
||||
self.func = func
|
||||
|
||||
# Attributes that will be aggregated for JUnit-like reports (GTest and Pytest)
|
||||
attributes_to_aggregate = [AttributeInfo('tests', int),
|
||||
AttributeInfo('failures', int),
|
||||
AttributeInfo('disabled', int),
|
||||
AttributeInfo('errors', int),
|
||||
AttributeInfo('time', float)]
|
||||
|
||||
results_to_process = [
|
||||
# CTest results don't need aggregation, just merging.
|
||||
[_build_ctest_test_results_path(cmake_build_path), 'Site', 'Testing', []],
|
||||
# GTest and Pytest results need aggregation and merging.
|
||||
[_build_gtest_test_results_path(cmake_build_path), 'testsuites', 'testsuite', attributes_to_aggregate],
|
||||
[_build_pytest_test_results_path(cmake_build_path), 'testsuites', 'testsuite', attributes_to_aggregate]
|
||||
]
|
||||
|
||||
for result in results_to_process:
|
||||
_merge_xml_results(result[0], prefix, 'Merged.xml', result[1], result[2], result[3])
|
||||
|
||||
|
||||
def summarize_test_results(cmake_build_path, total):
|
||||
"""
|
||||
Writes a summary of the test results.
|
||||
:param cmake_build_path: Path to the CMake build directory.
|
||||
:param total: Total number of times the tests were executed.
|
||||
:return: A list of tests failed with their failure rate.
|
||||
"""
|
||||
failed_tests = {}
|
||||
|
||||
ctest_results_file = os.path.join(_build_ctest_test_results_path(cmake_build_path), 'Merged.xml')
|
||||
base_tree = xet.parse(ctest_results_file)
|
||||
base_tree_root = base_tree.getroot()
|
||||
testing_nodes = base_tree_root.findall('Testing')
|
||||
|
||||
for testing_node in testing_nodes:
|
||||
test_nodes = testing_node.findall('Test')
|
||||
|
||||
for test_node in test_nodes:
|
||||
if test_node.get('Status') == 'failed':
|
||||
name_element = test_node.find('Name')
|
||||
name = name_element.text
|
||||
failed_tests[name] = failed_tests.get(name, 0) + 1
|
||||
|
||||
report = []
|
||||
for test, count in failed_tests.items():
|
||||
percent = count/total
|
||||
report.append(f'{test} failed {count}/{total} times for a failure rate of ~{percent:.2%}')
|
||||
|
||||
return report
|
||||
Executable
+45
@@ -0,0 +1,45 @@
|
||||
"""
|
||||
All or portions of this file Copyright (c) Amazon.com, Inc. or its affiliates or
|
||||
its licensors.
|
||||
|
||||
For complete copyright and license terms please see the LICENSE at the root of this
|
||||
distribution (the "License"). All use of this software is governed by the License,
|
||||
or, if provided, by the license below or the license accompanying this file. Do not
|
||||
remove or modify any license notices. This file is distributed on an "AS IS" BASIS,
|
||||
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
|
||||
|
||||
Sanity tests to verify whether suite filtering is hooked up
|
||||
"""
|
||||
import pytest
|
||||
|
||||
def test_Sanity_Untagged_Pass(): # should happen in main pass also
|
||||
pass
|
||||
|
||||
@pytest.mark.SUITE_smoke
|
||||
def test_Sanity_Smoke_Pass():
|
||||
pass
|
||||
|
||||
@pytest.mark.SUITE_main
|
||||
def test_Sanity_Main_Pass(): # should happen in main pass
|
||||
pass
|
||||
|
||||
@pytest.mark.SUITE_periodic
|
||||
def test_Sanity_Periodic_Pass():
|
||||
pass
|
||||
|
||||
@pytest.mark.SUITE_benchmark
|
||||
def test_Sanity_Benchmark_Pass():
|
||||
pass
|
||||
|
||||
@pytest.mark.SUITE_sandbox
|
||||
def test_Sanity_Sandbox_Pass():
|
||||
pass
|
||||
|
||||
@pytest.mark.REQUIRES_gpu
|
||||
def test_Sanity_RequireGpu_Pass():
|
||||
pass
|
||||
|
||||
@pytest.mark.skip
|
||||
def test_Insanity():
|
||||
raise RuntimeError
|
||||
Reference in New Issue
Block a user