Integrating latest 47acbe8

ctest_scripts/CMakeLists.txt

@@ -22,7 +22,7 @@ endif()
 ################################################################################
 
 if(PAL_TRAIT_BUILD_TESTS_SUPPORTED AND PAL_TRAIT_BUILD_HOST_TOOLS)
-    foreach(project ${LY_PROJECTS})
+    foreach(project ${LY_PROJECTS_NAME})
         add_custom_target(${project}.Assets
             COMMENT "Processing ${project} assets..."
             COMMAND $<TARGET_FILE:AZ::AssetProcessorBatch> --zeroAnalysisMode --gamefolder=${project} --platforms=${LY_ASSET_DEPLOY_ASSET_TYPE}
@@ -56,7 +56,7 @@ endforeach()
 
 
 # EPB Sanity test is being registered here to validate that the ly_add_editor_python_test function works.
-if(PAL_TRAIT_BUILD_HOST_TOOLS AND PAL_TRAIT_BUILD_TESTS_SUPPORTED AND AutomatedTesting IN_LIST LY_PROJECTS)
+if(PAL_TRAIT_BUILD_HOST_TOOLS AND PAL_TRAIT_BUILD_TESTS_SUPPORTED AND AutomatedTesting IN_LIST LY_PROJECTS_NAME)
     ly_add_editor_python_test(
         NAME epb_sanity_smoke_no_gpu
         TEST_PROJECT AutomatedTesting

ctest_scripts/ctest_driver.py

@@ -13,6 +13,7 @@ A wrapper to simplify invoking CTest with common parameters and sub-filter to sp
 import argparse
 import multiprocessing
 import os
+import result_processing.result_processing as rp
 import subprocess
 import sys
 import shutil
@@ -31,6 +32,7 @@ BUILD_CONFIGURATIONS = [
     "release",
 ]
 
+
 def _regex_matching_any(words):
     """
     :param words: iterable of strings to match
@@ -39,17 +41,18 @@ def _regex_matching_any(words):
     return "^(" + "|".join(words) + ")$"
 
 
-def run_single_test_suite(suite, ctest_path, cmake_build_path, build_config, disable_gpu, only_gpu, generate_xml, extra_args):
+def run_single_test_suite(suite, ctest_path, cmake_build_path, build_config, disable_gpu, only_gpu, generate_xml, repeat, extra_args):
     """
     Starts CTest to filter down to a specific suite
     :param suite: subset of tests to run, see SUITES_AND_DESCRIPTIONS
     :param ctest_path: path to ctest.exe
     :param cmake_build_path: path to build output
     :param build_config: cmake build variant to select
-    :param disable_gpu: run only non-gpu tests
-    :param only_gpu: run only gpu-required tests
-    :param generate_xml: enable to produce the CTest xml file
-    :param extrargs: forward args to ctest
+    :param disable_gpu: optional, run only non-gpu tests
+    :param only_gpu: optional, run only gpu-required tests
+    :param generate_xml: optional, enable to produce the CTest xml file
+    :param repeat: optional, number of times to run the tests in the suite
+    :param extrargs: optional, forward args to ctest
     :return: CTest exit code
     """
     ctest_command = [
@@ -98,14 +101,50 @@ def run_single_test_suite(suite, ctest_path, cmake_build_path, build_config, dis
         ctest_command.append(extra_arg)
 
     ctest_command_string = ' '.join(ctest_command) # ONLY used for display
-    print("Executing CTest with command:\n"
+    print(f"Executing CTest {repeat} time(s) with command:\n"
           f"  {ctest_command_string}\n"
           "in working directory:\n"
           f"  {cmake_build_path}\n")
 
-    result = subprocess.run(ctest_command, shell=False, cwd=cmake_build_path, stdout=sys.stdout, stderr=sys.stderr)
+    error_code = 0
+
+    if repeat:
+        # Run the tests multiple times. Previous test results are deleted, new test results are combined in a file per
+        # test runner.
+
+        test_result_prefix = 'Repeat'
+        repeat = int(repeat)
+        if generate_xml:
+            rp.clean_test_results(cmake_build_path)
+
+        for iteration in range(repeat):
+            print(f"Executing CTest iteration {iteration + 1}/{repeat}")
+            result = subprocess.run(ctest_command, shell=False, cwd=cmake_build_path, stdout=sys.stdout, stderr=sys.stderr)
+
+            if generate_xml:
+                rp.rename_test_results(cmake_build_path, test_result_prefix, iteration + 1, repeat)
+            if result.returncode:
+                error_code = result.returncode
+
+        if generate_xml:
+            rp.collect_test_results(cmake_build_path, test_result_prefix)
+            summary = rp.summarize_test_results(cmake_build_path, repeat)
+            
+            print()  # empty line
+            print('Test stability summary:')
+            if summary:
+                print('The following test(s) failed:')
+                for line in summary: print(line)
+            else:
+                print(f'All tests were executed {repeat} times and passed 100% of the time.')
+
+    else:
+        # Run the tests one time. Previous test results are not deleted but
+        # might be overwritten.
+        result = subprocess.run(ctest_command, shell=False, cwd=cmake_build_path, stdout=sys.stdout, stderr=sys.stderr)
+        error_code = result.returncode
     
-    return result.returncode
+    return error_code
 
 
 def main():
@@ -153,6 +192,10 @@ def main():
                         help="Which subset of tests to execute")
     parser.add_argument('--generate-xml', action='store_true', 
                         help='Enable this option to produce the CTest xml file.')
+    parser.add_argument('-r', '--repeat', help="Run the tests the specified number times to identify intermittent "
+                                               "failures (e.g. --repeat 3 for running the test three times). When used"
+                                               "with --generate-xml, the resulting test reports will be combined, "
+                                               "aggregated and summarized.", type=int)
     group = parser.add_mutually_exclusive_group()
     group.add_argument('--no-gpu', action='store_true',
                         help="Disable tests that require a GPU")
@@ -203,6 +246,7 @@ def main():
         disable_gpu=args.no_gpu,
         only_gpu=args.only_gpu,
         generate_xml=args.generate_xml,
+        repeat=args.repeat,
         extra_args=unknown_args)
 
 

ctest_scripts/result_processing/__init__.py

@@ -0,0 +1,10 @@
+"""
+All or portions of this file Copyright (c) Amazon.com, Inc. or its affiliates or
+its licensors.
+
+For complete copyright and license terms please see the LICENSE at the root of this
+distribution (the "License"). All use of this software is governed by the License,
+or, if provided, by the license below or the license accompanying this file. Do not
+remove or modify any license notices. This file is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+"""

ctest_scripts/result_processing/result_processing.py

@@ -0,0 +1,239 @@
+"""
+All or portions of this file Copyright (c) Amazon.com, Inc. or its affiliates or
+its licensors.
+
+For complete copyright and license terms please see the LICENSE at the root of this
+distribution (the "License"). All use of this software is governed by the License,
+or, if provided, by the license below or the license accompanying this file. Do not
+remove or modify any license notices. This file is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+
+Helper functions for test result xml merging and processing.
+"""
+import glob
+import os
+import xml.etree.ElementTree as xet
+
+TEST_RESULTS_DIR = 'Testing'
+
+
+def _get_ctest_tag_content(cmake_build_path):
+    """
+    Get the content of the CTest TAG file. This file contains the name of the CTest test results directory.
+    :param cmake_build_path: Path to the CMake build directory.
+    :return: First line of the TAG file.
+    """
+    tag_file_path = os.path.join(cmake_build_path, TEST_RESULTS_DIR, 'TAG')
+
+    if not os.path.exists(tag_file_path):
+        raise FileNotFoundError(f'Could not find CTest TAG file at {tag_file_path}')
+    
+    first_line = None
+
+    with open(tag_file_path) as tag_file:
+        first_line = tag_file.readline().strip()
+
+    return first_line
+
+
+def _build_ctest_test_results_path(cmake_build_path):
+    """
+    Build the path to the CTest test results directory.
+    :param cmake_build_path: Path to the CMake build directory.
+    :return: Path to the CTest test results directory.
+    """
+    tag_content = _get_ctest_tag_content(cmake_build_path)
+    if not tag_content:
+        raise Exception('TAG file is empty.')
+
+    ctest_results_path = os.path.join(cmake_build_path, TEST_RESULTS_DIR, tag_content)
+    return ctest_results_path
+
+
+def _build_gtest_test_results_path(cmake_build_path):
+    """
+    Build the path to the GTest test results directory.
+    :param cmake_build_path: Path to the CMake build directory.
+    :return: Path to the GTest test results directory.
+    """
+    gtest_results_path = os.path.join(cmake_build_path, TEST_RESULTS_DIR, 'Gtest')
+
+    return gtest_results_path
+
+
+def _build_pytest_test_results_path(cmake_build_path):
+    """
+    Build the path to the Pytest test results directory.
+    :param cmake_build_path: Path to the CMake build directory.
+    :return: Path to the Pytest test results directory.
+    """
+    pytest_results_path = os.path.join(cmake_build_path, TEST_RESULTS_DIR, 'Pytest')
+
+    return pytest_results_path
+
+
+def _get_all_test_results_paths(cmake_build_path, ctest_path_error_ok=False):
+    """
+    Build and return the test result paths for all test harnesses.
+    :param cmake_build_path: Path to the CMake build directory.
+    :param ctest_path_error_ok: Ignore errors that occur while building CTest results path.
+    :return: Test result paths for all test harnesses.
+    """
+    paths = [
+        _build_gtest_test_results_path(cmake_build_path),
+        _build_pytest_test_results_path(cmake_build_path)
+    ]
+
+    try:
+        paths.append(_build_ctest_test_results_path(cmake_build_path))
+    except FileNotFoundError as e:
+        if ctest_path_error_ok:
+            print(e)
+        else:
+            raise
+
+    return paths
+
+
+def _merge_xml_results(xml_results_path, prefix, merged_xml_name, parent_element_name, child_element_name,
+                       attributes_to_aggregate):
+    """
+    Merge the contents of XML test result files.
+    :param xml_results_path: Path to the directory containing the files to merge.
+    :param prefix: Test result file prefix.
+    :param merged_xml_name: Name for the merged test result file.
+    :param parent_element_name: Name of the XML element that will store the test results.
+    :param child_element_name: Name of the XML element that contains the test results.
+    :param attributes_to_aggregate: List of AttributeInfo items used for test result aggregation.
+    """
+    xml_files = glob.glob(os.path.join(xml_results_path, f'{prefix}*.xml'))
+
+    if not xml_files:
+        return
+
+    temp_dict = {}
+    for attribute in attributes_to_aggregate:
+        temp_dict[attribute.name] = attribute.func(0)
+
+    def _aggregate_attributes(nodes):
+        for node in nodes:
+            for attribute in attributes_to_aggregate:
+                value = node.attrib[attribute.name]
+                temp_dict[attribute.name] += attribute.func(value)
+
+    base_tree = xet.parse(xml_files[0])
+    base_tree_root = base_tree.getroot()
+
+    if base_tree_root.tag == parent_element_name:
+        parent_element = base_tree_root
+    else:
+        parent_element = base_tree_root.find(parent_element_name)
+
+    _aggregate_attributes(base_tree_root.findall(child_element_name))
+
+    for xml_file in xml_files[1:]:
+        root = xet.parse(xml_file).getroot()
+        child_nodes = root.findall(child_element_name)
+
+        _aggregate_attributes(child_nodes)
+        parent_element.extend(child_nodes)
+
+    for attribute in attributes_to_aggregate:
+        parent_element.attrib[attribute.name] = str(temp_dict[attribute.name])
+
+    base_tree.write(os.path.join(xml_results_path, merged_xml_name), encoding='UTF-8', xml_declaration=True)
+
+
+def clean_test_results(cmake_build_path):
+    """
+    Clean the test results directories.
+    :param cmake_build_path: Path to the CMake build directory.
+    """
+
+    # Using ctest_path_error_ok=True since the CTest path might not exist before tests are run for the first
+    # time in a clean build.
+
+    for path in _get_all_test_results_paths(cmake_build_path, ctest_path_error_ok=True):
+        xml_files = glob.glob(os.path.join(path, '*.xml'))
+        for xml_file in xml_files:
+            os.remove(xml_file)
+
+
+def rename_test_results(cmake_build_path, prefix, iteration, total):
+    """
+    Rename the test result files with a prefix to prevent files being overwritten by subsequent test runs.
+    :param cmake_build_path: Path to the CMake build directory.
+    :param prefix: Test result file prefix.
+    :param iteration: Test run number.
+    :param total: Total number of test runs.
+    """
+    for path in _get_all_test_results_paths(cmake_build_path):
+        xml_files = glob.glob(os.path.join(path, '*.xml'))
+        for xml_file in xml_files:
+            filename = os.path.basename(xml_file)
+            directory = os.path.dirname(xml_file)
+
+            if not filename.startswith(f'{prefix}-'):
+                new_name = os.path.join(directory, f'{prefix}-{iteration}-{total}-{filename}')
+                os.rename(xml_file, new_name)
+
+
+def collect_test_results(cmake_build_path, prefix):
+    """
+    Combines and aggregates test results for each test harness.
+    :param cmake_build_path: Path to the CMake build directory.
+    :param prefix: Test result file prefix.
+    """
+    class AttributeInfo:
+        def __init__(self, name, func):
+            self.name = name
+            self.func = func
+
+    # Attributes that will be aggregated for JUnit-like reports (GTest and Pytest)
+    attributes_to_aggregate = [AttributeInfo('tests', int),
+                               AttributeInfo('failures', int),
+                               AttributeInfo('disabled', int),
+                               AttributeInfo('errors', int),
+                               AttributeInfo('time', float)]
+
+    results_to_process = [
+        # CTest results don't need aggregation, just merging.
+        [_build_ctest_test_results_path(cmake_build_path), 'Site', 'Testing', []],
+        # GTest and Pytest results need aggregation and merging.
+        [_build_gtest_test_results_path(cmake_build_path), 'testsuites', 'testsuite', attributes_to_aggregate],
+        [_build_pytest_test_results_path(cmake_build_path), 'testsuites', 'testsuite', attributes_to_aggregate]
+    ]
+
+    for result in results_to_process:
+        _merge_xml_results(result[0], prefix, 'Merged.xml', result[1], result[2], result[3])
+
+
+def summarize_test_results(cmake_build_path, total):
+    """
+    Writes a summary of the test results.
+    :param cmake_build_path: Path to the CMake build directory.
+    :param total: Total number of times the tests were executed.
+    :return: A list of tests failed with their failure rate.
+    """
+    failed_tests = {}
+
+    ctest_results_file = os.path.join(_build_ctest_test_results_path(cmake_build_path), 'Merged.xml')
+    base_tree = xet.parse(ctest_results_file)
+    base_tree_root = base_tree.getroot()
+    testing_nodes = base_tree_root.findall('Testing')
+
+    for testing_node in testing_nodes:
+        test_nodes = testing_node.findall('Test')
+
+        for test_node in test_nodes:
+            if test_node.get('Status') == 'failed':
+                name_element = test_node.find('Name')
+                name = name_element.text
+                failed_tests[name] = failed_tests.get(name, 0) + 1
+
+    report = []
+    for test, count in failed_tests.items():
+        percent = count/total
+        report.append(f'{test} failed {count}/{total} times for a failure rate of ~{percent:.2%}')
+            
+    return report