birkeh/o3de
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

SPEC-5070 Move ctest_scripts to scripts/ctest

Browse Source 
* removing unused function and moving ctest_scripts to scripts/ctest

* Re-adding ebp-test

* Fixing typo that is making this test run in parallel with other tests

* Fixing hang when parameters are passed

* passing absolute path as a project

* small tweak to not print out during Python execution

* Moving the timeout to be in the build step

* Disable ebo_sanity_smoke_no_gpu

Co-authored-by: jackalbe <23512001+jackalbe@users.noreply.github.com>
This commit is contained in:
Esteban Papp
2021-04-23 09:43:11 -07:00
committed by GitHub
parent 0fa00a117c
commit 1c13b301fe
15 changed files with 25 additions and 10 deletions
Download Patch File Download Diff File Expand all files Collapse all files
scripts/ctest/result_processing/result_processing.py
Executable
+239
View File
@@ -0,0 +1,239 @@
"""
All or portions of this file Copyright (c) Amazon.com, Inc. or its affiliates or
its licensors.
For complete copyright and license terms please see the LICENSE at the root of this
distribution (the "License"). All use of this software is governed by the License,
or, if provided, by the license below or the license accompanying this file. Do not
remove or modify any license notices. This file is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
Helper functions for test result xml merging and processing.
"""
import glob
import os
import xml.etree.ElementTree as xet
TEST_RESULTS_DIR = 'Testing'
def _get_ctest_tag_content(cmake_build_path):
"""
Get the content of the CTest TAG file. This file contains the name of the CTest test results directory.
:param cmake_build_path: Path to the CMake build directory.
:return: First line of the TAG file.
"""
tag_file_path = os.path.join(cmake_build_path, TEST_RESULTS_DIR, 'TAG')
if not os.path.exists(tag_file_path):
raise FileNotFoundError(f'Could not find CTest TAG file at {tag_file_path}')
first_line = None
with open(tag_file_path) as tag_file:
first_line = tag_file.readline().strip()
return first_line
def _build_ctest_test_results_path(cmake_build_path):
"""
Build the path to the CTest test results directory.
:param cmake_build_path: Path to the CMake build directory.
:return: Path to the CTest test results directory.
"""
tag_content = _get_ctest_tag_content(cmake_build_path)
if not tag_content:
raise Exception('TAG file is empty.')
ctest_results_path = os.path.join(cmake_build_path, TEST_RESULTS_DIR, tag_content)
return ctest_results_path
def _build_gtest_test_results_path(cmake_build_path):
"""
Build the path to the GTest test results directory.
:param cmake_build_path: Path to the CMake build directory.
:return: Path to the GTest test results directory.
"""
gtest_results_path = os.path.join(cmake_build_path, TEST_RESULTS_DIR, 'Gtest')
return gtest_results_path
def _build_pytest_test_results_path(cmake_build_path):
"""
Build the path to the Pytest test results directory.
:param cmake_build_path: Path to the CMake build directory.
:return: Path to the Pytest test results directory.
"""
pytest_results_path = os.path.join(cmake_build_path, TEST_RESULTS_DIR, 'Pytest')
return pytest_results_path
def _get_all_test_results_paths(cmake_build_path, ctest_path_error_ok=False):
"""
Build and return the test result paths for all test harnesses.
:param cmake_build_path: Path to the CMake build directory.
:param ctest_path_error_ok: Ignore errors that occur while building CTest results path.
:return: Test result paths for all test harnesses.
"""
paths = [
_build_gtest_test_results_path(cmake_build_path),
_build_pytest_test_results_path(cmake_build_path)
]
try:
paths.append(_build_ctest_test_results_path(cmake_build_path))
except FileNotFoundError as e:
if ctest_path_error_ok:
print(e)
else:
raise
return paths
def _merge_xml_results(xml_results_path, prefix, merged_xml_name, parent_element_name, child_element_name,
attributes_to_aggregate):
"""
Merge the contents of XML test result files.
:param xml_results_path: Path to the directory containing the files to merge.
:param prefix: Test result file prefix.
:param merged_xml_name: Name for the merged test result file.
:param parent_element_name: Name of the XML element that will store the test results.
:param child_element_name: Name of the XML element that contains the test results.
:param attributes_to_aggregate: List of AttributeInfo items used for test result aggregation.
"""
xml_files = glob.glob(os.path.join(xml_results_path, f'{prefix}*.xml'))
if not xml_files:
return
temp_dict = {}
for attribute in attributes_to_aggregate:
temp_dict[attribute.name] = attribute.func(0)
def _aggregate_attributes(nodes):
for node in nodes:
for attribute in attributes_to_aggregate:
value = node.attrib[attribute.name]
temp_dict[attribute.name] += attribute.func(value)
base_tree = xet.parse(xml_files[0])
base_tree_root = base_tree.getroot()
if base_tree_root.tag == parent_element_name:
parent_element = base_tree_root
else:
parent_element = base_tree_root.find(parent_element_name)
_aggregate_attributes(base_tree_root.findall(child_element_name))
for xml_file in xml_files[1:]:
root = xet.parse(xml_file).getroot()
child_nodes = root.findall(child_element_name)
_aggregate_attributes(child_nodes)
parent_element.extend(child_nodes)
for attribute in attributes_to_aggregate:
parent_element.attrib[attribute.name] = str(temp_dict[attribute.name])
base_tree.write(os.path.join(xml_results_path, merged_xml_name), encoding='UTF-8', xml_declaration=True)
def clean_test_results(cmake_build_path):
"""
Clean the test results directories.
:param cmake_build_path: Path to the CMake build directory.
"""
# Using ctest_path_error_ok=True since the CTest path might not exist before tests are run for the first
# time in a clean build.
for path in _get_all_test_results_paths(cmake_build_path, ctest_path_error_ok=True):
xml_files = glob.glob(os.path.join(path, '*.xml'))
for xml_file in xml_files:
os.remove(xml_file)
def rename_test_results(cmake_build_path, prefix, iteration, total):
"""
Rename the test result files with a prefix to prevent files being overwritten by subsequent test runs.
:param cmake_build_path: Path to the CMake build directory.
:param prefix: Test result file prefix.
:param iteration: Test run number.
:param total: Total number of test runs.
"""
for path in _get_all_test_results_paths(cmake_build_path):
xml_files = glob.glob(os.path.join(path, '*.xml'))
for xml_file in xml_files:
filename = os.path.basename(xml_file)
directory = os.path.dirname(xml_file)
if not filename.startswith(f'{prefix}-'):
new_name = os.path.join(directory, f'{prefix}-{iteration}-{total}-{filename}')
os.rename(xml_file, new_name)
def collect_test_results(cmake_build_path, prefix):
"""
Combines and aggregates test results for each test harness.
:param cmake_build_path: Path to the CMake build directory.
:param prefix: Test result file prefix.
"""
class AttributeInfo:
def __init__(self, name, func):
self.name = name
self.func = func
# Attributes that will be aggregated for JUnit-like reports (GTest and Pytest)
attributes_to_aggregate = [AttributeInfo('tests', int),
AttributeInfo('failures', int),
AttributeInfo('disabled', int),
AttributeInfo('errors', int),
AttributeInfo('time', float)]
results_to_process = [
# CTest results don't need aggregation, just merging.
[_build_ctest_test_results_path(cmake_build_path), 'Site', 'Testing', []],
# GTest and Pytest results need aggregation and merging.
[_build_gtest_test_results_path(cmake_build_path), 'testsuites', 'testsuite', attributes_to_aggregate],
[_build_pytest_test_results_path(cmake_build_path), 'testsuites', 'testsuite', attributes_to_aggregate]
]
for result in results_to_process:
_merge_xml_results(result[0], prefix, 'Merged.xml', result[1], result[2], result[3])
def summarize_test_results(cmake_build_path, total):
"""
Writes a summary of the test results.
:param cmake_build_path: Path to the CMake build directory.
:param total: Total number of times the tests were executed.
:return: A list of tests failed with their failure rate.
"""
failed_tests = {}
ctest_results_file = os.path.join(_build_ctest_test_results_path(cmake_build_path), 'Merged.xml')
base_tree = xet.parse(ctest_results_file)
base_tree_root = base_tree.getroot()
testing_nodes = base_tree_root.findall('Testing')
for testing_node in testing_nodes:
test_nodes = testing_node.findall('Test')
for test_node in test_nodes:
if test_node.get('Status') == 'failed':
name_element = test_node.find('Name')
name = name_element.text
failed_tests[name] = failed_tests.get(name, 0) + 1
report = []
for test, count in failed_tests.items():
percent = count/total
report.append(f'{test} failed {count}/{total} times for a failure rate of ~{percent:.2%}')
return report