birkeh/o3de
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Change TIAF seed criteria. (#1857)

Browse Source 
We currently cannot predict when a given snapshot will be taken. This is
problematic as test impact analysis assumes that the coverage data will
be updated after each run. As we have no control over when our coverage
data will persist we will instead always seed when the pipeline and
branch is one of the seeding pipelines, otherwise we will attempt to
read that seed data and perform test impact analysis (if no seed data is
present we will fall back to a regular, uninstrumented run of all test
targets).

This approach has the following implications:

1. PR builds will benefit from test impact analysis as the seed data is
    (hopefully) available at the time the snapshot is created.

2. Builds for the aforementioned seeding branches and pipelines will not
    benefit from test impact analysis and will incur the cost of seeding
    regardless of whether or not that generated seed data ends up in the
    snapshot.

Signed-off-by: John <jonawals@amazon.com>
This commit is contained in:
jonawals
2021-07-07 11:47:36 +01:00
committed by GitHub
parent 64697350bc
commit a3d14e217e
3 changed files with 25 additions and 23 deletions
Download Patch File Download Diff File Expand all files Collapse all files
scripts/build/Platform/Windows/build_config.json
+1 -1
View File
@@ -89,7 +89,7 @@
"CONFIGURATION": "profile",
"SCRIPT_PATH": "scripts/build/TestImpactAnalysis/tiaf_driver.py",
"SCRIPT_PARAMETERS":
"--config=\"!OUTPUT_DIRECTORY!/bin/TestImpactFramework/persistent/tiaf.profile.json\" --suite=main --testFailurePolicy=continue --destBranch=!CHANGE_TARGET! --pipeline=!PIPELINE_NAME! --destCommit=!CHANGE_ID! --branchesOfTruth=!BUILD_SNAPSHOTS! --pipelinesOfTruth=default"
"--config=\"!OUTPUT_DIRECTORY!/bin/TestImpactFramework/persistent/tiaf.profile.json\" --suite=main --test-failure-policy=continue --src-branch=!BRANCH_NAME! --dst-branch=!CHANGE_TARGET! --pipeline=!PIPELINE_NAME! --dest-commit=!CHANGE_ID! --seeding-branches=!BUILD_SNAPSHOTS! --seeding-pipelines=default"
}
},
"debug_vs2019": {
scripts/build/TestImpactAnalysis/tiaf.py
+17 -16
View File
@@ -20,37 +20,37 @@ def is_child_path(parent_path, child_path):
return os.path.commonpath([os.path.abspath(parent_path)]) == os.path.commonpath([os.path.abspath(parent_path), os.path.abspath(child_path)])
class TestImpact:
def __init__(self, config_file, dst_commit, dst_branch, pipeline, branches_of_truth, pipelines_of_truth):
def __init__(self, config_file, dst_commit, src_branch, dst_branch, pipeline, seeding_branches, seeding_pipelines):
# Commit
self.__dst_commit = dst_commit
print(f"Commit: '{self.__dst_commit}'.")
self.__src_commit = None
self.__has_src_commit = False
# Branch
self.__src_branch = src_branch
print(f"Source branch: '{self.__src_branch}'.")
self.__dst_branch = dst_branch
print(f"Destination branch: '{self.__dst_branch}'.")
self.__branches_of_truth = branches_of_truth
print(f"Branches of truth: '{self.__branches_of_truth}'.")
if self.__dst_branch in self.__branches_of_truth:
self.__is_branch_of_truth = True
print(f"Seeding branches: '{seeding_branches}'.")
if self.__src_branch in seeding_branches:
self.__is_seeding_branch = True
else:
self.__is_branch_of_truth = False
print(f"Is branch of truth: '{self.__is_branch_of_truth}'.")
self.__is_seeding_branch = False
print(f"Is seeding branch: '{self.__is_seeding_branch}'.")
# Pipeline
self.__pipeline = pipeline
print(f"Pipeline: '{self.__pipeline}'.")
self.__pipelines_of_truth = pipelines_of_truth
print(f"Pipelines of truth: '{self.__pipelines_of_truth}'.")
if self.__pipeline in self.__pipelines_of_truth:
self.__is_pipeline_of_truth = True
print(f"Seeding pipelines: '{seeding_pipelines}'.")
if self.__pipeline in seeding_pipelines:
self.__is_seeding_pipeline = True
else:
self.__is_pipeline_of_truth = False
print(f"Is pipeline of truth: '{self.__is_pipeline_of_truth}'.")
self.__is_seeding_pipeline = False
print(f"Is seeding pipeline: '{self.__is_seeding_pipeline}'.")
# Config
self.__parse_config_file(config_file)
# Sequence
if self.__use_test_impact_analysis:
if self.__is_pipeline_of_truth and self.__is_branch_of_truth:
if self.__is_seeding_branch and self.__is_seeding_pipeline:
self.__is_seeding = True
else:
self.__is_seeding = False
@@ -65,6 +65,7 @@ class TestImpact:
self.__repo = Repo(self.__repo_dir)
# TIAF
self.__use_test_impact_analysis = config["jenkins"]["use_test_impact_analysis"]
print(f"Is using test impact analysis: '{self.__use_test_impact_analysis}'.")
self.__tiaf_bin = config["repo"]["tiaf_bin"]
if self.__use_test_impact_analysis and not os.path.isfile(self.__tiaf_bin):
raise FileNotFoundError("Could not find tiaf binary")
@@ -205,10 +206,10 @@ class TestImpact:
args.append(f"--fpolicy={test_failure_policy}")
print(f"Test failure policy is set to '{test_failure_policy}'.")
else:
print("Test impact analysis ie disabled.")
print("Test impact analysis is disabled.")
# Sequence type
args.append("--sequence=regular")
print("Sequence type is set to 'seed'.")
print("Sequence type is set to 'regular'.")
# Pipeline of truth sequence
if self.__is_pipeline_of_truth:
# Test failure policy
scripts/build/TestImpactAnalysis/tiaf_driver.py
+7 -6
View File
@@ -35,13 +35,14 @@ def parse_args():
parser = argparse.ArgumentParser()
parser.add_argument('--config', dest="config", type=file_path, help="Path to the test impact analysis framework configuration file", required=True)
parser.add_argument('--destBranch', dest="dst_branch", help="For PR builds, the destination branch to be merged to, otherwise empty")
parser.add_argument('--branchesOfTruth', dest="branches_of_truth", type=lambda arg: arg.split(','), help="Comma separated branches that seeding will occur on", required=True)
parser.add_argument('--src-branch', dest="src_branch", help="The branch that is being build", required=True)
parser.add_argument('--dst-branch', dest="dst_branch", help="For PR builds, the destination branch to be merged to, otherwise empty")
parser.add_argument('--seeding-branches', dest="seeding_branches", type=lambda arg: arg.split(','), help="Comma separated branches that seeding will occur on", required=True)
parser.add_argument('--pipeline', dest="pipeline", help="Pipeline the test impact analysis framework is running on", required=True)
parser.add_argument('--pipelinesOfTruth', dest="pipelines_of_truth", type=lambda arg: arg.split(','), help="Comma separated pipeline that seeding will occur on", required=True)
parser.add_argument('--destCommit', dest="dst_commit", help="Commit to run test impact analysis on (ignored when seeding)", required=True)
parser.add_argument('--seeding-pipelines', dest="seeding_pipelines", type=lambda arg: arg.split(','), help="Comma separated pipeline that seeding will occur on", required=True)
parser.add_argument('--dest-commit', dest="dst_commit", help="Commit to run test impact analysis on (ignored when seeding)", required=True)
parser.add_argument('--suite', dest="suite", help="Test suite to run", required=True)
parser.add_argument('--testFailurePolicy', dest="test_failure_policy", type=test_failure_policy, help="Test failure policy for regular and test impact sequences (ignored when seeding)", required=True)
parser.add_argument('--test-failure-policy', dest="test_failure_policy", type=test_failure_policy, help="Test failure policy for regular and test impact sequences (ignored when seeding)", required=True)
parser.add_argument('--safeMode', dest="safe_mode", action='store_true', help="Run impact analysis tests in safe mode (ignored when seeding)")
parser.add_argument('--testTimeout', dest="test_timeout", type=timout_type, help="Maximum run time (in seconds) of any test target before being terminated", required=False)
parser.add_argument('--globalTimeout', dest="global_timeout", type=timout_type, help="Maximum run time of the sequence before being terminated", required=False)
@@ -54,7 +55,7 @@ def parse_args():
if __name__ == "__main__":
try:
args = parse_args()
tiaf = TestImpact(args.config, args.dst_commit, args.dst_branch, args.pipeline, args.branches_of_truth, args.pipelines_of_truth)
tiaf = TestImpact(args.config, args.dst_commit, args.src_branch, args.dst_branch, args.pipeline, args.seeding_branches, args.seeding_pipelines)
return_code = tiaf.run(args.suite, args.test_failure_policy, args.safe_mode, args.test_timeout, args.global_timeout)
# Non-gating will be removed from this script and handled at the job level in SPEC-7413
#sys.exit(return_code)