From a3d14e217ee51060f10bab8a9b72482c20149b47 Mon Sep 17 00:00:00 2001 From: jonawals Date: Wed, 7 Jul 2021 11:47:36 +0100 Subject: [PATCH] Change TIAF seed criteria. (#1857) We currently cannot predict when a given snapshot will be taken. This is problematic as test impact analysis assumes that the coverage data will be updated after each run. As we have no control over when our coverage data will persist we will instead always seed when the pipeline and branch is one of the seeding pipelines, otherwise we will attempt to read that seed data and perform test impact analysis (if no seed data is present we will fall back to a regular, uninstrumented run of all test targets). This approach has the following implications: 1. PR builds will benefit from test impact analysis as the seed data is (hopefully) available at the time the snapshot is created. 2. Builds for the aforementioned seeding branches and pipelines will not benefit from test impact analysis and will incur the cost of seeding regardless of whether or not that generated seed data ends up in the snapshot. Signed-off-by: John --- .../build/Platform/Windows/build_config.json | 2 +- scripts/build/TestImpactAnalysis/tiaf.py | 33 ++++++++++--------- .../build/TestImpactAnalysis/tiaf_driver.py | 13 ++++---- 3 files changed, 25 insertions(+), 23 deletions(-) diff --git a/scripts/build/Platform/Windows/build_config.json b/scripts/build/Platform/Windows/build_config.json index b210d0bfb8..0e8a6b8746 100644 --- a/scripts/build/Platform/Windows/build_config.json +++ b/scripts/build/Platform/Windows/build_config.json @@ -89,7 +89,7 @@ "CONFIGURATION": "profile", "SCRIPT_PATH": "scripts/build/TestImpactAnalysis/tiaf_driver.py", "SCRIPT_PARAMETERS": - "--config=\"!OUTPUT_DIRECTORY!/bin/TestImpactFramework/persistent/tiaf.profile.json\" --suite=main --testFailurePolicy=continue --destBranch=!CHANGE_TARGET! --pipeline=!PIPELINE_NAME! --destCommit=!CHANGE_ID! --branchesOfTruth=!BUILD_SNAPSHOTS! --pipelinesOfTruth=default" + "--config=\"!OUTPUT_DIRECTORY!/bin/TestImpactFramework/persistent/tiaf.profile.json\" --suite=main --test-failure-policy=continue --src-branch=!BRANCH_NAME! --dst-branch=!CHANGE_TARGET! --pipeline=!PIPELINE_NAME! --dest-commit=!CHANGE_ID! --seeding-branches=!BUILD_SNAPSHOTS! --seeding-pipelines=default" } }, "debug_vs2019": { diff --git a/scripts/build/TestImpactAnalysis/tiaf.py b/scripts/build/TestImpactAnalysis/tiaf.py index 7d071138aa..1ba0a3f195 100644 --- a/scripts/build/TestImpactAnalysis/tiaf.py +++ b/scripts/build/TestImpactAnalysis/tiaf.py @@ -20,37 +20,37 @@ def is_child_path(parent_path, child_path): return os.path.commonpath([os.path.abspath(parent_path)]) == os.path.commonpath([os.path.abspath(parent_path), os.path.abspath(child_path)]) class TestImpact: - def __init__(self, config_file, dst_commit, dst_branch, pipeline, branches_of_truth, pipelines_of_truth): + def __init__(self, config_file, dst_commit, src_branch, dst_branch, pipeline, seeding_branches, seeding_pipelines): # Commit self.__dst_commit = dst_commit print(f"Commit: '{self.__dst_commit}'.") self.__src_commit = None self.__has_src_commit = False # Branch + self.__src_branch = src_branch + print(f"Source branch: '{self.__src_branch}'.") self.__dst_branch = dst_branch print(f"Destination branch: '{self.__dst_branch}'.") - self.__branches_of_truth = branches_of_truth - print(f"Branches of truth: '{self.__branches_of_truth}'.") - if self.__dst_branch in self.__branches_of_truth: - self.__is_branch_of_truth = True + print(f"Seeding branches: '{seeding_branches}'.") + if self.__src_branch in seeding_branches: + self.__is_seeding_branch = True else: - self.__is_branch_of_truth = False - print(f"Is branch of truth: '{self.__is_branch_of_truth}'.") + self.__is_seeding_branch = False + print(f"Is seeding branch: '{self.__is_seeding_branch}'.") # Pipeline self.__pipeline = pipeline print(f"Pipeline: '{self.__pipeline}'.") - self.__pipelines_of_truth = pipelines_of_truth - print(f"Pipelines of truth: '{self.__pipelines_of_truth}'.") - if self.__pipeline in self.__pipelines_of_truth: - self.__is_pipeline_of_truth = True + print(f"Seeding pipelines: '{seeding_pipelines}'.") + if self.__pipeline in seeding_pipelines: + self.__is_seeding_pipeline = True else: - self.__is_pipeline_of_truth = False - print(f"Is pipeline of truth: '{self.__is_pipeline_of_truth}'.") + self.__is_seeding_pipeline = False + print(f"Is seeding pipeline: '{self.__is_seeding_pipeline}'.") # Config self.__parse_config_file(config_file) # Sequence if self.__use_test_impact_analysis: - if self.__is_pipeline_of_truth and self.__is_branch_of_truth: + if self.__is_seeding_branch and self.__is_seeding_pipeline: self.__is_seeding = True else: self.__is_seeding = False @@ -65,6 +65,7 @@ class TestImpact: self.__repo = Repo(self.__repo_dir) # TIAF self.__use_test_impact_analysis = config["jenkins"]["use_test_impact_analysis"] + print(f"Is using test impact analysis: '{self.__use_test_impact_analysis}'.") self.__tiaf_bin = config["repo"]["tiaf_bin"] if self.__use_test_impact_analysis and not os.path.isfile(self.__tiaf_bin): raise FileNotFoundError("Could not find tiaf binary") @@ -205,10 +206,10 @@ class TestImpact: args.append(f"--fpolicy={test_failure_policy}") print(f"Test failure policy is set to '{test_failure_policy}'.") else: - print("Test impact analysis ie disabled.") + print("Test impact analysis is disabled.") # Sequence type args.append("--sequence=regular") - print("Sequence type is set to 'seed'.") + print("Sequence type is set to 'regular'.") # Pipeline of truth sequence if self.__is_pipeline_of_truth: # Test failure policy diff --git a/scripts/build/TestImpactAnalysis/tiaf_driver.py b/scripts/build/TestImpactAnalysis/tiaf_driver.py index 3c8ec4b32a..a9be7c8cbc 100644 --- a/scripts/build/TestImpactAnalysis/tiaf_driver.py +++ b/scripts/build/TestImpactAnalysis/tiaf_driver.py @@ -35,13 +35,14 @@ def parse_args(): parser = argparse.ArgumentParser() parser.add_argument('--config', dest="config", type=file_path, help="Path to the test impact analysis framework configuration file", required=True) - parser.add_argument('--destBranch', dest="dst_branch", help="For PR builds, the destination branch to be merged to, otherwise empty") - parser.add_argument('--branchesOfTruth', dest="branches_of_truth", type=lambda arg: arg.split(','), help="Comma separated branches that seeding will occur on", required=True) + parser.add_argument('--src-branch', dest="src_branch", help="The branch that is being build", required=True) + parser.add_argument('--dst-branch', dest="dst_branch", help="For PR builds, the destination branch to be merged to, otherwise empty") + parser.add_argument('--seeding-branches', dest="seeding_branches", type=lambda arg: arg.split(','), help="Comma separated branches that seeding will occur on", required=True) parser.add_argument('--pipeline', dest="pipeline", help="Pipeline the test impact analysis framework is running on", required=True) - parser.add_argument('--pipelinesOfTruth', dest="pipelines_of_truth", type=lambda arg: arg.split(','), help="Comma separated pipeline that seeding will occur on", required=True) - parser.add_argument('--destCommit', dest="dst_commit", help="Commit to run test impact analysis on (ignored when seeding)", required=True) + parser.add_argument('--seeding-pipelines', dest="seeding_pipelines", type=lambda arg: arg.split(','), help="Comma separated pipeline that seeding will occur on", required=True) + parser.add_argument('--dest-commit', dest="dst_commit", help="Commit to run test impact analysis on (ignored when seeding)", required=True) parser.add_argument('--suite', dest="suite", help="Test suite to run", required=True) - parser.add_argument('--testFailurePolicy', dest="test_failure_policy", type=test_failure_policy, help="Test failure policy for regular and test impact sequences (ignored when seeding)", required=True) + parser.add_argument('--test-failure-policy', dest="test_failure_policy", type=test_failure_policy, help="Test failure policy for regular and test impact sequences (ignored when seeding)", required=True) parser.add_argument('--safeMode', dest="safe_mode", action='store_true', help="Run impact analysis tests in safe mode (ignored when seeding)") parser.add_argument('--testTimeout', dest="test_timeout", type=timout_type, help="Maximum run time (in seconds) of any test target before being terminated", required=False) parser.add_argument('--globalTimeout', dest="global_timeout", type=timout_type, help="Maximum run time of the sequence before being terminated", required=False) @@ -54,7 +55,7 @@ def parse_args(): if __name__ == "__main__": try: args = parse_args() - tiaf = TestImpact(args.config, args.dst_commit, args.dst_branch, args.pipeline, args.branches_of_truth, args.pipelines_of_truth) + tiaf = TestImpact(args.config, args.dst_commit, args.src_branch, args.dst_branch, args.pipeline, args.seeding_branches, args.seeding_pipelines) return_code = tiaf.run(args.suite, args.test_failure_policy, args.safe_mode, args.test_timeout, args.global_timeout) # Non-gating will be removed from this script and handled at the job level in SPEC-7413 #sys.exit(return_code)