diff --git a/Code/Tools/TestImpactFramework/Frontend/Console/Static/Code/Source/TestImpactCommandLineOptions.cpp b/Code/Tools/TestImpactFramework/Frontend/Console/Static/Code/Source/TestImpactCommandLineOptions.cpp index 0a4c80afa4..7eff998dda 100644 --- a/Code/Tools/TestImpactFramework/Frontend/Console/Static/Code/Source/TestImpactCommandLineOptions.cpp +++ b/Code/Tools/TestImpactFramework/Frontend/Console/Static/Code/Source/TestImpactCommandLineOptions.cpp @@ -22,22 +22,22 @@ namespace TestImpact enum { // Options - Config, - ChangeList, - OutputChangeList, - Sequence, - TestPrioritizationPolicy, - ExecutionFailurePolicy, - FailedTestCoveragePolicy, - TestFailurePolicy, - IntegrityFailurePolicy, - TestShardingPolicy, - TargetOutputCapture, - MaxConcurrency, - TestTargetTimeout, - GlobalTimeout, - SuiteFilter, - SafeMode, + ConfigKey, + ChangeListKey, + OutputChangeListKey, + SequenceKey, + TestPrioritizationPolicyKey, + ExecutionFailurePolicyKey, + FailedTestCoveragePolicyKey, + TestFailurePolicyKey, + IntegrityFailurePolicyKey, + TestShardingPolicyKey, + TargetOutputCaptureKey, + MaxConcurrencyKey, + TestTargetTimeoutKey, + GlobalTimeoutKey, + SuiteFilterKey, + SafeModeKey, // Values None, Seed, @@ -93,17 +93,17 @@ namespace TestImpact RepoPath ParseConfigurationFile(const AZ::CommandLine& cmd) { - return ParsePathOption(OptionKeys[Config], cmd).value_or(LY_TEST_IMPACT_DEFAULT_CONFIG_FILE); + return ParsePathOption(OptionKeys[ConfigKey], cmd).value_or(LY_TEST_IMPACT_DEFAULT_CONFIG_FILE); } AZStd::optional ParseChangeListFile(const AZ::CommandLine& cmd) { - return ParsePathOption(OptionKeys[ChangeList], cmd); + return ParsePathOption(OptionKeys[ChangeListKey], cmd); } bool ParseOutputChangeList(const AZ::CommandLine& cmd) { - return ParseOnOffOption(OptionKeys[OutputChangeList], BinaryStateValue{ false, true }, cmd).value_or(false); + return ParseOnOffOption(OptionKeys[OutputChangeListKey], BinaryStateValue{ false, true }, cmd).value_or(false); } TestSequenceType ParseTestSequenceType(const AZ::CommandLine& cmd) @@ -118,7 +118,7 @@ namespace TestImpact {OptionKeys[ImpactAnalysisOrSeed], TestSequenceType::ImpactAnalysisOrSeed} }; - return ParseMultiStateOption(OptionKeys[Sequence], states, cmd).value_or(TestSequenceType::None); + return ParseMultiStateOption(OptionKeys[SequenceKey], states, cmd).value_or(TestSequenceType::None); } Policy::TestPrioritization ParseTestPrioritizationPolicy(const AZ::CommandLine& cmd) @@ -129,7 +129,7 @@ namespace TestImpact {OptionKeys[Locality], Policy::TestPrioritization::DependencyLocality} }; - return ParseBinaryStateOption(OptionKeys[TestPrioritizationPolicy], states, cmd).value_or(Policy::TestPrioritization::None); + return ParseBinaryStateOption(OptionKeys[TestPrioritizationPolicyKey], states, cmd).value_or(Policy::TestPrioritization::None); } Policy::ExecutionFailure ParseExecutionFailurePolicy(const AZ::CommandLine& cmd) @@ -140,7 +140,7 @@ namespace TestImpact {OptionKeys[Continue], Policy::ExecutionFailure::Continue}, {OptionKeys[Ignore], Policy::ExecutionFailure::Ignore} }; - return ParseMultiStateOption(OptionKeys[ExecutionFailurePolicy], states, cmd).value_or(Policy::ExecutionFailure::Continue); + return ParseMultiStateOption(OptionKeys[ExecutionFailurePolicyKey], states, cmd).value_or(Policy::ExecutionFailure::Continue); } Policy::FailedTestCoverage ParseFailedTestCoveragePolicy(const AZ::CommandLine& cmd) @@ -151,7 +151,7 @@ namespace TestImpact {OptionKeys[Keep], Policy::FailedTestCoverage::Keep} }; - return ParseMultiStateOption(OptionKeys[FailedTestCoveragePolicy], states, cmd).value_or(Policy::FailedTestCoverage::Keep); + return ParseMultiStateOption(OptionKeys[FailedTestCoveragePolicyKey], states, cmd).value_or(Policy::FailedTestCoverage::Keep); } Policy::TestFailure ParseTestFailurePolicy(const AZ::CommandLine& cmd) @@ -162,7 +162,7 @@ namespace TestImpact Policy::TestFailure::Continue }; - return ParseAbortContinueOption(OptionKeys[TestFailurePolicy], states, cmd).value_or(Policy::TestFailure::Abort); + return ParseAbortContinueOption(OptionKeys[TestFailurePolicyKey], states, cmd).value_or(Policy::TestFailure::Abort); } Policy::IntegrityFailure ParseIntegrityFailurePolicy(const AZ::CommandLine& cmd) @@ -173,7 +173,7 @@ namespace TestImpact Policy::IntegrityFailure::Continue }; - return ParseAbortContinueOption(OptionKeys[IntegrityFailurePolicy], states, cmd).value_or(Policy::IntegrityFailure::Abort); + return ParseAbortContinueOption(OptionKeys[IntegrityFailurePolicyKey], states, cmd).value_or(Policy::IntegrityFailure::Abort); } Policy::TestSharding ParseTestShardingPolicy(const AZ::CommandLine& cmd) @@ -184,12 +184,12 @@ namespace TestImpact Policy::TestSharding::Always }; - return ParseOnOffOption("shard", states, cmd).value_or(Policy::TestSharding::Never); + return ParseOnOffOption(OptionKeys[TestShardingPolicyKey], states, cmd).value_or(Policy::TestSharding::Never); } Policy::TargetOutputCapture ParseTargetOutputCapture(const AZ::CommandLine& cmd) { - if (const auto numSwitchValues = cmd.GetNumSwitchValues(OptionKeys[TargetOutputCapture]); + if (const auto numSwitchValues = cmd.GetNumSwitchValues(OptionKeys[TargetOutputCaptureKey]); numSwitchValues) { AZ_TestImpact_Eval( @@ -198,7 +198,7 @@ namespace TestImpact Policy::TargetOutputCapture targetOutputCapture = Policy::TargetOutputCapture::None; for (auto i = 0; i < numSwitchValues; i++) { - const auto option = cmd.GetSwitchValue(OptionKeys[TargetOutputCapture], i); + const auto option = cmd.GetSwitchValue(OptionKeys[TargetOutputCaptureKey], i); if (option == OptionKeys[StdOut]) { if (targetOutputCapture == Policy::TargetOutputCapture::File) @@ -236,23 +236,23 @@ namespace TestImpact AZStd::optional ParseMaxConcurrency(const AZ::CommandLine& cmd) { - return ParseUnsignedIntegerOption(OptionKeys[MaxConcurrency], cmd); + return ParseUnsignedIntegerOption(OptionKeys[MaxConcurrencyKey], cmd); } AZStd::optional ParseTestTargetTimeout(const AZ::CommandLine& cmd) { - return ParseSecondsOption(OptionKeys[TestTargetTimeout], cmd); + return ParseSecondsOption(OptionKeys[TestTargetTimeoutKey], cmd); } AZStd::optional ParseGlobalTimeout(const AZ::CommandLine& cmd) { - return ParseSecondsOption(OptionKeys[GlobalTimeout], cmd); + return ParseSecondsOption(OptionKeys[GlobalTimeoutKey], cmd); } bool ParseSafeMode(const AZ::CommandLine& cmd) { const BinaryStateValue states = { false, true }; - return ParseOnOffOption(OptionKeys[SafeMode], states, cmd).value_or(false); + return ParseOnOffOption(OptionKeys[SafeModeKey], states, cmd).value_or(false); } SuiteType ParseSuiteFilter(const AZ::CommandLine& cmd) @@ -264,7 +264,7 @@ namespace TestImpact {GetSuiteTypeName(SuiteType::Sandbox), SuiteType::Sandbox} }; - return ParseMultiStateOption(OptionKeys[SuiteFilter], states, cmd).value_or(SuiteType::Main); + return ParseMultiStateOption(OptionKeys[SuiteFilterKey], states, cmd).value_or(SuiteType::Main); } } diff --git a/scripts/build/TestImpactAnalysis/tiaf.py b/scripts/build/TestImpactAnalysis/tiaf.py index 01e2cde8ba..1523d39b4a 100644 --- a/scripts/build/TestImpactAnalysis/tiaf.py +++ b/scripts/build/TestImpactAnalysis/tiaf.py @@ -23,33 +23,40 @@ def is_child_path(parent_path, child_path): child_path = os.path.abspath(child_path) return os.path.commonpath([os.path.abspath(parent_path)]) == os.path.commonpath([os.path.abspath(parent_path), os.path.abspath(child_path)]) -# Enumerations for test sequence types -class SequenceType(Enum): - # Regular sequence as-per the tiaf regular sequence - REGULAR = 1 - # TIA sequence as-per the tiaf read-only impact analysis sequence - TEST_IMPACT_ANALYSIS = 2 - # Seed sequence as-per the tiaf seed sequence - SEED = 3 - class TestImpact: - def __init__(self, config_file, dst_commit): + def __init__(self, config_file, pipeline, dst_commit): + self.__pipeline = pipeline self.__parse_config_file(config_file) self.__init_repo(dst_commit) - self.__generate_change_list() + if self.__use_test_impact_analysis and not self.__is_pipeline_of_truth: + self.__generate_change_list() # Parse the configuration file and retrieve the data needed for launching the test impact analysis runtime def __parse_config_file(self, config_file): print(f"Attempting to parse configuration file '{config_file}'...") with open(config_file, "r") as config_data: config = json.load(config_data) + # Repository self.__repo_dir = config["repo"]["root"] + # Jenkins + self.__use_test_impact_analysis = config["jenkins"]["use_test_impact_analysis"] + self.__pipeline_of_truth = config["jenkins"]["pipeline_of_truth"] + print(f"Pipeline of truth: '{self.__pipeline_of_truth}'.") + print(f"This pipeline: '{self.__pipeline}'.") + if self.__pipeline in self.__pipeline_of_truth: + self.__is_pipeline_of_truth = True + else: + self.__is_pipeline_of_truth = False + print(f"Is pipeline of truth: '{self.__is_pipeline_of_truth}'.") + # TIAF binary self.__tiaf_bin = config["repo"]["tiaf_bin"] - if not os.path.isfile(self.__tiaf_bin): + if self.__use_test_impact_analysis and not os.path.isfile(self.__tiaf_bin): raise FileNotFoundError("Could not find tiaf binary") + # Workspaces self.__active_workspace = config["workspace"]["active"]["root"] self.__historic_workspace = config["workspace"]["historic"]["root"] self.__temp_workspace = config["workspace"]["temp"]["root"] + # Last commit hash last_commit_hash_path_rel = config["workspace"]["historic"]["relative_paths"]["last_run_hash_file"] self.__last_commit_hash_path = os.path.join(self.__historic_workspace, last_commit_hash_path_rel) print("The configuration file was parsed successfully.") @@ -71,7 +78,7 @@ class TestImpact: def __read_last_run_hash(self): self.__has_src_commit = False if os.path.isfile(self.__last_commit_hash_path): - print(f"Previous commit hash found at '{self.__last_commit_hash_path}'") + print(f"Previous commit hash found at '{self.__last_commit_hash_path}'.") with open(self.__last_commit_hash_path) as file: self.__src_commit = file.read() self.__has_src_commit = True @@ -144,43 +151,79 @@ class TestImpact: return # Runs the specified test sequence - def run(self, sequence_type, safe_mode, test_timeout, global_timeout): + def run(self, suite, safe_mode, test_timeout, global_timeout): args = [] - print("Please note: test impact analysis sequences will be run in read-only mode (seed sequences are unaffected).") - if sequence_type == SequenceType.REGULAR: - print("Sequence type: regular.") - args.append("--sequence=regular") - args.append("--fpolicy=abort") - elif sequence_type == SequenceType.SEED: - print("Sequence type: seed.") - args.append("--sequence=seed") - args.append("--fpolicy=continue") - elif sequence_type == SequenceType.TEST_IMPACT_ANALYSIS: - print("Sequence type: test impact analysis (no write).") - args.append("--fpolicy=abort") - if self.__has_change_list: - args.append(f"-changelist={self.__change_list_path}") - args.append("--sequence=tianowrite") - else: - print(f"No change list was generated, falling back to a regular sequence.") - print("Sequence type: Regular.") - args.append("--sequence=regular") - else: - raise ValueError(sequence_type) - + # Suite + args.append(f"--suite={suite}") + print(f"Test suite is set to '{suite}'.") + # Timeouts if test_timeout != None: args.append(f"--ttimeout={test_timeout}") print(f"Test target timeout is set to {test_timeout} seconds.") if global_timeout != None: args.append(f"--gtimeout={global_timeout}") print(f"Global sequence timeout is set to {test_timeout} seconds.") - + # If test impact analysis is enabled: + # -> Pipleine of truth will perform a seed that will continue until the sequence is complete regardless of test failues + # -> Non pipline of truth will attempt to perform an impact analysis sequence and exit early upon the fist test failure + # If test impact analysis is disabled: + # -> Pipleine of truth will perform a regular sequence that will continue until the sequence is complete regardless of test failues + # -> Non pipline of truth will perform a regular sequence and exit early upon the fist test failure + if self.__use_test_impact_analysis: + print("Test impact analysis ie enabled.") + # Pipeline of truth sequence + if self.__is_pipeline_of_truth: + # Sequence type + args.append("--sequence=seed") + print("Sequence type is set to 'seed'.") + # Test failure policy + args.append("--fpolicy=continue") + print("Test failure policy is set to 'continue'.") + # Non pipeline of truth sequence + else: + if self.__has_change_list: + # Change list + args.append(f"--changelist={self.__change_list_path}") + print(f"Change list is set to '{self.__change_list_path}'.") + # Sequence type + args.append("--sequence=tianowrite") + print("Sequence type is set to 'tianowrite'.") + # Safe mode + if safe_mode: + args.append("--safemode=on") + print("Safe mode set to 'on'.") + else: + args.append("--safemode=off") + print("Safe mode set to 'off'.") + else: + args.append("--sequence=regular") + print("Sequence type is set to 'regular'.") + # Test failure policy + args.append("--fpolicy=abort") + print("Test failure policy is set to 'abort'.") + else: + print("Test impact analysis ie disabled.") + # Sequence type + args.append("--sequence=regular") + print("Sequence type is set to 'seed'.") + # Pipeline of truth sequence + if self.__is_pipeline_of_truth: + # Test failure policy + args.append("--fpolicy=continue") + print("Test failure policy is set to 'continue'.") + # Non pipeline of truth sequence + else: + # Test failure policy + args.append("--fpolicy=abort") + print("Test failure policy is set to 'abort'.") + print("Args: ", end='') print(*args) result = subprocess.run([self.__tiaf_bin] + args) - if result.returncode == 0: + # If the sequence completed 9with or without failures) we will update the historical meta-data + if result.returncode == 0 or result.returncode == 7: print("Test impact analysis runtime returned successfully.") - if sequence_type == SequenceType.SEED: + if self.__is_pipeline_of_truth: print("Writing historical meta-data...") self.__write_last_run_hash(self.__dst_commit) print("Complete!") @@ -188,3 +231,48 @@ class TestImpact: print(f"The test impact analysis runtime returned with error: '{result.returncode}'.") return result.returncode + + + #args = [] + #print("Please note: test impact analysis sequences will be run in read-only mode (seed sequences are unaffected).") + #if sequence_type == SequenceType.REGULAR: + # print("Sequence type: regular.") + # args.append("--sequence=regular") + # args.append("--fpolicy=abort") + #elif sequence_type == SequenceType.SEED: + # print("Sequence type: seed.") + # args.append("--sequence=seed") + # args.append("--fpolicy=continue") + #elif sequence_type == SequenceType.TEST_IMPACT_ANALYSIS: + # print("Sequence type: test impact analysis (no write).") + # args.append("--fpolicy=abort") + # if self.__has_change_list: + # args.append(f"-changelist={self.__change_list_path}") + # args.append("--sequence=tianowrite") + # else: + # print(f"No change list was generated, falling back to a regular sequence.") + # print("Sequence type: Regular.") + # args.append("--sequence=regular") + #else: + # raise ValueError(sequence_type) + # + #if test_timeout != None: + # args.append(f"--ttimeout={test_timeout}") + # print(f"Test target timeout is set to {test_timeout} seconds.") + #if global_timeout != None: + # args.append(f"--gtimeout={global_timeout}") + # print(f"Global sequence timeout is set to {test_timeout} seconds.") + # + #print("Args: ", end='') + #print(*args) + #result = subprocess.run([self.__tiaf_bin] + args) + #if result.returncode == 0: + # print("Test impact analysis runtime returned successfully.") + # if sequence_type == SequenceType.SEED: + # print("Writing historical meta-data...") + # self.__write_last_run_hash(self.__dst_commit) + # print("Complete!") + #else: + # print(f"The test impact analysis runtime returned with error: '{result.returncode}'.") + #return result.returncode + diff --git a/scripts/build/TestImpactAnalysis/tiaf_driver.py b/scripts/build/TestImpactAnalysis/tiaf_driver.py index 1cc7ecb630..917fd66fc6 100644 --- a/scripts/build/TestImpactAnalysis/tiaf_driver.py +++ b/scripts/build/TestImpactAnalysis/tiaf_driver.py @@ -11,7 +11,6 @@ import argparse from tiaf import TestImpact -from tiaf import SequenceType import sys import os @@ -26,16 +25,6 @@ def parse_args(): else: raise FileNotFoundError(value) - def sequence_type(value): - if value == "regular": - return SequenceType.REGULAR - elif value == "tia": - return SequenceType.TEST_IMPACT_ANALYSIS - elif value == "seed": - return SequenceType.SEED - else: - raise ValueError(value) - def timout_type(value): value = int(value) if value <= 0: @@ -44,24 +33,20 @@ def parse_args(): parser = argparse.ArgumentParser() parser.add_argument('--config', dest="config", type=file_path, help="Path to the test impact analysis framework configuration file", required=True) - parser.add_argument('--destCommit', dest="dst_commit", help="Commit to run test impact analysis on (not required for seed)") - parser.add_argument('--sequenceType', dest="sequence_type", type=sequence_type, help="Test sequence type to run ('regular', 'seed' or 'tia')", required=True) - parser.add_argument('--suites', dest="suites", nargs='*', help="Suites to include for regular tes sequences (use '*' for all suites)") - parser.add_argument('--testTimeout', dest="test_timeout", type=timout_type, help="Maximum flight time (in seconds) of any test target before being terminated", required=False) - parser.add_argument('--globalTimeout', dest="global_timeout", type=timout_type, help="Maximum tun time of the sequence before being terminated", required=False) - parser.set_defaults(suites="*") - parser.set_defaults(safe_mode=False) + parser.add_argument('--pipeline', dest="pipeline", help="Pipeline the test impact analysis framework is running on", required=True) + parser.add_argument('--destCommit', dest="dst_commit", help="Commit to run test impact analysis on (ignored when seeding)", required=True) + parser.add_argument('--suite', dest="suite", help="Test suite to run", required=True) + parser.add_argument('--safeMode', dest="safe_mode", action='store_true', help="Run impact analysis tests in safe mode (ignored when seeding)") + parser.add_argument('--testTimeout', dest="test_timeout", type=timout_type, help="Maximum run time (in seconds) of any test target before being terminated", required=False) + parser.add_argument('--globalTimeout', dest="global_timeout", type=timout_type, help="Maximum run time of the sequence before being terminated", required=False) parser.set_defaults(test_timeout=None) parser.set_defaults(global_timeout=None) args = parser.parse_args() - - if args.sequence_type == SequenceType.TEST_IMPACT_ANALYSIS and args.dst_commit == None: - raise ValueError("Test impact analysis sequence must have a change list") return args if __name__ == "__main__": args = parse_args() - tiaf = TestImpact(args.config, args.dst_commit) - return_code = tiaf.run(args.sequence_type, args.test_timeout, args.global_timeout) + tiaf = TestImpact(args.config, args.pipeline, args.dst_commit) + return_code = tiaf.run(args.suite, args.safe_mode, args.test_timeout, args.global_timeout) sys.exit(return_code) \ No newline at end of file