diff --git a/scripts/build/Platform/Windows/build_config.json b/scripts/build/Platform/Windows/build_config.json index bd10c9e59a..081855b05a 100644 --- a/scripts/build/Platform/Windows/build_config.json +++ b/scripts/build/Platform/Windows/build_config.json @@ -21,6 +21,7 @@ ], "steps": [ "debug_vs2019", + "test_impact_seed", "test_debug_vs2019" ] }, @@ -30,6 +31,7 @@ ], "steps": [ "profile_vs2019", + "test_impact_analysis", "asset_profile_vs2019", "test_cpu_profile_vs2019" ] @@ -78,6 +80,24 @@ "SCRIPT_PARAMETERS": "--platform 3rdParty --type 3rdParty_all" } }, + "test_impact_seed": { + "TAGS": [ + ], + "COMMAND": "python_windows.cmd", + "PARAMETERS": { + "SCRIPT_PATH": "scripts/build/TestImpactAnalysis/tiaf_driver.py", + "SCRIPT_PARAMETERS": "--sequenceType seed --config \"build\\windows_vs2019\\bin\\TestImpactFramework\\persistent\\tiaf.debug.json\"" + } + }, + "test_impact_analysis": { + "TAGS": [ + ], + "COMMAND": "python_windows.cmd", + "PARAMETERS": { + "SCRIPT_PATH": "scripts/build/TestImpactAnalysis/tiaf_driver.py", + "SCRIPT_PARAMETERS": "--sequenceType tia --destCommit !CHANGE_ID! --config \"build\\windows_vs2019\\bin\\TestImpactFramework\\persistent\\tiaf.profile.json\"" + } + }, "debug_vs2019": { "TAGS": [ "weekly-build-metrics" @@ -86,7 +106,7 @@ "PARAMETERS": { "CONFIGURATION": "debug", "OUTPUT_DIRECTORY": "build\\windows_vs2019", - "CMAKE_OPTIONS": "-G \"Visual Studio 16 2019\" -DCMAKE_SYSTEM_VERSION=10.0 -DLY_UNITY_BUILD=TRUE -DLY_BUILD_WITH_INCREMENTAL_LINKING_DEBUG=FALSE", + "CMAKE_OPTIONS": "-G \"Visual Studio 16 2019\" -DCMAKE_SYSTEM_VERSION=10.0 -DLY_UNITY_BUILD=TRUE -DLY_BUILD_WITH_INCREMENTAL_LINKING_DEBUG=FALSE -DLY_TEST_IMPACT_ACTIVE=1 -DLY_TEST_IMPACT_INSTRUMENTATION_BIN=\"c:\\ly\\3rdParty\\ackages\\OpenCppCoverage\\Binary\\windows-x64\\OpenCppCoverage.exe\"", "CMAKE_LY_PROJECTS": "AutomatedTesting", "CMAKE_TARGET": "ALL_BUILD", "CMAKE_NATIVE_BUILD_ARGS": "/m /nologo" @@ -117,7 +137,7 @@ "PARAMETERS": { "CONFIGURATION": "profile", "OUTPUT_DIRECTORY": "build\\windows_vs2019", - "CMAKE_OPTIONS": "-G \"Visual Studio 16 2019\" -DCMAKE_SYSTEM_VERSION=10.0 -DLY_UNITY_BUILD=TRUE", + "CMAKE_OPTIONS": "-G \"Visual Studio 16 2019\" -DCMAKE_SYSTEM_VERSION=10.0 -DLY_UNITY_BUILD=TRUE -DLY_TEST_IMPACT_ACTIVE=1 -DLY_TEST_IMPACT_INSTRUMENTATION_BIN=\"c:\\ly\\3rdParty\\ackages\\OpenCppCoverage\\Binary\\windows-x64\\OpenCppCoverage.exe\"", "CMAKE_LY_PROJECTS": "AutomatedTesting", "CMAKE_TARGET": "ALL_BUILD", "CMAKE_NATIVE_BUILD_ARGS": "/m /nologo" diff --git a/scripts/build/TestImpactAnalysis/tiaf.py b/scripts/build/TestImpactAnalysis/tiaf.py index 4686c08f93..01e2cde8ba 100644 --- a/scripts/build/TestImpactAnalysis/tiaf.py +++ b/scripts/build/TestImpactAnalysis/tiaf.py @@ -25,8 +25,12 @@ def is_child_path(parent_path, child_path): # Enumerations for test sequence types class SequenceType(Enum): + # Regular sequence as-per the tiaf regular sequence REGULAR = 1 + # TIA sequence as-per the tiaf read-only impact analysis sequence TEST_IMPACT_ANALYSIS = 2 + # Seed sequence as-per the tiaf seed sequence + SEED = 3 class TestImpact: def __init__(self, config_file, dst_commit): @@ -43,7 +47,6 @@ class TestImpact: self.__tiaf_bin = config["repo"]["tiaf_bin"] if not os.path.isfile(self.__tiaf_bin): raise FileNotFoundError("Could not find tiaf binary") - self.__source_of_truth = config["repo"]["source_of_truth"] self.__active_workspace = config["workspace"]["active"]["root"] self.__historic_workspace = config["workspace"]["historic"]["root"] self.__temp_workspace = config["workspace"]["temp"]["root"] @@ -58,16 +61,7 @@ class TestImpact: self.__dst_commit = dst_commit self.__src_commit = None self.__has_src_commit = False - if self.__repo.current_branch == self.__source_of_truth: - self.__is_source_of_truth = True - else: - self.__is_source_of_truth = False print(f"The repository is located at '{self.__repo_dir}' and the current branch is '{self.__branch}'.") - print(f"The source of truth branch is '{self.__source_of_truth}'.") - if self.__is_source_of_truth: - print("I am the source of truth.") - else: - print("I am *not* the source of truth.") # Restricts change lists from checking in test impact analysis files def __check_for_restricted_files(self, file_path): @@ -77,7 +71,7 @@ class TestImpact: def __read_last_run_hash(self): self.__has_src_commit = False if os.path.isfile(self.__last_commit_hash_path): - print(f"Previous commit hash found at 'self.__last_commit_hash_path'") + print(f"Previous commit hash found at '{self.__last_commit_hash_path}'") with open(self.__last_commit_hash_path) as file: self.__src_commit = file.read() self.__has_src_commit = True @@ -104,7 +98,7 @@ class TestImpact: print(e) return # A diff was generated, attempt to parse the diff and construct the change list - print(f"Generated diff between commits {self.__src_commit} and {self.__dst_commit}.") + print(f"Generated diff between commits '{self.__src_commit}' and '{self.__dst_commit}': '{diff_path}'.") change_list = {} change_list["createdFiles"] = [] change_list["updatedFiles"] = [] @@ -152,35 +146,27 @@ class TestImpact: # Runs the specified test sequence def run(self, sequence_type, safe_mode, test_timeout, global_timeout): args = [] - if self.__has_change_list: - args.append(f"-changelist={self.__change_list_path}") - if sequence_type == SequenceType.REGULAR: - print("Sequence type: regular.") - args.append("--sequence=regular") - elif sequence_type == SequenceType.TEST_IMPACT_ANALYSIS: - print("Sequence type: test impact analysis.") - if self.__is_source_of_truth: - # Source of truth branch will allways attempt a seed if no test impact analysis data is available - print("This branch is the source of truth, a seed sequence will be run if there is no test impact analysis data.") - args.append("--sequence=tiaorseed") - args.append("--fpolicy=continue") - else: - # Non source of truth branches will fall back to a regular test run if no test impact analysis data is available - print("This branch is not the source of truth, a regular sequence will be run if there is no test impact analysis data.") - args.append("--sequence=tia") - args.append("--fpolicy=abort") - - if safe_mode == True: - print("Safe mode is on, the discarded test targets will be run after the selected test targets.") - args.append("--safemode=on") - else: - print("Safe mode is off, the discarded test targets will not be run.") - else: - raise ValueError(sequence_type) - else: - print(f"No change list was generated, this will cause test impact analysis sequences on branches other than the source of truth to fall back to a regular sequence.") - print("Sequence type: Regular.") + print("Please note: test impact analysis sequences will be run in read-only mode (seed sequences are unaffected).") + if sequence_type == SequenceType.REGULAR: + print("Sequence type: regular.") args.append("--sequence=regular") + args.append("--fpolicy=abort") + elif sequence_type == SequenceType.SEED: + print("Sequence type: seed.") + args.append("--sequence=seed") + args.append("--fpolicy=continue") + elif sequence_type == SequenceType.TEST_IMPACT_ANALYSIS: + print("Sequence type: test impact analysis (no write).") + args.append("--fpolicy=abort") + if self.__has_change_list: + args.append(f"-changelist={self.__change_list_path}") + args.append("--sequence=tianowrite") + else: + print(f"No change list was generated, falling back to a regular sequence.") + print("Sequence type: Regular.") + args.append("--sequence=regular") + else: + raise ValueError(sequence_type) if test_timeout != None: args.append(f"--ttimeout={test_timeout}") @@ -193,8 +179,10 @@ class TestImpact: print(*args) result = subprocess.run([self.__tiaf_bin] + args) if result.returncode == 0: - print("Test impact analysis runtime returned successfully. Updating historical artifacts...") - self.__write_last_run_hash(self.__dst_commit) + print("Test impact analysis runtime returned successfully.") + if sequence_type == SequenceType.SEED: + print("Writing historical meta-data...") + self.__write_last_run_hash(self.__dst_commit) print("Complete!") else: print(f"The test impact analysis runtime returned with error: '{result.returncode}'.") diff --git a/scripts/build/TestImpactAnalysis/tiaf_driver.py b/scripts/build/TestImpactAnalysis/tiaf_driver.py index 4f20fa1070..1cc7ecb630 100644 --- a/scripts/build/TestImpactAnalysis/tiaf_driver.py +++ b/scripts/build/TestImpactAnalysis/tiaf_driver.py @@ -31,6 +31,8 @@ def parse_args(): return SequenceType.REGULAR elif value == "tia": return SequenceType.TEST_IMPACT_ANALYSIS + elif value == "seed": + return SequenceType.SEED else: raise ValueError(value) @@ -42,23 +44,24 @@ def parse_args(): parser = argparse.ArgumentParser() parser.add_argument('--config', dest="config", type=file_path, help="Path to the test impact analysis framework configuration file", required=True) - parser.add_argument('--destCommit', dest="dst_commit", help="Commit to run test impact analysis on (if empty, HEAD^ will be used)") - parser.add_argument('--sequenceType', dest="sequence_type", type=sequence_type, help="Test sequence type to run ('regular' or 'tia')", required=True) + parser.add_argument('--destCommit', dest="dst_commit", help="Commit to run test impact analysis on (not required for seed)") + parser.add_argument('--sequenceType', dest="sequence_type", type=sequence_type, help="Test sequence type to run ('regular', 'seed' or 'tia')", required=True) parser.add_argument('--suites', dest="suites", nargs='*', help="Suites to include for regular tes sequences (use '*' for all suites)") - parser.add_argument('--safeMode', dest='safe_mode', help="If set, will run any test impact analysis runs in safe mode (unselected tests will still be run)") parser.add_argument('--testTimeout', dest="test_timeout", type=timout_type, help="Maximum flight time (in seconds) of any test target before being terminated", required=False) parser.add_argument('--globalTimeout', dest="global_timeout", type=timout_type, help="Maximum tun time of the sequence before being terminated", required=False) - parser.set_defaults(dst_commit="HEAD^") parser.set_defaults(suites="*") parser.set_defaults(safe_mode=False) parser.set_defaults(test_timeout=None) parser.set_defaults(global_timeout=None) args = parser.parse_args() + + if args.sequence_type == SequenceType.TEST_IMPACT_ANALYSIS and args.dst_commit == None: + raise ValueError("Test impact analysis sequence must have a change list") return args if __name__ == "__main__": args = parse_args() tiaf = TestImpact(args.config, args.dst_commit) - return_code = tiaf.run(args.sequence_type, args.safe_mode, args.test_timeout, args.global_timeout) + return_code = tiaf.run(args.sequence_type, args.test_timeout, args.global_timeout) sys.exit(return_code) \ No newline at end of file