Add seed and tia jobs

scripts/build/Platform/Windows/build_config.json

@@ -21,6 +21,7 @@
     ],
     "steps": [
       "debug_vs2019",
+      "test_impact_seed",
       "test_debug_vs2019"
     ]
   },
@@ -30,6 +31,7 @@
     ],
     "steps": [
       "profile_vs2019",
+      "test_impact_analysis",
       "asset_profile_vs2019",
       "test_cpu_profile_vs2019"
     ]
@@ -78,6 +80,24 @@
       "SCRIPT_PARAMETERS": "--platform 3rdParty --type 3rdParty_all"
     }
   },
+  "test_impact_seed": {
+    "TAGS": [
+    ],
+    "COMMAND": "python_windows.cmd",
+    "PARAMETERS": {
+      "SCRIPT_PATH": "scripts/build/TestImpactAnalysis/tiaf_driver.py",
+      "SCRIPT_PARAMETERS": "--sequenceType seed --config \"build\\windows_vs2019\\bin\\TestImpactFramework\\persistent\\tiaf.debug.json\""
+    }
+  },
+  "test_impact_analysis": {
+    "TAGS": [
+    ],
+    "COMMAND": "python_windows.cmd",
+    "PARAMETERS": {
+      "SCRIPT_PATH": "scripts/build/TestImpactAnalysis/tiaf_driver.py",
+      "SCRIPT_PARAMETERS": "--sequenceType tia --destCommit !CHANGE_ID! --config \"build\\windows_vs2019\\bin\\TestImpactFramework\\persistent\\tiaf.profile.json\""
+    }
+  },
   "debug_vs2019": {
     "TAGS": [
         "weekly-build-metrics"
@@ -86,7 +106,7 @@
     "PARAMETERS": {
       "CONFIGURATION": "debug",
       "OUTPUT_DIRECTORY": "build\\windows_vs2019",
-      "CMAKE_OPTIONS": "-G \"Visual Studio 16 2019\" -DCMAKE_SYSTEM_VERSION=10.0 -DLY_UNITY_BUILD=TRUE -DLY_BUILD_WITH_INCREMENTAL_LINKING_DEBUG=FALSE",
+      "CMAKE_OPTIONS": "-G \"Visual Studio 16 2019\" -DCMAKE_SYSTEM_VERSION=10.0 -DLY_UNITY_BUILD=TRUE -DLY_BUILD_WITH_INCREMENTAL_LINKING_DEBUG=FALSE -DLY_TEST_IMPACT_ACTIVE=1 -DLY_TEST_IMPACT_INSTRUMENTATION_BIN=\"c:\\ly\\3rdParty\\ackages\\OpenCppCoverage\\Binary\\windows-x64\\OpenCppCoverage.exe\"",
       "CMAKE_LY_PROJECTS": "AutomatedTesting",
       "CMAKE_TARGET": "ALL_BUILD",
       "CMAKE_NATIVE_BUILD_ARGS": "/m /nologo"
@@ -117,7 +137,7 @@
     "PARAMETERS": {
       "CONFIGURATION": "profile",
       "OUTPUT_DIRECTORY": "build\\windows_vs2019",
-      "CMAKE_OPTIONS": "-G \"Visual Studio 16 2019\" -DCMAKE_SYSTEM_VERSION=10.0 -DLY_UNITY_BUILD=TRUE",
+      "CMAKE_OPTIONS": "-G \"Visual Studio 16 2019\" -DCMAKE_SYSTEM_VERSION=10.0 -DLY_UNITY_BUILD=TRUE  -DLY_TEST_IMPACT_ACTIVE=1 -DLY_TEST_IMPACT_INSTRUMENTATION_BIN=\"c:\\ly\\3rdParty\\ackages\\OpenCppCoverage\\Binary\\windows-x64\\OpenCppCoverage.exe\"",
       "CMAKE_LY_PROJECTS": "AutomatedTesting",
       "CMAKE_TARGET": "ALL_BUILD",
       "CMAKE_NATIVE_BUILD_ARGS": "/m /nologo"

scripts/build/TestImpactAnalysis/tiaf.py

@@ -25,8 +25,12 @@ def is_child_path(parent_path, child_path):
 
 # Enumerations for test sequence types
 class SequenceType(Enum):
+    # Regular sequence as-per the tiaf regular sequence
     REGULAR = 1
+     # TIA sequence as-per the tiaf read-only impact analysis sequence
     TEST_IMPACT_ANALYSIS = 2
+    # Seed sequence as-per the tiaf seed sequence
+    SEED = 3
 
 class TestImpact:
     def __init__(self, config_file, dst_commit):
@@ -43,7 +47,6 @@ class TestImpact:
             self.__tiaf_bin = config["repo"]["tiaf_bin"]
             if not os.path.isfile(self.__tiaf_bin):
                 raise FileNotFoundError("Could not find tiaf binary")
-            self.__source_of_truth = config["repo"]["source_of_truth"]
             self.__active_workspace = config["workspace"]["active"]["root"]
             self.__historic_workspace = config["workspace"]["historic"]["root"]
             self.__temp_workspace = config["workspace"]["temp"]["root"]
@@ -58,16 +61,7 @@ class TestImpact:
         self.__dst_commit = dst_commit
         self.__src_commit = None
         self.__has_src_commit = False
-        if self.__repo.current_branch == self.__source_of_truth:
-            self.__is_source_of_truth = True
-        else:
-            self.__is_source_of_truth = False
         print(f"The repository is located at '{self.__repo_dir}' and the current branch is '{self.__branch}'.")
-        print(f"The source of truth branch is '{self.__source_of_truth}'.")
-        if self.__is_source_of_truth:
-            print("I am the source of truth.")
-        else:
-            print("I am *not* the source of truth.")
 
     # Restricts change lists from checking in test impact analysis files
     def __check_for_restricted_files(self, file_path):
@@ -77,7 +71,7 @@ class TestImpact:
     def __read_last_run_hash(self):
         self.__has_src_commit = False
         if os.path.isfile(self.__last_commit_hash_path):
-            print(f"Previous commit hash found at 'self.__last_commit_hash_path'")            
+            print(f"Previous commit hash found at '{self.__last_commit_hash_path}'")            
             with open(self.__last_commit_hash_path) as file:
                 self.__src_commit = file.read()
                 self.__has_src_commit = True
@@ -104,7 +98,7 @@ class TestImpact:
                 print(e)
                 return
             # A diff was generated, attempt to parse the diff and construct the change list
-            print(f"Generated diff between commits {self.__src_commit} and {self.__dst_commit}.") 
+            print(f"Generated diff between commits '{self.__src_commit}' and '{self.__dst_commit}': '{diff_path}'.") 
             change_list = {}
             change_list["createdFiles"] = []
             change_list["updatedFiles"] = []
@@ -152,35 +146,27 @@ class TestImpact:
     # Runs the specified test sequence
     def run(self, sequence_type, safe_mode, test_timeout, global_timeout):
         args = []
-        if self.__has_change_list:
-            args.append(f"-changelist={self.__change_list_path}")
-            if sequence_type == SequenceType.REGULAR:
-                print("Sequence type: regular.")
-                args.append("--sequence=regular")
-            elif sequence_type == SequenceType.TEST_IMPACT_ANALYSIS:
-                print("Sequence type: test impact analysis.")
-                if self.__is_source_of_truth:
-                    # Source of truth branch will allways attempt a seed if no test impact analysis data is available
-                    print("This branch is the source of truth, a seed sequence will be run if there is no test impact analysis data.")
-                    args.append("--sequence=tiaorseed")
-                    args.append("--fpolicy=continue")
-                else:
-                    # Non source of truth branches will fall back to a regular test run if no test impact analysis data is available
-                    print("This branch is not the source of truth, a regular sequence will be run if there is no test impact analysis data.")
-                    args.append("--sequence=tia")
-                    args.append("--fpolicy=abort")
-                
-                if safe_mode == True:
-                    print("Safe mode is on, the discarded test targets will be run after the selected test targets.")
-                    args.append("--safemode=on")
-                else:
-                    print("Safe mode is off, the discarded test targets will not be run.")
-            else:
-                raise ValueError(sequence_type)
-        else:
-            print(f"No change list was generated, this will cause test impact analysis sequences on branches other than the source of truth to fall back to a regular sequence.")
-            print("Sequence type: Regular.")
+        print("Please note: test impact analysis sequences will be run in read-only mode (seed sequences are unaffected).")
+        if sequence_type == SequenceType.REGULAR:
+            print("Sequence type: regular.")
             args.append("--sequence=regular")
+            args.append("--fpolicy=abort")
+        elif sequence_type == SequenceType.SEED:
+            print("Sequence type: seed.")
+            args.append("--sequence=seed")
+            args.append("--fpolicy=continue")
+        elif sequence_type == SequenceType.TEST_IMPACT_ANALYSIS:
+            print("Sequence type: test impact analysis (no write).")
+            args.append("--fpolicy=abort")
+            if self.__has_change_list:
+                args.append(f"-changelist={self.__change_list_path}")
+                args.append("--sequence=tianowrite")
+            else:
+                print(f"No change list was generated, falling back to a regular sequence.")
+                print("Sequence type: Regular.")
+                args.append("--sequence=regular")
+        else:
+            raise ValueError(sequence_type)
 
         if test_timeout != None:
             args.append(f"--ttimeout={test_timeout}")
@@ -193,8 +179,10 @@ class TestImpact:
         print(*args)
         result = subprocess.run([self.__tiaf_bin] + args)
         if result.returncode == 0:
-            print("Test impact analysis runtime returned successfully. Updating historical artifacts...")
-            self.__write_last_run_hash(self.__dst_commit)
+            print("Test impact analysis runtime returned successfully.")
+            if sequence_type == SequenceType.SEED:
+                print("Writing historical meta-data...")
+                self.__write_last_run_hash(self.__dst_commit)
             print("Complete!")
         else:
             print(f"The test impact analysis runtime returned with error: '{result.returncode}'.")

scripts/build/TestImpactAnalysis/tiaf_driver.py

@@ -31,6 +31,8 @@ def parse_args():
             return SequenceType.REGULAR
         elif value == "tia":
             return SequenceType.TEST_IMPACT_ANALYSIS
+        elif value == "seed":
+            return SequenceType.SEED
         else:
             raise ValueError(value)
 
@@ -42,23 +44,24 @@ def parse_args():
             
     parser = argparse.ArgumentParser()
     parser.add_argument('--config', dest="config", type=file_path, help="Path to the test impact analysis framework configuration file", required=True)
-    parser.add_argument('--destCommit', dest="dst_commit", help="Commit to run test impact analysis on (if empty, HEAD^ will be used)")
-    parser.add_argument('--sequenceType', dest="sequence_type", type=sequence_type, help="Test sequence type to run ('regular' or 'tia')", required=True)
+    parser.add_argument('--destCommit', dest="dst_commit", help="Commit to run test impact analysis on (not required for seed)")
+    parser.add_argument('--sequenceType', dest="sequence_type", type=sequence_type, help="Test sequence type to run ('regular', 'seed' or 'tia')", required=True)
     parser.add_argument('--suites', dest="suites", nargs='*', help="Suites to include for regular tes sequences (use '*' for all suites)")
-    parser.add_argument('--safeMode', dest='safe_mode', help="If set, will run any test impact analysis runs in safe mode (unselected tests will still be run)")
     parser.add_argument('--testTimeout', dest="test_timeout", type=timout_type, help="Maximum flight time (in seconds) of any test target before being terminated", required=False)
     parser.add_argument('--globalTimeout', dest="global_timeout", type=timout_type, help="Maximum tun time of the sequence before being terminated", required=False)
-    parser.set_defaults(dst_commit="HEAD^")
     parser.set_defaults(suites="*")
     parser.set_defaults(safe_mode=False)
     parser.set_defaults(test_timeout=None)
     parser.set_defaults(global_timeout=None)
     args = parser.parse_args()
+
+    if args.sequence_type == SequenceType.TEST_IMPACT_ANALYSIS and args.dst_commit == None:
+        raise ValueError("Test impact analysis sequence must have a change list")
     
     return args
 
 if __name__ == "__main__":
     args = parse_args()
     tiaf = TestImpact(args.config, args.dst_commit)
-    return_code = tiaf.run(args.sequence_type, args.safe_mode, args.test_timeout, args.global_timeout)
+    return_code = tiaf.run(args.sequence_type, args.test_timeout, args.global_timeout)
     sys.exit(return_code)