Merge pull request #50 from aws-lumberyard-dev/TIF/Jenkins

Runtime refactor

Code/Tools/TestImpactFramework/Runtime/Code/Include/TestImpactFramework/TestImpactClientFailureReport.h

@@ -44,19 +44,6 @@ namespace TestImpact
             AZStd::string m_commandString;
         };
 
-        //! Represents a test target that terminated abnormally.
-        class LauncherFailure
-            : public ExecutionFailure
-        {
-        public:
-            LauncherFailure(const AZStd::string& targetName, const AZStd::string& command, int returnCode);
-
-            //! The return code of the test target that terminated abnormally.
-            int GetReturnCode() const;
-        private:
-            int m_returnCode;
-        };
-
         //! Represents an individual test of a test target that failed.
         class TestFailure
         {
@@ -107,6 +94,7 @@ namespace TestImpact
 
         private:
             AZStd::vector<TestCaseFailure> m_testCaseFailures;
+            size_t m_numTestFailures = 0;
         };
 
         //! Base class for reporting failing test sequences.
@@ -115,63 +103,27 @@ namespace TestImpact
         public:
             SequenceFailure(
                 AZStd::vector<ExecutionFailure>&& executionFailures,
-                AZStd::vector<LauncherFailure>&& launcherFailures,
+                AZStd::vector<TestRunFailure>&& testRunFailures,
+                AZStd::vector<TargetFailure>&& timedOutTests,
                 AZStd::vector<TargetFailure>&& unexecutedTests);
 
             //! Returns the test targets in this sequence that failed to execute.
             const AZStd::vector<ExecutionFailure>& GetExecutionFailures() const;
 
-            //! Returns the test targets in this sequence that terminated abnormally.
-            const AZStd::vector<LauncherFailure>& GetLauncherFailures() const;
-
-            //! Returns the test targets in this sequence that were not executed due to the sequence terminating prematurely.
-            const AZStd::vector<TargetFailure>& GetUnexecutedTest() const;
-
-        private:
-            AZStd::vector<ExecutionFailure> m_executionFailures;
-            AZStd::vector<LauncherFailure> m_launcherFailures;
-            AZStd::vector<TargetFailure> m_unexecutedTests;
-        };
-
-        //! Represents the report for a failed regular test sequence run without test impact analysis.
-        class RegularSequenceFailure
-            : public SequenceFailure
-        {
-        public:
-            RegularSequenceFailure(
-                AZStd::vector<ExecutionFailure>&& executionFailures,
-                AZStd::vector<LauncherFailure>&& launcherFailures,
-                AZStd::vector<TestRunFailure>&& testRunFailures,
-                AZStd::vector<TargetFailure>&& unexecutedTests);
-
             //! Returns the test targets that contain failing tests.
             const AZStd::vector<TestRunFailure>& GetTestRunFailures() const;
 
+            //! Returns the test targets in this sequence that were terminated for exceeding their allotted runtime.
+            const AZStd::vector<TargetFailure>& GetTimedOutTests() const;
+
+            //! Returns the test targets in this sequence that were not executed due to the sequence terminating prematurely.
+            const AZStd::vector<TargetFailure>& GetUnexecutedTests() const;
+
         private:
+            AZStd::vector<ExecutionFailure> m_executionFailures;
             AZStd::vector<TestRunFailure> m_testRunFailures;
-        };
+            AZStd::vector<TargetFailure> m_timedOutTests;
-
+            AZStd::vector<TargetFailure> m_unexecutedTests;
-        //! Represents the report for a failed test sequence run with test impact analysis.
-        class ImpactAnalysisSequenceFailure
-            : public SequenceFailure
-        {
-        public:
-            ImpactAnalysisSequenceFailure(
-                AZStd::vector<ExecutionFailure>&& executionFailures,
-                AZStd::vector<LauncherFailure>&& launcherFailures,
-                AZStd::vector<TestRunFailure>&& selectedTestRunFailures,
-                AZStd::vector<TestRunFailure>&& discardedTestRunFailures,
-                AZStd::vector<TargetFailure>&& unexecutedTests);
-
-            //! Returns the test targets that were selected to run but contain failing tests.
-            const AZStd::vector<TestRunFailure> GetSelectedTestRunFailures() const;
-
-            //! Returns the test targets that were not selected but still run but contain failing tests.
-            const AZStd::vector<TestRunFailure> GetDiscardedTestRunFailures() const;
-
-        private:
-            AZStd::vector<TestRunFailure> m_selectedTestRunFailures;
-            AZStd::vector<TestRunFailure> m_discardedTestRunFailures;
         };
     } // namespace Client
 } // namespace TestImpact

Code/Tools/TestImpactFramework/Runtime/Code/Include/TestImpactFramework/TestImpactClientTestSelection.h

@@ -25,6 +25,7 @@ namespace TestImpact
         class TestRunSelection
         {
         public:
+            TestRunSelection(const AZStd::vector<AZStd::string>& includedTests, const AZStd::vector<AZStd::string>& excludedTests);
             TestRunSelection(AZStd::vector<AZStd::string>&& includedTests, AZStd::vector<AZStd::string>&& excludedTests);
 
             //! Returns the test runs that were selected to be run and will actually be run.
@@ -37,7 +38,10 @@ namespace TestImpact
             size_t GetNumIncludedTestRuns() const;
 
             //! Returns the number of selected test runs that will not be run.
-            size_t GetNumNumExcludedTestRuns() const;
+            size_t GetNumExcludedTestRuns() const;
+
+            //! Returns the total number of test runs selected regardless of whether or not they will actually be run.
+            size_t GetTotalNumTests() const;
 
         private:
             AZStd::vector<AZStd::string> m_includedTestRuns;

Code/Tools/TestImpactFramework/Runtime/Code/Include/TestImpactFramework/TestImpactFileUtils.h

@@ -59,4 +59,27 @@ namespace TestImpact
         AZ_TestImpact_Eval(
             file.Write(bytes.data(), bytes.size()), ExceptionType, AZStd::string::format("Couldn't write contents for file %s", path.c_str()));
     }
+
+    //! Delete the files that match the pattern from the specified directory.
+    //! @param path The path to the directory to pattern match the files for deletion.
+    //! @param pattern The pattern to match files for deletion.
+    inline void DeleteFiles(const RepoPath& path, const AZStd::string& pattern)
+    {
+        AZ::IO::SystemFile::FindFiles(AZStd::string::format("%s/%s", path.c_str(), pattern.c_str()).c_str(),
+            [&path](const char* file, bool isFile)
+        {
+            if (isFile)
+            {
+                AZ::IO::SystemFile::Delete(AZStd::string::format("%s/%s", path.c_str(), file).c_str());
+            }
+
+            return true;
+        });
+    }
+
+    //! Deletes the specified file.
+    inline void DeleteFile(const RepoPath& file)
+    {
+        DeleteFiles(file.ParentPath(), file.Filename().Native());
+    }
 } // namespace TestImpact

Code/Tools/TestImpactFramework/Runtime/Code/Include/TestImpactFramework/TestImpactRepoPath.h

@@ -21,22 +21,78 @@ namespace TestImpact
     //! to the test impact analysis data as otherwise querying/retrieving test impact analysis data for the same source albeit
     //! with different path separators will be considered different files entirely.
     class RepoPath
-        : public AZ::IO::Path
     {
     public:
+        using string_type = AZ::IO::Path::string_type;
+        using string_view_type = AZ::IO::Path::string_view_type;
+        using value_type = AZ::IO::Path::value_type;
+
         constexpr RepoPath() = default;
         constexpr RepoPath(const RepoPath&) = default;
         constexpr RepoPath(RepoPath&&) noexcept = default;
-        constexpr RepoPath(const string_type&) noexcept;
+        constexpr RepoPath::RepoPath(const string_type& path) noexcept;
-        constexpr RepoPath(const value_type*) noexcept;
+        constexpr RepoPath::RepoPath(const string_view_type& path) noexcept;
-        constexpr RepoPath(const AZ::IO::PathView&);
+        constexpr RepoPath::RepoPath(const value_type* path) noexcept;
-        constexpr RepoPath(const AZ::IO::Path&);
+        constexpr RepoPath::RepoPath(const AZ::IO::PathView& path);
+        constexpr RepoPath::RepoPath(const AZ::IO::Path& path);
 
         RepoPath& operator=(const RepoPath&) noexcept = default;
         RepoPath& operator=(const string_type&) noexcept;
         RepoPath& operator=(const value_type*) noexcept;
         RepoPath& operator=(const AZ::IO::Path& str) noexcept;
 
-        using AZ::IO::Path::operator AZ::IO::PathView;
+        const char* c_str() const { return m_path.c_str(); }
+        AZStd::string  String() const { return m_path.String(); }
+        constexpr AZ::IO::PathView Stem() const { return m_path.Stem(); }
+        constexpr AZ::IO::PathView Extension() const { return m_path.Extension(); }
+        constexpr bool empty() const { return m_path.empty(); }
+        constexpr AZ::IO::PathView ParentPath() const { return m_path.ParentPath(); }
+        constexpr AZ::IO::PathView Filename() const { return m_path.Filename(); }
+        AZ::IO::Path LexicallyRelative(const RepoPath& base) const { return m_path.LexicallyRelative(base.m_path); }
+        [[nodiscard]] bool IsRelativeTo(const RepoPath& base) const { return m_path.IsRelativeTo(base.m_path); }
+        constexpr AZ::IO::PathView RootName() const { return m_path.RootName(); }
+        constexpr AZ::IO::PathView RelativePath() const { return m_path.RelativePath(); }
+
+        // Wrappers around the AZ::IO::Path concatenation operator
+        friend RepoPath operator/(const RepoPath& lhs, const AZ::IO::PathView& rhs);
+        friend RepoPath operator/(const RepoPath& lhs, AZStd::string_view rhs);
+        friend RepoPath operator/(const RepoPath& lhs, const typename value_type* rhs);
+        friend RepoPath operator/(const RepoPath& lhs, const RepoPath& rhs);
+        RepoPath& operator/=(const AZ::IO::PathView& rhs);
+        RepoPath& operator/=(AZStd::string_view rhs);
+        RepoPath& operator/=(const typename value_type* rhs);
+        RepoPath& operator/=(const RepoPath& rhs);
+
+        friend bool operator==(const RepoPath& lhs, const RepoPath& rhs) noexcept;
+        friend bool operator!=(const RepoPath& lhs, const RepoPath& rhs) noexcept;
+        friend bool operator<(const RepoPath& lhs, const RepoPath& rhs) noexcept;
+
+    private:
+        AZ::IO::Path m_path;
     };
+
+    constexpr RepoPath::RepoPath(const string_type& path) noexcept
+        : m_path(AZ::IO::Path(path).MakePreferred())
+    {
+    }
+
+    constexpr RepoPath::RepoPath(const string_view_type& path) noexcept
+        : m_path(AZ::IO::Path(path).MakePreferred())
+    {
+    }
+
+    constexpr RepoPath::RepoPath(const value_type* path) noexcept
+        : m_path(AZ::IO::Path(path).MakePreferred())
+    {
+    }
+
+    constexpr RepoPath::RepoPath(const AZ::IO::PathView& path)
+        : m_path(AZ::IO::Path(path).MakePreferred())
+    {
+    }
+
+    constexpr RepoPath::RepoPath(const AZ::IO::Path& path)
+        : m_path(AZ::IO::Path(path).MakePreferred())
+    {
+    }
 } // namespace TestImpact

Code/Tools/TestImpactFramework/Runtime/Code/Include/TestImpactFramework/TestImpactRuntime.h

@@ -29,9 +29,12 @@
 
 namespace TestImpact
 {
+    class ChangeDependencyList;
     class DynamicDependencyMap;
+    class TestSelectorAndPrioritizer;
     class TestEngine;
     class TestTarget;
+    class SourceCoveringTestsList;
 
     //! Callback for a test sequence that isn't using test impact analysis to determine selected tests.
     //! @param tests The tests that will be run for this sequence.
@@ -66,16 +69,22 @@ namespace TestImpact
     //! Callback for end of a test sequence.
     //! @param failureReport The test runs that failed for any reason during this sequence.
     //! @param duration The total duration of this test sequence.
-    using TestSequenceCompleteCallback = AZStd::function<void(Client::RegularSequenceFailure&& failureReport, AZStd::chrono::milliseconds duration)>;
+    using TestSequenceCompleteCallback = AZStd::function<void(
+        Client::SequenceFailure&& failureReport,
+        AZStd::chrono::milliseconds duration)>;
 
     //! Callback for end of a test impact analysis test sequence.
-    //! @param failureReport The test runs that failed for any reason during this sequence.
+    //! @param selectedFailureReport The selected test runs that failed for any reason during this sequence.
+    //! @param discardedFailureReport The discarded test runs that failed for any reason during this sequence.
     //! @param duration The total duration of this test sequence.
-    using ImpactAnalysisTestSequenceCompleteCallback = AZStd::function<void(Client::ImpactAnalysisSequenceFailure&& failureReport, AZStd::chrono::milliseconds duration)>;
+    using SafeTestSequenceCompleteCallback = AZStd::function<void(
+        Client::SequenceFailure&& selectedFailureReport,
+        Client::SequenceFailure&& discardedFailureReport,
+        AZStd::chrono::milliseconds duration)>;
 
     //! Callback for test runs that have completed for any reason.
-    //! test The test that has completed.
+    //! @param selectedTests The test that has completed.
-    using TestCompleteCallback = AZStd::function<void(Client::TestRun&& test)>;
+    using TestRunCompleteCallback = AZStd::function<void(Client::TestRun&& selectedTests)>;
 
     //! The API exposed to the client responsible for all test runs and persistent data management.
     class Runtime
@@ -95,7 +104,7 @@ namespace TestImpact
             Policy::TestFailure testFailurePolicy,
             Policy::IntegrityFailure integrationFailurePolicy,
             Policy::TestSharding testShardingPolicy,
-            TargetOutputCapture targetOutputCapture,
+            Policy::TargetOutputCapture targetOutputCapture,
             AZStd::optional<size_t> maxConcurrency = AZStd::nullopt);
 
         ~Runtime();
@@ -103,78 +112,110 @@ namespace TestImpact
         //! Runs a test sequence where all tests with a matching suite in the suite filter and also not on the excluded list are selected.
         //! @param suitesFilter The test suites that will be included in the test selection.
         //! @param testTargetTimeout The maximum duration individual test targets may be in flight for (infinite if empty).
-        //! @param testTargetTimeout The maximum duration the entire test sequence may run for (infinite if empty).
+        //! @param globalTimeout The maximum duration the entire test sequence may run for (infinite if empty).
         //! @param testSequenceStartCallback The client function to be called after the test targets have been selected but prior to running the tests.
         //! @param testSequenceCompleteCallback The client function to be called after the test sequence has completed.
         //! @param testRunCompleteCallback The client function to be called after an individual test run has completed.
+        //! @returns
         TestSequenceResult RegularTestSequence(
             const AZStd::unordered_set<AZStd::string> suitesFilter,
             AZStd::optional<AZStd::chrono::milliseconds> testTargetTimeout,
             AZStd::optional<AZStd::chrono::milliseconds> globalTimeout,
             AZStd::optional<TestSequenceStartCallback> testSequenceStartCallback,
             AZStd::optional<TestSequenceCompleteCallback> testSequenceCompleteCallback,
-            AZStd::optional<TestCompleteCallback> testRunCompleteCallback);
+            AZStd::optional<TestRunCompleteCallback> testRunCompleteCallback);
 
         //! Runs a test sequence where tests are selected according to test impact analysis so long as they are not on the excluded list.
         //! @param changeList The change list used to determine the tests to select.
         //! @param testPrioritizationPolicy Determines how selected tests will be prioritized.
         //! @param testTargetTimeout The maximum duration individual test targets may be in flight for (infinite if empty).
-        //! @param testTargetTimeout The maximum duration the entire test sequence may run for (infinite if empty).
+        //! @param globalTimeout The maximum duration the entire test sequence may run for (infinite if empty).
         //! @param testSequenceStartCallback The client function to be called after the test targets have been selected but prior to running the tests.
         //! @param testSequenceCompleteCallback The client function to be called after the test sequence has completed.
         //! @param testRunCompleteCallback The client function to be called after an individual test run has completed.
+        //! @returns
         TestSequenceResult ImpactAnalysisTestSequence(
             const ChangeList& changeList,
             Policy::TestPrioritization testPrioritizationPolicy,
             AZStd::optional<AZStd::chrono::milliseconds> testTargetTimeout,
             AZStd::optional<AZStd::chrono::milliseconds> globalTimeout,
             AZStd::optional<ImpactAnalysisTestSequenceStartCallback> testSequenceStartCallback,
-            AZStd::optional<ImpactAnalysisTestSequenceCompleteCallback> testSequenceCompleteCallback,
+            AZStd::optional<TestSequenceCompleteCallback> testSequenceCompleteCallback,
-            AZStd::optional<TestCompleteCallback> testRunCompleteCallback);
+            AZStd::optional<TestRunCompleteCallback> testRunCompleteCallback);
 
         //! Runs a test sequence as per the ImpactAnalysisTestSequence where the tests not selected are also run (albeit without instrumentation).
         //! @param changeList The change list used to determine the tests to select.
         //! @param suitesFilter The test suites that will be included in the test selection.
         //! @param testPrioritizationPolicy Determines how selected tests will be prioritized.
         //! @param testTargetTimeout The maximum duration individual test targets may be in flight for (infinite if empty).
-        //! @param testTargetTimeout The maximum duration the entire test sequence may run for (infinite if empty).
+        //! @param globalTimeout The maximum duration the entire test sequence may run for (infinite if empty).
         //! @param testSequenceStartCallback The client function to be called after the test targets have been selected but prior to running the tests.
         //! @param testSequenceCompleteCallback The client function to be called after the test sequence has completed.
         //! @param testRunCompleteCallback The client function to be called after an individual test run has completed.
-        TestSequenceResult SafeImpactAnalysisTestSequence(
+        //! @returns
+        AZStd::pair<TestSequenceResult, TestSequenceResult> SafeImpactAnalysisTestSequence(
             const ChangeList& changeList,
             const AZStd::unordered_set<AZStd::string> suitesFilter,
             Policy::TestPrioritization testPrioritizationPolicy,
             AZStd::optional<AZStd::chrono::milliseconds> testTargetTimeout,
             AZStd::optional<AZStd::chrono::milliseconds> globalTimeout,
             AZStd::optional<SafeImpactAnalysisTestSequenceStartCallback> testSequenceStartCallback,
-            AZStd::optional<ImpactAnalysisTestSequenceCompleteCallback> testSequenceCompleteCallback,
+            AZStd::optional<SafeTestSequenceCompleteCallback> testSequenceCompleteCallback,
-            AZStd::optional<TestCompleteCallback> testRunCompleteCallback);
+            AZStd::optional<TestRunCompleteCallback> testRunCompleteCallback);
 
         //! Runs all tests not on the excluded list and uses their coverage data to seed the test impact analysis data (ant existing data will be overwritten).
-        //! @param testTargetTimeout The maximum duration the entire test sequence may run for (infinite if empty).
+        //! @param testTargetTimeout The maximum duration individual test targets may be in flight for (infinite if empty).
+        //! @param globalTimeout The maximum duration the entire test sequence may run for (infinite if empty).
         //! @param testSequenceStartCallback The client function to be called after the test targets have been selected but prior to running the tests.
         //! @param testSequenceCompleteCallback The client function to be called after the test sequence has completed.
         //! @param testRunCompleteCallback The client function to be called after an individual test run has completed.
+        //! 
         TestSequenceResult SeededTestSequence(
+            AZStd::optional<AZStd::chrono::milliseconds> testTargetTimeout,
             AZStd::optional<AZStd::chrono::milliseconds> globalTimeout,
             AZStd::optional<TestSequenceStartCallback> testSequenceStartCallback,
             AZStd::optional<TestSequenceCompleteCallback> testSequenceCompleteCallback,
-            AZStd::optional<TestCompleteCallback> testRunCompleteCallback);
+            AZStd::optional<TestRunCompleteCallback> testRunCompleteCallback);
 
         //! Returns true if the runtime has test impact analysis data (either preexisting or generated).
         bool HasImpactAnalysisData() const;
 
     private:
+        //! Updates the test enumeration cache for test targets that had sources modified by a given change list.
+        //! @param changeDependencyList The resolved change dependency list generated for the change list.
+        void EnumerateMutatedTestTargets(const ChangeDependencyList& changeDependencyList);
+
+        //! Selects the test targets covering a given change list and updates the enumeration cache of the test targets with sources
+        //! modified in that change list.
+        //! @param changeList The change list for which the covering tests and enumeration cache updates will be generated for.
+        //! @param testPrioritizationPolicy The test prioritization strategy to use for the selected test targets.
+        //! @returns The pair of selected test targets and discarded test targets.
+        AZStd::pair<AZStd::vector<const TestTarget*>, AZStd::vector<const TestTarget*>> SelectCoveringTestTargetsAndUpdateEnumerationCache(
+            const ChangeList& changeList,
+            Policy::TestPrioritization testPrioritizationPolicy);
+
+        //! Selects the test targets from the specified list of test targets that are not on the test target exclusion list.
+        //! @param testTargets The list of test targets to select from.
+        //! @returns The subset of test targets in the specified list that are not on the target exclude list.
+        AZStd::pair<AZStd::vector<const TestTarget*>, AZStd::vector<const TestTarget*>> SelectTestTargetsByExcludeList(
+            AZStd::vector<const TestTarget*> testTargets) const;
+
+        //! Prepares the dynamic dependency map for a seed update by clearing all existing data and deleting the file that will be serialized.
+        void ClearDynamicDependencyMapAndRemoveExistingFile();
+
+        //! Updates the dynamic dependency map and serializes the entire map to disk.
+        void UpdateAndSerializeDynamicDependencyMap(const SourceCoveringTestsList& sourceCoverageTestsList);
+
         RuntimeConfig m_config;
         Policy::ExecutionFailure m_executionFailurePolicy;
         Policy::ExecutionFailureDrafting m_executionFailureDraftingPolicy;
         Policy::TestFailure m_testFailurePolicy;
         Policy::IntegrityFailure m_integrationFailurePolicy;
         Policy::TestSharding m_testShardingPolicy;
-        TargetOutputCapture m_targetOutputCapture;
+        Policy::TargetOutputCapture m_targetOutputCapture;
         size_t m_maxConcurrency = 0;
         AZStd::unique_ptr<DynamicDependencyMap> m_dynamicDependencyMap;
+        AZStd::unique_ptr<TestSelectorAndPrioritizer> m_testSelectorAndPrioritizer;
         AZStd::unique_ptr<TestEngine> m_testEngine;
         AZStd::unordered_set<const TestTarget*> m_testTargetExcludeList;
         AZStd::unordered_set<const TestTarget*> m_testTargetShardList;

Code/Tools/TestImpactFramework/Runtime/Code/Include/TestImpactFramework/TestImpactTestSequence.h

@@ -61,16 +61,16 @@ namespace TestImpact
             Never, //!< Do not shard any test targets.
             Always //!< Shard all test targets that have been marked for test sharding.
         };
-    }
 
-    //! Standard output capture of test target runs. 
+        //! Standard output capture of test target runs. 
-    enum class TargetOutputCapture
+        enum class TargetOutputCapture
-    {
+        {
-        None, //!< Do not capture any output.
+            None, //!< Do not capture any output.
-        StdOut, //!< Send captured output to standard output
+            StdOut, //!< Send captured output to standard output
-        File, //!< Write captured output to file.
+            File, //!< Write captured output to file.
-        StdOutAndFile //!< Send captured output to standard output and write to file.
+            StdOutAndFile //!< Send captured output to standard output and write to file.
-    };
+        };
+    }   
 
     //! Configuration for test targets that opt in to test sharding.
     enum class ShardConfiguration

Code/Tools/TestImpactFramework/Runtime/Code/Source/Dependency/TestImpactDynamicDependencyMap.cpp

@@ -34,7 +34,7 @@ namespace TestImpact
                 else
                 {
                     // This is a new entry on the dependency map so create an entry with this parent target and no covering targets
-                    m_sourceDependencyMap.emplace(source, DependencyData{ {target}, {} });
+                    m_sourceDependencyMap.emplace(source.String(), DependencyData{ {target}, {} });
                 }
             }
 
@@ -56,6 +56,7 @@ namespace TestImpact
         for (const auto& target : m_testTargets.GetTargets())
         {
             mapBuildTargetSources(&target);
+            m_testTargetSourceCoverageCount[&target] = 0;
         }
     }
 
@@ -142,10 +143,21 @@ namespace TestImpact
                 DependencyException, AZStd::string::format("Couldn't replace source coverage for %s, source file is an autogen input file",
                     sourceCoverage.GetPath().c_str()).c_str());
 
-            auto [it, inserted] = m_sourceDependencyMap.insert(sourceCoverage.GetPath().String());
+            auto [sourceDependencyIt, inserted] = m_sourceDependencyMap.insert(sourceCoverage.GetPath().String());
-            auto& [key, sourceDependency] = *it;
+            auto& [key, sourceDependency] = *sourceDependencyIt;
 
-            // Clear any existing coverage for the delta
+            // Knock down the source coverage count for the test targets and clear any existing coverage for the delta
+            for (const auto& testTarget : sourceDependency.m_coveringTestTargets)
+            {
+                if (auto coveringTestTargetIt = m_testTargetSourceCoverageCount.find(testTarget);
+                    coveringTestTargetIt != m_testTargetSourceCoverageCount.end())
+                {
+                    if (coveringTestTargetIt->second > 0)
+                    {
+                        coveringTestTargetIt->second--;
+                    }
+                }
+            }
             sourceDependency.m_coveringTestTargets.clear();
 
             // Update the dependency with any new coverage data
@@ -157,6 +169,9 @@ namespace TestImpact
                     // Source to covering test target mapping
                     sourceDependency.m_coveringTestTargets.insert(testTarget);
 
+                    // Test target covering sources count
+                    m_testTargetSourceCoverageCount[testTarget]++;
+
                     // Build target to covering test target mapping
                     for (const auto& parentTarget : sourceDependency.m_parentTargets)
                     {
@@ -173,7 +188,7 @@ namespace TestImpact
             // If the new coverage data results in a parentless and coverageless entry, consider it a dead entry and remove accordingly
             if (sourceDependency.m_coveringTestTargets.empty() && sourceDependency.m_parentTargets.empty())
             {
-                m_sourceDependencyMap.erase(it);
+                m_sourceDependencyMap.erase(sourceDependencyIt);
             }
         }
     }
@@ -198,6 +213,14 @@ namespace TestImpact
         }
     }
 
+    void DynamicDependencyMap::ClearAllSourceCoverage()
+    {
+        for (const auto& [path, coverage] : m_sourceDependencyMap)
+        {
+            ReplaceSourceCoverage(SourceCoveringTestsList(AZStd::vector<SourceCoveringTests>{ SourceCoveringTests(RepoPath(path)) }));
+        }
+    }
+
     const ProductionTargetList& DynamicDependencyMap::GetProductionTargetList() const
     {
         return m_productionTargets;
@@ -405,4 +428,33 @@ namespace TestImpact
 
         return ChangeDependencyList(AZStd::move(createDependencies), AZStd::move(updateDependencies), AZStd::move(deleteDependencies));
     }
+
+    AZStd::vector<const TestTarget*> DynamicDependencyMap::GetCoveringTests() const
+    {
+        AZStd::vector<const TestTarget*> covering;
+        for (const auto& [testTarget, coveringSources] : m_testTargetSourceCoverageCount)
+        {
+            if (coveringSources > 0)
+            {
+                covering.push_back(testTarget);
+            }
+        }
+
+        return covering;
+    }
+
+    AZStd::vector<const TestTarget*> DynamicDependencyMap::GetNotCoveringTests() const
+    {
+        AZStd::vector<const TestTarget*> notCovering;
+        for(const auto& [testTarget, coveringSources] : m_testTargetSourceCoverageCount)
+        {
+            if(coveringSources == 0)
+            {
+                notCovering.push_back(testTarget);
+            }
+        }
+
+        return notCovering;
+    }
+
 } // namespace TestImpact

Code/Tools/TestImpactFramework/Runtime/Code/Source/Dependency/TestImpactDynamicDependencyMap.h

@@ -85,18 +85,27 @@ namespace TestImpact
         //! @param sourceCoverageDelta The source coverage delta to replace in the dependency map.
         void ReplaceSourceCoverage(const SourceCoveringTestsList& sourceCoverageDelta);
 
+        //! Clears all of the existing source coverage in the dependency map.
+        void ClearAllSourceCoverage();
+
         //! Exports the coverage of all sources in the dependency map.
         SourceCoveringTestsList ExportSourceCoverage() const;
 
         //! Gets the list of orphaned source files in the dependency map that have coverage data but belong to no parent build targets.
         AZStd::vector<AZStd::string> GetOrphanSourceFiles() const;
 
-        //! Applies the specified change list to the dynamic dependency map and resolves the change list to a change dependency list
+        //! Applies the specified change list to the dependency map and resolves the change list to a change dependency list
         //! containing the updated source dependencies for each source file in the change list.
         //! @param changeList The change list to apply and resolve.
         //! @returns The change list as resolved to the appropriate source dependencies.
         [[nodiscard]] ChangeDependencyList ApplyAndResoveChangeList(const ChangeList& changeList);
 
+        //! Returns the test targets that cover one or more sources in the repository.
+        AZStd::vector<const TestTarget*> GetCoveringTests() const;
+
+        //! Returns the test targets that do not cover any sources in the repository.
+        AZStd::vector<const TestTarget*> GetNotCoveringTests() const;
+
     private:
         //! Clears the source coverage of the specified sources.
         //! @note The covering targets for the parent test target(s) will not be pruned if those covering targets are removed.
@@ -116,5 +125,8 @@ namespace TestImpact
 
         //! Mapping of autogen input sources to their generated output sources.
         AZStd::unordered_map<AZStd::string, AZStd::vector<AZStd::string>> m_autogenInputToOutputMap;
+
+        //! Number of sources that each test target in the repository covers.
+        AZStd::unordered_map<const TestTarget*, size_t> m_testTargetSourceCoverageCount;
     };
 } // namespace TestImpact

Code/Tools/TestImpactFramework/Runtime/Code/Source/Dependency/TestImpactSourceCoveringTestsList.cpp

@@ -65,7 +65,7 @@ namespace TestImpact
     {
         AZStd::sort(m_coverage.begin(), m_coverage.end(), [](const SourceCoveringTests& lhs, const SourceCoveringTests& rhs)
         {
-            return lhs.GetPath() < rhs.GetPath();
+            return lhs.GetPath().String() < rhs.GetPath().String();
         });
     }
 

Code/Tools/TestImpactFramework/Runtime/Code/Source/Dependency/TestImpactSourceCoveringTestsSerializer.cpp

@@ -0,0 +1,93 @@
+/*
+ * All or portions of this file Copyright (c) Amazon.com, Inc. or its affiliates or
+ * its licensors.
+ *
+ * For complete copyright and license terms please see the LICENSE at the root of this
+ * distribution (the "License"). All use of this software is governed by the License,
+ * or, if provided, by the license below or the license accompanying this file. Do not
+ * remove or modify any license notices. This file is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ *
+ */
+
+#include <Dependency/TestImpactDependencyException.h>
+#include <Dependency/TestImpactSourceCoveringTestsSerializer.h>
+
+#include <AzCore/JSON/document.h>
+#include <AzCore/JSON/prettywriter.h>
+#include <AzCore/JSON/rapidjson.h>
+#include <AzCore/JSON/stringbuffer.h>
+
+namespace TestImpact
+{
+    // Tag used to indicate whether a given line is the name or a covering test target
+    constexpr char TargetTag = '-';
+
+    AZStd::string SerializeSourceCoveringTestsList(const SourceCoveringTestsList& sourceCoveringTestsList)
+    {
+        AZStd::string output;
+        output.reserve(1U << 24); // Reserve approx. 16Mib as the outputs can be quite large
+
+        for (const auto& source : sourceCoveringTestsList.GetCoverage())
+        {
+            // Source file
+            output += source.GetPath().String();
+            output += "\n";
+
+            // Covering test targets
+            for (const auto& testTarget : source.GetCoveringTestTargets())
+            {
+                output += AZStd::string::format("%c%s\n", TargetTag, testTarget.c_str());
+            }
+        }
+
+        // Add the newline so the deserializer can properly terminate on the last read line
+        output += "\n";
+
+        return output;
+    }
+
+    SourceCoveringTestsList DeserializeSourceCoveringTestsList(const AZStd::string& sourceCoveringTestsListString)
+    {
+        AZStd::vector<SourceCoveringTests> sourceCoveringTests;
+        AZStd::string source;
+        AZStd::vector<AZStd::string> coveringTests;
+        sourceCoveringTests.reserve(1U << 16); // Reserve for approx. 65k source files
+        const AZStd::string delim = "\n";
+        auto start = 0U;
+        auto end = sourceCoveringTestsListString.find(delim);
+
+        while (end != AZStd::string::npos)
+        {
+            const auto line = sourceCoveringTestsListString.substr(start, end - start);
+            if (line.starts_with(TargetTag))
+            {
+                // This is a test target covering the most recent source discovered
+                coveringTests.push_back(line.substr(1, line.length() - 1));
+            }
+            else
+            {
+                // This is a new source file so assign the accumulated test targets to the current source file before proceeding
+                if (!coveringTests.empty())
+                {
+                    sourceCoveringTests.push_back(SourceCoveringTests(source, AZStd::move(coveringTests)));
+                    coveringTests.clear();
+                }
+
+                source = line;
+            }
+
+            start = end + delim.length();
+            end = sourceCoveringTestsListString.find(delim, start);
+        }
+
+        // Ensure we properly assign the accumulated test targets to the most recent source discovered
+        if (!coveringTests.empty())
+        {
+            sourceCoveringTests.push_back(SourceCoveringTests(source, AZStd::move(coveringTests)));
+            coveringTests.clear();
+        }
+
+        return SourceCoveringTestsList(AZStd::move(sourceCoveringTests));
+    }
+} // namespace TestImpact

Code/Tools/TestImpactFramework/Runtime/Code/Source/Dependency/TestImpactSourceCoveringTestsSerializer.h

@@ -0,0 +1,26 @@
+/*
+ * All or portions of this file Copyright (c) Amazon.com, Inc. or its affiliates or
+ * its licensors.
+ *
+ * For complete copyright and license terms please see the LICENSE at the root of this
+ * distribution (the "License"). All use of this software is governed by the License,
+ * or, if provided, by the license below or the license accompanying this file. Do not
+ * remove or modify any license notices. This file is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ *
+ */
+
+#pragma once
+
+#include <Dependency/TestImpactSourceCoveringTestsList.h>
+
+#include <AzCore/std/string/string.h>
+
+namespace TestImpact
+{
+    //! Serializes the specified source covering tests list to plain text format.
+    AZStd::string SerializeSourceCoveringTestsList(const SourceCoveringTestsList& sourceCoveringTestsList);
+
+    //! Deserializes a source covering tests list from the specified source covering tests data in plain text format.
+    SourceCoveringTestsList DeserializeSourceCoveringTestsList(const AZStd::string& sourceCoveringTestsListString);
+} // namespace TestImpact

Code/Tools/TestImpactFramework/Runtime/Code/Source/Dependency/TestImpactTestSelectorAndPrioritizer.cpp

@@ -25,7 +25,7 @@ namespace TestImpact
     }
 
     AZStd::vector<const TestTarget*> TestSelectorAndPrioritizer::SelectTestTargets(
-        const ChangeDependencyList& changeDependencyList, TestSelectionStrategy testSelectionStrategy)
+        const ChangeDependencyList& changeDependencyList, Policy::TestPrioritization testSelectionStrategy)
     {
         const auto selectedTestTargetAndDependerMap = SelectTestTargets(changeDependencyList);
         const auto prioritizedSelectedTests = PrioritizeSelectedTestTargets(selectedTestTargetAndDependerMap, testSelectionStrategy);
@@ -210,7 +210,7 @@ namespace TestImpact
 
     AZStd::vector<const TestTarget*> TestSelectorAndPrioritizer::PrioritizeSelectedTestTargets(
         const SelectedTestTargetAndDependerMap& selectedTestTargetAndDependerMap,
-        [[maybe_unused]]TestSelectionStrategy testSelectionStrategy)
+        [[maybe_unused]] Policy::TestPrioritization testSelectionStrategy)
     {
         AZStd::vector<const TestTarget*> selectedTestTargets;
 

Code/Tools/TestImpactFramework/Runtime/Code/Source/Dependency/TestImpactTestSelectorAndPrioritizer.h

@@ -12,6 +12,8 @@
 
 #pragma once
 
+#include <TestImpactFramework/TestImpactTestSequence.h>
+
 #include <Artifact/Static/TestImpactDependencyGraphData.h>
 #include <Dependency/TestImpactChangeDependencyList.h>
 
@@ -25,13 +27,6 @@ namespace TestImpact
     class BuildTarget;
     class TestTarget;
 
-    //! Strategy for selecting tests given a set of source changes.
-    enum class TestSelectionStrategy : bool
-    {
-        SelectOnly, //!< Select tests only, do not attempt prioritization of those selected tests.
-        SelectAndPriotitize //!< Select tests and prioritize according to dependency graph locality of coverer and coveree.
-    };
-
     //! Map of build targets and their dependency graph data.
     //! For test targets, the dependency graph data is that of the build targets which the test target depends on.
     //! For production targets, the dependency graph is that of the build targets that depend on it (dependers).
@@ -52,7 +47,7 @@ namespace TestImpact
         //! Select the covering test targets for the given set of source changes and optionally prioritizes said test selection.
         //! @param changeDependencyList The resolved list of source dependencies for the CRUD source changes.
         //! @param testSelectionStrategy The test selection and prioritization strategy to apply to the given CRUD source changes.
-        AZStd::vector<const TestTarget*> SelectTestTargets(const ChangeDependencyList& changeDependencyList, TestSelectionStrategy testSelectionStrategy);
+        AZStd::vector<const TestTarget*> SelectTestTargets(const ChangeDependencyList& changeDependencyList, Policy::TestPrioritization testSelectionStrategy);
 
     private:
         //! Map of selected test targets and the production targets they cover for the given set of source changes.
@@ -69,7 +64,7 @@ namespace TestImpact
         //! @param testSelectionStrategy The test selection strategy to prioritize the selected tests.
         //! @returns The selected tests either in either arbitrary order or in prioritized with highest priority first.
         AZStd::vector<const TestTarget*> PrioritizeSelectedTestTargets(
-            const SelectedTestTargetAndDependerMap& selectedTestTargetAndDependerMap, TestSelectionStrategy testSelectionStrategy);
+            const SelectedTestTargetAndDependerMap& selectedTestTargetAndDependerMap, Policy::TestPrioritization testSelectionStrategy);
 
         const DynamicDependencyMap* m_dynamicDependencyMap;
         DependencyGraphDataMap m_dependencyGraphDataMap;

Code/Tools/TestImpactFramework/Runtime/Code/Source/Platform/Windows/TestEngine/TestImpactWin32_TestEngineJobFailure.cpp

@@ -0,0 +1,35 @@
+/*
+ * All or portions of this file Copyright (c) Amazon.com, Inc. or its affiliates or
+ * its licensors.
+ *
+ * For complete copyright and license terms please see the LICENSE at the root of this
+ * distribution (the "License"). All use of this software is governed by the License,
+ * or, if provided, by the license below or the license accompanying this file. Do not
+ * remove or modify any license notices. This file is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ *
+ */
+
+#include <TestEngine/TestImpactTestEngineJobFailure.h>
+
+namespace TestImpact
+{
+    // Known error codes for test instrumentation
+    namespace ErrorCodes
+    {
+        namespace OpenCppCoverage
+        {
+            static constexpr ReturnCode InvalidArgs = 0x9F8C8E5C;
+        }
+    }
+
+    AZStd::optional<Client::TestRunResult> CheckForKnownTestInstrumentErrorCode(ReturnCode returnCode)
+    {
+        if (returnCode == ErrorCodes::OpenCppCoverage::InvalidArgs)
+        {
+            return Client::TestRunResult::FailedToExecute;
+        }
+
+        return AZStd::nullopt;
+    }
+} // namespace TestImpact

Code/Tools/TestImpactFramework/Runtime/Code/Source/TestEngine/TestImpactTestEngineJobFailure.cpp

@@ -0,0 +1,82 @@
+/*
+ * All or portions of this file Copyright (c) Amazon.com, Inc. or its affiliates or
+ * its licensors.
+ *
+ * For complete copyright and license terms please see the LICENSE at the root of this
+ * distribution (the "License"). All use of this software is governed by the License,
+ * or, if provided, by the license below or the license accompanying this file. Do not
+ * remove or modify any license notices. This file is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ *
+ */
+
+#include <TestEngine/TestImpactTestEngineJobFailure.h>
+
+namespace TestImpact
+{
+    // Known error codes for test runner and test library
+    namespace ErrorCodes
+    {
+        namespace GTest
+        {
+            static constexpr ReturnCode Unsuccessful = 1;
+        }
+
+        namespace AZTestRunner
+        {
+            static constexpr ReturnCode InvalidArgs = 101;
+            static constexpr ReturnCode FailedToFindTargetBinary = 102;
+            static constexpr ReturnCode SymbolNotFound = 103;
+            static constexpr ReturnCode ModuleSkipped = 104;
+        }
+    }
+
+    AZStd::optional<Client::TestRunResult> CheckForKnownTestRunnerErrorCode(int returnCode)
+    {
+        switch (returnCode)
+        {
+        // We will consider test targets that technically execute but their launcher or unit test library return a know error
+        // code that pertains to incorrect argument usage as test targets that failed to execute
+        case ErrorCodes::AZTestRunner::InvalidArgs:
+        case ErrorCodes::AZTestRunner::FailedToFindTargetBinary:
+        case ErrorCodes::AZTestRunner::ModuleSkipped:
+        case ErrorCodes::AZTestRunner::SymbolNotFound:
+            return Client::TestRunResult::FailedToExecute;
+        default:
+            return AZStd::nullopt;
+        }
+    }
+
+    AZStd::optional<Client::TestRunResult> CheckForKnownTestLibraryErrorCode(int returnCode)
+    {
+        if (returnCode == ErrorCodes::GTest::Unsuccessful)
+        {
+            return Client::TestRunResult::TestFailures;
+        }
+
+        return AZStd::nullopt;
+    }
+
+    AZStd::optional<Client::TestRunResult> CheckForAnyKnownErrorCode(ReturnCode returnCode)
+    {
+        if (const auto result = CheckForKnownTestInstrumentErrorCode(returnCode);
+            result != AZStd::nullopt)
+        {
+            return result.value();
+        }
+
+        if (const auto result = CheckForKnownTestRunnerErrorCode(returnCode);
+            result != AZStd::nullopt)
+        {
+            return result.value();
+        }
+
+        if (const auto result = CheckForKnownTestLibraryErrorCode(returnCode);
+            result != AZStd::nullopt)
+        {
+            return result.value();
+        }
+
+        return AZStd::nullopt;
+    }
+} // namespace TestImpact

Code/Tools/TestImpactFramework/Runtime/Code/Source/TestEngine/TestImpactTestEngineJobFailure.h

@@ -0,0 +1,34 @@
+/*
+ * All or portions of this file Copyright (c) Amazon.com, Inc. or its affiliates or
+ * its licensors.
+ *
+ * For complete copyright and license terms please see the LICENSE at the root of this
+ * distribution (the "License"). All use of this software is governed by the License,
+ * or, if provided, by the license below or the license accompanying this file. Do not
+ * remove or modify any license notices. This file is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ *
+ */
+
+#pragma once
+
+#include <TestImpactFramework/TestImpactClientTestRun.h>
+
+#include <Process/TestImpactProcessInfo.h>
+
+#include <AzCore/std/optional.h>
+
+namespace TestImpact
+{
+    //! Checks for known test instrumentation error return codes and returns the corresponding client test run result or empty.
+    AZStd::optional<Client::TestRunResult> CheckForKnownTestInstrumentErrorCode(ReturnCode returnCode);
+
+    //! Checks for known test runner error return codes and returns the corresponding client test run result or empty.
+    AZStd::optional<Client::TestRunResult> CheckForKnownTestRunnerErrorCode(ReturnCode returnCode);
+
+    //! Checks for known test library error return codes and returns the corresponding client test run result or empty.
+    AZStd::optional<Client::TestRunResult> CheckForKnownTestLibraryErrorCode(ReturnCode returnCode);
+
+    //! Checks for all known error return codes and returns the corresponding client test run result or empty.
+    AZStd::optional<Client::TestRunResult> CheckForAnyKnownErrorCode(ReturnCode returnCode);
+} // namespace TestImpact