Add previously approved changes.

Signed-off-by: John <jonawals@amazon.com>

Code/Tools/TestImpactFramework/Frontend/Console/Code/Source/TestImpactConsoleMain.cpp

@@ -77,7 +77,6 @@ namespace TestImpact
 
         //! Wrapper around impact analysis sequences to handle the case where the safe mode option is active.
         ReturnCode WrappedImpactAnalysisTestSequence(
-            TestSequenceEventHandler& sequenceEventHandler,
             const CommandLineOptions& options,
             Runtime& runtime,
             const AZStd::optional<ChangeList>& changeList)
@@ -94,44 +93,30 @@ namespace TestImpact
             {
                 if (options.GetTestSequenceType() == TestSequenceType::ImpactAnalysis)
                 {
-                    auto [selectedResult, discardedResult] = runtime.SafeImpactAnalysisTestSequence(
+                    auto safeImpactAnalysisSequenceReport = runtime.SafeImpactAnalysisTestSequence(
                         changeList.value(),
                         options.GetTestPrioritizationPolicy(),
                         options.GetTestTargetTimeout(),
                         options.GetGlobalTimeout(),
-                        AZStd::ref(sequenceEventHandler),
-                        AZStd::ref(sequenceEventHandler),
-                        AZStd::ref(sequenceEventHandler));
-
-                    // Handling the possible timeout and failure permutations of the selected and discarded test results is splitting hairs
-                    // so apply the following, admittedly arbitrary, rules to determine what the composite test sequence result should be
-                    if (selectedResult == TestSequenceResult::Success && discardedResult == TestSequenceResult::Success)
-                    {
-                        // Trivial case: both sequences succeeded
-                        result = TestSequenceResult::Success;
-                    }
-                    else if (selectedResult == TestSequenceResult::Failure || discardedResult == TestSequenceResult::Failure)
-                    {
-                        // One sequence failed whilst the other sequence either succeeded or timed out
-                        result = TestSequenceResult::Failure;
-                    }
-                    else
-                    {
-                        // One or both sequences timed out or failed
-                        result = TestSequenceResult::Timeout;
-                    }
+                        SafeImpactAnalysisTestSequenceStartCallback,
+                        SafeImpactAnalysisTestSequenceCompleteCallback,
+                        TestRunCompleteCallback);
+        
+                    result = safeImpactAnalysisSequenceReport.GetResult();
                 }
                 else if (options.GetTestSequenceType() == TestSequenceType::ImpactAnalysisNoWrite)
                 {
                     // A no-write impact analysis sequence with safe mode enabled is functionally identical to a regular sequence type
                     // due to a) the selected tests being run without instrumentation and b) the discarded tests also being run without
                     // instrumentation
-                    result = runtime.RegularTestSequence(
+                    auto sequenceReport = runtime.RegularTestSequence(
                         options.GetTestTargetTimeout(),
                         options.GetGlobalTimeout(),
-                        AZStd::ref(sequenceEventHandler),
-                        AZStd::ref(sequenceEventHandler),
-                        AZStd::ref(sequenceEventHandler));
+                        TestSequenceStartCallback,
+                        TestSequenceCompleteCallback,
+                        TestRunCompleteCallback);
+
+                    result = sequenceReport.GetResult();
                 }
                 else
                 {
@@ -153,18 +138,20 @@ namespace TestImpact
                 {
                     throw(Exception("Unexpected sequence type"));
                 }
-
-                result = runtime.ImpactAnalysisTestSequence(
+        
+                auto impactAnalysisSequenceReport = runtime.ImpactAnalysisTestSequence(
                     changeList.value(),
                     options.GetTestPrioritizationPolicy(),
                     dynamicDependencyMapPolicy,
                     options.GetTestTargetTimeout(),
                     options.GetGlobalTimeout(),
-                    AZStd::ref(sequenceEventHandler),
-                    AZStd::ref(sequenceEventHandler),
-                    AZStd::ref(sequenceEventHandler));
-            }
+                    ImpactAnalysisTestSequenceStartCallback,
+                    ImpactAnalysisTestSequenceCompleteCallback,
+                    TestRunCompleteCallback);
 
+                result = impactAnalysisSequenceReport.GetResult();
+            }
+        
             return GetReturnCodeForTestSequenceResult(result);
         };
 
@@ -217,53 +204,51 @@ namespace TestImpact
                     std::cout << "Test impact analysis data for this repository was not found, seed or regular sequence fallbacks will be used.\n";
                 }
 
-                TestSequenceEventHandler sequenceEventHandler(options.GetSuiteFilter());
-
                 switch (const auto type = options.GetTestSequenceType())
                 {
                 case TestSequenceType::Regular:
                 {
-                    const auto result = runtime.RegularTestSequence(
+                    const auto sequenceReport = runtime.RegularTestSequence(
                         options.GetTestTargetTimeout(),
                         options.GetGlobalTimeout(),
-                        AZStd::ref(sequenceEventHandler),
-                        AZStd::ref(sequenceEventHandler),
-                        AZStd::ref(sequenceEventHandler));
+                        TestSequenceStartCallback,
+                        TestSequenceCompleteCallback,
+                        TestRunCompleteCallback);
 
-                    return GetReturnCodeForTestSequenceResult(result);
+                    return GetReturnCodeForTestSequenceResult(sequenceReport.GetResult());
                 }
                 case TestSequenceType::Seed:
                 {
-                    const auto result = runtime.SeededTestSequence(
+                    const auto sequenceReport = runtime.SeededTestSequence(
                         options.GetTestTargetTimeout(),
                         options.GetGlobalTimeout(),
-                        AZStd::ref(sequenceEventHandler),
-                        AZStd::ref(sequenceEventHandler),
-                        AZStd::ref(sequenceEventHandler));
+                        TestSequenceStartCallback,
+                        TestSequenceCompleteCallback,
+                        TestRunCompleteCallback);
 
-                    return GetReturnCodeForTestSequenceResult(result);
+                    return GetReturnCodeForTestSequenceResult(sequenceReport.GetResult());
                 }
                 case TestSequenceType::ImpactAnalysisNoWrite:
                 case TestSequenceType::ImpactAnalysis:
                 {
-                    return WrappedImpactAnalysisTestSequence(sequenceEventHandler, options, runtime, changeList);
+                    return WrappedImpactAnalysisTestSequence(options, runtime, changeList);
                 }
                 case TestSequenceType::ImpactAnalysisOrSeed:
                 {
                     if (runtime.HasImpactAnalysisData())
                     {
-                        return WrappedImpactAnalysisTestSequence(sequenceEventHandler, options, runtime, changeList);
+                        return WrappedImpactAnalysisTestSequence(options, runtime, changeList);
                     }
                     else
                     {
-                        const auto result = runtime.SeededTestSequence(
+                        const auto sequenceReport = runtime.SeededTestSequence(
                             options.GetTestTargetTimeout(),
                             options.GetGlobalTimeout(),
-                            AZStd::ref(sequenceEventHandler),
-                            AZStd::ref(sequenceEventHandler),
-                            AZStd::ref(sequenceEventHandler));
+                            TestSequenceStartCallback,
+                            TestSequenceCompleteCallback,
+                            TestRunCompleteCallback);
 
-                        return GetReturnCodeForTestSequenceResult(result);
+                        return GetReturnCodeForTestSequenceResult(sequenceReport.GetResult());
                     }
                 }
                 default:

Code/Tools/TestImpactFramework/Frontend/Console/Code/Source/TestImpactConsoleTestSequenceEventHandler.cpp

@@ -32,72 +32,73 @@ namespace TestImpact
                 std::cout << "Of which " << numExcludedTests << " tests have been excluded and " << numDraftedTests << " tests have been drafted.\n";
             }
 
-            void FailureReport(const Client::SequenceFailure& failureReport, AZStd::chrono::milliseconds duration)
+            void FailureReport(const Client::TestRunReport& testRunReport)
             {
-                std::cout << "Sequence completed in " << (duration.count() / 1000.f) << "s with";
+                std::cout << "Sequence completed in " << (testRunReport.GetDuration().count() / 1000.f) << "s with";
 
-                if (!failureReport.GetExecutionFailures().empty() ||
-                    !failureReport.GetTestRunFailures().empty() ||
-                    !failureReport.GetTimedOutTests().empty() ||
-                    !failureReport.GetUnexecutedTests().empty())
+                if (!testRunReport.GetExecutionFailureTests().empty() ||
+                    !testRunReport.GetFailingTests().empty() ||
+                    !testRunReport.GetTimedOutTests().empty() ||
+                    !testRunReport.GetUnexecutedTests().empty())
                 {
                     std::cout << ":\n";
                     std::cout << SetColor(Foreground::White, Background::Red).c_str()
-                        << failureReport.GetTestRunFailures().size()
+                        << testRunReport.GetFailingTests().size()
                         << ResetColor().c_str() << " test failures\n";
 
                     std::cout << SetColor(Foreground::White, Background::Red).c_str()
-                        << failureReport.GetExecutionFailures().size()
+                        << testRunReport.GetExecutionFailureTests().size()
                         << ResetColor().c_str() << " execution failures\n";
 
                     std::cout << SetColor(Foreground::White, Background::Red).c_str()
-                        << failureReport.GetTimedOutTests().size()
+                        << testRunReport.GetTimedOutTests().size()
                         << ResetColor().c_str() << " test timeouts\n";
 
                     std::cout << SetColor(Foreground::White, Background::Red).c_str()
-                        << failureReport.GetUnexecutedTests().size()
+                        << testRunReport.GetUnexecutedTests().size()
                         << ResetColor().c_str() << " unexecuted tests\n";
 
-                    if (!failureReport.GetTestRunFailures().empty())
+                    if (!testRunReport.GetFailingTests().empty())
                     {
                         std::cout << "\nTest failures:\n";
-                        for (const auto& testRunFailure : failureReport.GetTestRunFailures())
+                        for (const auto& testRunFailure : testRunReport.GetFailingTests())
                         {
-                            std::cout << "  " << testRunFailure.GetTargetName().c_str();
                             for (const auto& testCaseFailure : testRunFailure.GetTestCaseFailures())
                             {
-                                std::cout << "." << testCaseFailure.GetName().c_str();
                                 for (const auto& testFailure : testCaseFailure.GetTestFailures())
                                 {
-                                    std::cout << "." << testFailure.GetName().c_str() << "\n";
+                                    std::cout << "  "
+                                        << testRunFailure.GetTargetName().c_str()
+                                        << "." << testCaseFailure.GetName().c_str()
+                                        << "." << testFailure.GetName().c_str() << "\n";
                                 }
                             }
                         }
                     }
 
-                    if (!failureReport.GetExecutionFailures().empty())
+                    if (!testRunReport.GetExecutionFailureTests().empty())
                     {
                         std::cout << "\nExecution failures:\n";
-                        for (const auto& executionFailure : failureReport.GetExecutionFailures())
+                        for (const auto& executionFailure : testRunReport.GetExecutionFailureTests())
                         {
                             std::cout << "  " << executionFailure.GetTargetName().c_str() << "\n";
                             std::cout << executionFailure.GetCommandString().c_str() << "\n";
                         }
                     }
 
-                    if (!failureReport.GetTimedOutTests().empty())
+                    if (!testRunReport.GetTimedOutTests().empty())
                     {
                         std::cout << "\nTimed out tests:\n";
-                        for (const auto& testTimeout : failureReport.GetTimedOutTests())
+                        for (const auto& testTimeout : testRunReport.GetTimedOutTests())
                         {
                             std::cout << "  " << testTimeout.GetTargetName().c_str() << "\n";
                         }
                     }
 
-                    if (!failureReport.GetUnexecutedTests().empty())
+                    if (!testRunReport.GetUnexecutedTests().empty())
                     {
                         std::cout << "\nUnexecuted tests:\n";
-                        for (const auto& unexecutedTest : failureReport.GetUnexecutedTests())
+                        for (const auto& unexecutedTest : testRunReport.GetUnexecutedTests())
                         {
                             std::cout << "  " << unexecutedTest.GetTargetName().c_str() << "\n";
                         }
@@ -105,50 +106,42 @@ namespace TestImpact
                 }
                 else
                 {
-                    std::cout << SetColor(Foreground::White, Background::Green).c_str() << " \100% passes!\n" << ResetColor().c_str();
+                    std::cout << SetColor(Foreground::White, Background::Green).c_str() << " \100% passes!\n" << ResetColor().c_str() << "\n";
                 }
             }
         }
 
-        TestSequenceEventHandler::TestSequenceEventHandler(SuiteType suiteFilter)
-            : m_suiteFilter(suiteFilter)
+        void TestSequenceStartCallback(SuiteType suiteType, const Client::TestRunSelection& selectedTests)
         {
+            Output::TestSuiteFilter(suiteType);
+            std::cout << selectedTests.GetNumIncludedTestRuns() << " tests selected, " << selectedTests.GetNumExcludedTestRuns()
+                      << " excluded.\n";
         }
 
-        // TestSequenceStartCallback
-        void TestSequenceEventHandler::operator()(Client::TestRunSelection&& selectedTests)
+        void TestSequenceCompleteCallback(SuiteType suiteType, const Client::TestRunSelection& selectedTests)
         {
-            ClearState();
-            m_numTests = selectedTests.GetNumIncludedTestRuns();
-
-            Output::TestSuiteFilter(m_suiteFilter);
+            Output::TestSuiteFilter(suiteType);
             std::cout << selectedTests.GetNumIncludedTestRuns() << " tests selected, " << selectedTests.GetNumExcludedTestRuns() << " excluded.\n";
         }
 
-        // ImpactAnalysisTestSequenceStartCallback
-        void TestSequenceEventHandler::operator()(
-            Client::TestRunSelection&& selectedTests,
-            AZStd::vector<AZStd::string>&& discardedTests,
-            AZStd::vector<AZStd::string>&& draftedTests)
+        void ImpactAnalysisTestSequenceStartCallback(
+            SuiteType suiteType,
+            const Client::TestRunSelection& selectedTests,
+            const AZStd::vector<AZStd::string>& discardedTests,
+            const AZStd::vector<AZStd::string>& draftedTests)
         {
-            ClearState();
-            m_numTests = selectedTests.GetNumIncludedTestRuns() + draftedTests.size();
-
-            Output::TestSuiteFilter(m_suiteFilter);
+            Output::TestSuiteFilter(suiteType);
             Output::ImpactAnalysisTestSelection(
                 selectedTests.GetTotalNumTests(), discardedTests.size(), selectedTests.GetNumExcludedTestRuns(), draftedTests.size());
         }
 
-        // SafeImpactAnalysisTestSequenceStartCallback
-        void TestSequenceEventHandler::operator()(
-            Client::TestRunSelection&& selectedTests,
-            Client::TestRunSelection&& discardedTests,
-            AZStd::vector<AZStd::string>&& draftedTests)
+        void SafeImpactAnalysisTestSequenceStartCallback(
+            SuiteType suiteType,
+            const Client::TestRunSelection& selectedTests,
+            const Client::TestRunSelection& discardedTests,
+            const AZStd::vector<AZStd::string>& draftedTests)
         {
-            ClearState();
-            m_numTests = selectedTests.GetNumIncludedTestRuns() + draftedTests.size();
-
-            Output::TestSuiteFilter(m_suiteFilter);
+            Output::TestSuiteFilter(suiteType);
             Output::ImpactAnalysisTestSelection(
                 selectedTests.GetTotalNumTests(),
                 discardedTests.GetTotalNumTests(),
@@ -156,40 +149,45 @@ namespace TestImpact
                 draftedTests.size());
         }
 
-        // TestSequenceCompleteCallback
-        void TestSequenceEventHandler::operator()(
-            Client::SequenceFailure&& failureReport,
-            AZStd::chrono::milliseconds duration)
+        void TestSequenceCompleteCallback(const Client::SequenceReport& sequenceReport)
         {
             
-            Output::FailureReport(failureReport, duration);
+            Output::FailureReport(sequenceReport.GetSelectedTestRunReport());
             std::cout << "Updating and serializing the test impact analysis data, this may take a moment...\n";
         }
 
-        // SafeTestSequenceCompleteCallback
-        void TestSequenceEventHandler::operator()(
-            Client::SequenceFailure&& selectedFailureReport,
-            Client::SequenceFailure&& discardedFailureReport,
-            AZStd::chrono::milliseconds selectedDuration,
-            AZStd::chrono::milliseconds discaredDuration)
+        void ImpactAnalysisTestSequenceCompleteCallback(const Client::ImpactAnalysisSequenceReport& sequenceReport)
         {
             std::cout << "Selected test run:\n";
-            Output::FailureReport(selectedFailureReport, selectedDuration);
+            Output::FailureReport(sequenceReport.GetSelectedTestRunReport());
 
-            std::cout << "Discarded test run:\n";
-            Output::FailureReport(discardedFailureReport, discaredDuration);
+            std::cout << "Drafted test run:\n";
+            Output::FailureReport(sequenceReport.GetDraftedTestRunReport());
 
             std::cout << "Updating and serializing the test impact analysis data, this may take a moment...\n";
         }
 
-        // TestRunCompleteCallback
-        void TestSequenceEventHandler::operator()([[maybe_unused]] Client::TestRun&& test)
+        void SafeImpactAnalysisTestSequenceCompleteCallback(const Client::SafeImpactAnalysisSequenceReport& sequenceReport)
         {
-            m_numTestsComplete++;
-            const auto progress = AZStd::string::format("(%03u/%03u)", m_numTestsComplete, m_numTests, test.GetTargetName().c_str());
+            std::cout << "Selected test run:\n";
+            Output::FailureReport(sequenceReport.GetSelectedTestRunReport());
+
+            std::cout << "Discarded test run:\n";
+            Output::FailureReport(sequenceReport.GetDiscardedTestRunReport());
+
+            std::cout << "Drafted test run:\n";
+            Output::FailureReport(sequenceReport.GetDraftedTestRunReport());
+
+            std::cout << "Updating and serializing the test impact analysis data, this may take a moment...\n";
+        }
+
+        void TestRunCompleteCallback(const Client::TestRun& testRun, size_t numTestRunsCompleted, size_t totalNumTestRuns)
+        {
+            const auto progress =
+                AZStd::string::format("(%03u/%03u)", numTestRunsCompleted, totalNumTestRuns, testRun.GetTargetName().c_str());
 
             AZStd::string result;
-            switch (test.GetResult())
+            switch (testRun.GetResult())
             {
             case Client::TestRunResult::AllTestsPass:
             {
@@ -216,15 +214,14 @@ namespace TestImpact
                 result = SetColorForString(Foreground::White, Background::Magenta, "TIME");
                 break;
             }
+            default:
+            {
+                AZ_Error("TestRunCompleteCallback", false, "Unexpected test result to handle: %u", aznumeric_cast<AZ::u32>(testRun.GetResult()));
+            }
             }
 
-            std::cout << progress.c_str() << " " << result.c_str() << " " << test.GetTargetName().c_str() << " (" << (test.GetDuration().count() / 1000.f) << "s)\n";
-        }
-
-        void TestSequenceEventHandler::ClearState()
-        {
-            m_numTests = 0;
-            m_numTestsComplete = 0;
+            std::cout << progress.c_str() << " " << result.c_str() << " " << testRun.GetTargetName().c_str() << " ("
+                      << (testRun.GetDuration().count() / 1000.f) << "s)\n";
         }
     } // namespace Console
 } // namespace TestImpact

Code/Tools/TestImpactFramework/Frontend/Console/Code/Source/TestImpactConsoleTestSequenceEventHandler.h

@@ -8,7 +8,7 @@
 
 #include <TestImpactFramework/TestImpactTestSequence.h>
 #include <TestImpactFramework/TestImpactClientTestSelection.h>
-#include <TestImpactFramework/TestImpactClientFailureReport.h>
+#include <TestImpactFramework/TestImpactClientSequenceReport.h>
 #include <TestImpactFramework/TestImpactClientTestRun.h>
 
 #include <AzCore/std/chrono/chrono.h>
@@ -21,48 +21,33 @@ namespace TestImpact
 {
     namespace Console
     {
-        //! Event handler for all test sequence types.
-        class TestSequenceEventHandler
-        {
-        public:
-            explicit TestSequenceEventHandler(SuiteType suiteFilter);
+        //! Handler for TestSequenceStartCallback event.
+        void TestSequenceStartCallback(SuiteType suiteType, const Client::TestRunSelection& selectedTests);
 
-            //! TestSequenceStartCallback.
-            void operator()(Client::TestRunSelection&& selectedTests);
+        //! Handler for TestSequenceStartCallback event.
+        void ImpactAnalysisTestSequenceStartCallback(
+            SuiteType suiteType,
+            const Client::TestRunSelection& selectedTests,
+            const AZStd::vector<AZStd::string>& discardedTests,
+            const AZStd::vector<AZStd::string>& draftedTests);
 
-            //! ImpactAnalysisTestSequenceStartCallback.
-            void operator()(
-                Client::TestRunSelection&& selectedTests,
-                AZStd::vector<AZStd::string>&& discardedTests,
-                AZStd::vector<AZStd::string>&& draftedTests);
+        //! Handler for SafeImpactAnalysisTestSequenceStartCallback event.
+        void SafeImpactAnalysisTestSequenceStartCallback(
+            SuiteType suiteType,
+            const Client::TestRunSelection& selectedTests,
+            const Client::TestRunSelection& discardedTests,
+            const AZStd::vector<AZStd::string>& draftedTests);
 
-            //! SafeImpactAnalysisTestSequenceStartCallback.
-            void operator()(
-                Client::TestRunSelection&& selectedTests,
-                Client::TestRunSelection&& discardedTests,
-                AZStd::vector<AZStd::string>&& draftedTests);
+        //! Handler for TestSequenceCompleteCallback event.
+        void TestSequenceCompleteCallback(const Client::SequenceReport& sequenceReport);
 
-            //! TestSequenceCompleteCallback.
-            void operator()(
-                Client::SequenceFailure&& failureReport,
-                AZStd::chrono::milliseconds duration);
+        //! Handler for ImpactAnalysisTestSequenceCompleteCallback event.
+        void ImpactAnalysisTestSequenceCompleteCallback(const Client::ImpactAnalysisSequenceReport& sequenceReport);
 
-            //! SafeTestSequenceCompleteCallback.
-            void operator()(
-                Client::SequenceFailure&& selectedFailureReport,
-                Client::SequenceFailure&& discardedFailureReport,
-                AZStd::chrono::milliseconds selectedDuration,
-                AZStd::chrono::milliseconds discaredDuration);
+        //! Handler for SafeImpactAnalysisTestSequenceCompleteCallback event.
+        void SafeImpactAnalysisTestSequenceCompleteCallback(const Client::SafeImpactAnalysisSequenceReport& sequenceReport);
 
-            //! TestRunCompleteCallback.
-            void operator()(Client::TestRun&& test);
-
-        private:
-            void ClearState();
-
-            SuiteType m_suiteFilter;
-            size_t m_numTests = 0;
-            size_t m_numTestsComplete = 0;
-        };
+        //! Handler for TestRunCompleteCallback event.
+        void TestRunCompleteCallback(const Client::TestRun& testRun, size_t numTestRunsCompleted, size_t totalNumTestRuns);
     } // namespace Console
 } // namespace TestImpact

Code/Tools/TestImpactFramework/Runtime/Code/Include/TestImpactFramework/TestImpactClientFailureReport.h

@@ -1,125 +0,0 @@
-/*
- * Copyright (c) Contributors to the Open 3D Engine Project.
- * For complete copyright and license terms please see the LICENSE at the root of this distribution.
- *
- * SPDX-License-Identifier: Apache-2.0 OR MIT
- *
- */
-
-#pragma once
-
-#include <AzCore/std/string/string.h>
-#include <AzCore/std/containers/vector.h>
-
-namespace TestImpact
-{
-    namespace Client
-    {
-        //! Represents a test target that failed, either due to failing to execute, completing in an abnormal state or completing with failing tests.
-        class TargetFailure
-        {
-        public:
-            TargetFailure(const AZStd::string& targetName);
-
-            //! Returns the name of the test target this failure pertains to.
-            const AZStd::string& GetTargetName() const;
-        private:
-            AZStd::string m_targetName;
-        };
-
-        //! Represents a test target that failed to execute.
-        class ExecutionFailure
-            : public TargetFailure
-        {
-        public:
-            ExecutionFailure(const AZStd::string& targetName, const AZStd::string& command);
-
-            //! Returns the command string used to execute this test target.
-            const AZStd::string& GetCommandString() const;
-        private:
-            AZStd::string m_commandString;
-        };
-
-        //! Represents an individual test of a test target that failed.
-        class TestFailure
-        {
-        public:
-            TestFailure(const AZStd::string& testName, const AZStd::string& errorMessage);
-
-            //! Returns the name of the test that failed.
-            const AZStd::string& GetName() const;
-
-            //! Returns the error message of the test that failed.
-            const AZStd::string& GetErrorMessage() const;
-
-        private:
-            AZStd::string m_name;
-            AZStd::string m_errorMessage;
-        };
-
-        //! Represents a collection of tests that failed.
-        //! @note Only the failing tests are included in the collection.
-        class TestCaseFailure
-        {
-        public:
-            TestCaseFailure(const AZStd::string& testCaseName, AZStd::vector<TestFailure>&& testFailures);
-
-            //! Returns the name of the test case containing the failing tests.
-            const AZStd::string& GetName() const;
-
-            //! Returns the collection of tests in this test case that failed.
-            const AZStd::vector<TestFailure>& GetTestFailures() const;
-
-        private:
-            AZStd::string m_name;
-            AZStd::vector<TestFailure> m_testFailures;
-        };
-
-        //! Represents a test target that launched successfully but contains failing tests.
-        class TestRunFailure
-            : public TargetFailure
-        {
-        public:
-            TestRunFailure(const AZStd::string& targetName, AZStd::vector<TestCaseFailure>&& testFailures);
-
-            //! Returns the total number of failing tests in this run.
-            size_t GetNumTestFailures() const;
-
-            //! Returns the test cases in this run containing failing tests.
-            const AZStd::vector<TestCaseFailure>& GetTestCaseFailures() const;
-
-        private:
-            AZStd::vector<TestCaseFailure> m_testCaseFailures;
-            size_t m_numTestFailures = 0;
-        };
-
-        //! Base class for reporting failing test sequences.
-        class SequenceFailure
-        {
-        public:
-            SequenceFailure(
-                AZStd::vector<ExecutionFailure>&& executionFailures,
-                AZStd::vector<TestRunFailure>&& testRunFailures,
-                AZStd::vector<TargetFailure>&& timedOutTests,
-                AZStd::vector<TargetFailure>&& unexecutedTests);
-
-            //! Returns the test targets in this sequence that failed to execute.
-            const AZStd::vector<ExecutionFailure>& GetExecutionFailures() const;
-
-            //! Returns the test targets that contain failing tests.
-            const AZStd::vector<TestRunFailure>& GetTestRunFailures() const;
-
-            //! Returns the test targets in this sequence that were terminated for exceeding their allotted runtime.
-            const AZStd::vector<TargetFailure>& GetTimedOutTests() const;
-
-            //! Returns the test targets in this sequence that were not executed due to the sequence terminating prematurely.
-            const AZStd::vector<TargetFailure>& GetUnexecutedTests() const;
-
-        private:
-            AZStd::vector<ExecutionFailure> m_executionFailures;
-            AZStd::vector<TestRunFailure> m_testRunFailures;
-            AZStd::vector<TargetFailure> m_timedOutTests;
-            AZStd::vector<TargetFailure> m_unexecutedTests;
-        };
-    } // namespace Client
-} // namespace TestImpact

Code/Tools/TestImpactFramework/Runtime/Code/Include/TestImpactFramework/TestImpactClientSequenceReport.h

@@ -0,0 +1,240 @@
+/*
+ * Copyright (c) Contributors to the Open 3D Engine Project. For complete copyright and license terms please see the LICENSE at the root of this distribution.
+ * 
+ * SPDX-License-Identifier: Apache-2.0 OR MIT
+ *
+ */
+
+#pragma once
+
+#include <TestImpactFramework/TestImpactClientTestRun.h>
+#include <TestImpactFramework/TestImpactClientTestSelection.h>
+#include <TestImpactFramework/TestImpactTestSequence.h>
+
+namespace TestImpact
+{
+    namespace Client
+    {
+        //! Report detailing the result and duration of a given set of test runs along with the details of each individual test run.
+        class TestRunReport
+        {
+        public:
+            //! Constructs the report for the given set of test runs that were run together in the same set.
+            //! @param result The result of this set of test runs.
+            //! @param startTime The time point his set of test runs started.
+            //! @param duration The duration this set of test runs took to complete.
+            //! @param passingTests The set of test runs that executed successfully with no failing tests.
+            //! @param failing tests The set of test runs that executed successfully but had one or more failing tests.
+            //! @param executionFailureTests The set of test runs that failed to execute.
+            //! @param timedOutTests The set of test runs that executed successfully but were terminated prematurely due to timing out.
+            //! @param unexecutedTests The set of test runs that were queued up for execution but did not get the opportunity to execute.
+            TestRunReport(
+                TestSequenceResult result,
+                AZStd::chrono::high_resolution_clock::time_point startTime,
+                AZStd::chrono::milliseconds duration,
+                AZStd::vector<TestRun>&& passingTests,
+                AZStd::vector<TestRunWithTestFailures>&& failingTests,
+                AZStd::vector<TestRun>&& executionFailureTests,
+                AZStd::vector<TestRun>&& timedOutTests,
+                AZStd::vector<TestRun>&& unexecutedTests);
+
+            //! Returns the result of this sequence of test runs.
+            TestSequenceResult GetResult() const;
+
+            //! Returns the time this sequence of test runs started relative to T0.
+            AZStd::chrono::high_resolution_clock::time_point GetStartTime() const;
+
+            //! Returns the time this sequence of test runs ended relative to T0.
+            AZStd::chrono::high_resolution_clock::time_point GetEndTime() const;
+
+            //! Returns the duration this sequence of test runs took to complete.
+            AZStd::chrono::milliseconds GetDuration() const;
+
+            //! Returns the number of passing test runs.
+            size_t GetNumPassingTests() const;
+
+            //! Returns the number of failing test runs.
+            size_t GetNumFailingTests() const;
+
+            //! Returns the number of timed out test runs.
+            size_t GetNumTimedOutTests() const;
+
+            //! Returns the number of unexecuted test runs.
+            size_t GetNumUnexecutedTests() const;
+
+            //! Returns the set of test runs that executed successfully with no failing tests.
+            const AZStd::vector<TestRun>& GetPassingTests() const;
+
+            //! Returns the set of test runs that executed successfully but had one or more failing tests.
+            const AZStd::vector<TestRunWithTestFailures>& GetFailingTests() const;
+
+            //! Returns the set of test runs that failed to execute.
+            const AZStd::vector<TestRun>& GetExecutionFailureTests() const;
+
+            //! Returns the set of test runs that executed successfully but were terminated prematurely due to timing out.
+            const AZStd::vector<TestRun>& GetTimedOutTests() const;
+
+            //! Returns the set of test runs that were queued up for execution but did not get the opportunity to execute.
+            const AZStd::vector<TestRun>& GetUnexecutedTests() const;
+        private:
+            TestSequenceResult m_result;
+            AZStd::chrono::high_resolution_clock::time_point m_startTime;
+            AZStd::chrono::milliseconds m_duration;
+            AZStd::vector<TestRun> m_passingTests;
+            AZStd::vector<TestRunWithTestFailures> m_failingTests;
+            AZStd::vector<TestRun> m_executionFailureTests;
+            AZStd::vector<TestRun> m_timedOutTests;
+            AZStd::vector<TestRun> m_unexecutedTests;
+        };
+
+        //! Report detailing a test run sequence of selected tests.
+        class SequenceReport
+        {
+        public:
+            //! Constructs the report for a sequence of selected tests.
+            //! @param suiteType The suite from which the tests have been selected from.
+            //! @param selectedTests The target names of the selected tests.
+            //! @param selectedTestRunReport The report for the set of selected test runs.
+            SequenceReport(SuiteType suiteType, const TestRunSelection& selectedTests, TestRunReport&& selectedTestRunReport);
+
+            //! Returns the tests selected for running in the sequence.
+            TestRunSelection GetSelectedTests() const;
+
+            //! Returns the report for the selected test runs.
+            TestRunReport GetSelectedTestRunReport() const;
+
+            //! Returns the start time of the sequence.
+            AZStd::chrono::high_resolution_clock::time_point GetStartTime() const;
+
+            //! Returns the end time of the sequence.
+            AZStd::chrono::high_resolution_clock::time_point GetEndTime() const;
+
+            //! Returns the result of the sequence.
+            virtual TestSequenceResult GetResult() const;
+
+            //! Returns the entire duration the sequence took from start to finish.
+            virtual AZStd::chrono::milliseconds GetDuration() const;
+
+            //! Get the total number of tests in the sequence that passed.
+            virtual size_t GetTotalNumPassingTests() const;
+
+            //! Get the total number of tests in the sequence that contain one or more test failures.
+            virtual size_t GetTotalNumFailingTests() const;
+
+            //! Get the total number of tests in the sequence that timed out whilst in flight.
+            virtual size_t GetTotalNumTimedOutTests() const;
+
+            //! Get the total number of tests in the sequence that were queued for execution but did not get the oppurtunity to execute.
+            virtual size_t GetTotalNumUnexecutedTests() const;
+
+        private:
+            SuiteType m_suite;
+            TestRunSelection m_selectedTests;
+            TestRunReport m_selectedTestRunReport;
+        };
+
+        //! Report detailing a test run sequence of selected and drafted tests.
+        class DraftingSequenceReport
+            : public SequenceReport
+        {
+        public:
+            //! Constructs the report for a sequence of selected and drafted tests.
+            //! @param suiteType The suite from which the tests have been selected from.
+            //! @param selectedTests The target names of the selected tests.
+            //! @param draftedTests The target names of the drafted tests.
+            //! @param selectedTestRunReport The report for the set of selected test runs.
+            //! @param draftedTestRunReport The report for the set of drafted test runs.
+            DraftingSequenceReport(
+                SuiteType suiteType,
+                const TestRunSelection& selectedTests,
+                const AZStd::vector<AZStd::string>& draftedTests,
+                TestRunReport&& selectedTestRunReport,
+                TestRunReport&& draftedTestRunReport);
+
+            // SequenceReport overrides ...
+            TestSequenceResult GetResult() const override;
+            AZStd::chrono::milliseconds GetDuration() const override;
+            size_t GetTotalNumPassingTests() const override;
+            size_t GetTotalNumFailingTests() const override;
+            size_t GetTotalNumTimedOutTests() const override;
+            size_t GetTotalNumUnexecutedTests() const override;
+
+             //! Returns the tests drafted for running in the sequence.
+            const AZStd::vector<AZStd::string>& GetDraftedTests() const;
+
+            //! Returns the report for the drafted test runs.
+            TestRunReport GetDraftedTestRunReport() const;
+
+        private:
+            AZStd::vector<AZStd::string> m_draftedTests;
+            TestRunReport m_draftedTestRunReport;
+        };
+
+        //! Report detailing an impact analysis sequence of selected, discarded and drafted tests.
+        class ImpactAnalysisSequenceReport
+            : public DraftingSequenceReport
+        {
+        public:
+            //! Constructs the report for a sequence of selected and drafted tests.
+            //! @param suiteType The suite from which the tests have been selected from.
+            //! @param selectedTests The target names of the selected tests.
+            //! @param discardedTests The target names of the discarded tests.
+            //! @param draftedTests The target names of the drafted tests.
+            //! @param selectedTestRunReport The report for the set of selected test runs.
+            //! @param draftedTestRunReport The report for the set of drafted test runs.
+            ImpactAnalysisSequenceReport(
+                SuiteType suiteType,
+                const TestRunSelection& selectedTests,
+                const AZStd::vector<AZStd::string>& discardedTests,
+                const AZStd::vector<AZStd::string>& draftedTests,
+                TestRunReport&& selectedTestRunReport,
+                TestRunReport&& draftedTestRunReport);
+
+            //! Returns the tests discarded from running in the sequence.
+            const AZStd::vector<AZStd::string>& GetDiscardedTests() const;
+        private:
+            AZStd::vector<AZStd::string> m_discardedTests;
+        };
+
+        //! Report detailing an impact analysis sequence of selected, discarded and drafted tests.
+        class SafeImpactAnalysisSequenceReport
+            : public DraftingSequenceReport
+        {
+        public:
+            //! Constructs the report for a sequence of selected and drafted tests.
+            //! @param suiteType The suite from which the tests have been selected from.
+            //! @param selectedTests The target names of the selected tests.
+            //! @param discardedTests The target names of the discarded tests.
+            //! @param draftedTests The target names of the drafted tests.
+            //! @param selectedTestRunReport The report for the set of selected test runs.
+            //! @param discardedTestRunReport The report for the set of discarded test runs.
+            //! @param draftedTestRunReport The report for the set of drafted test runs.
+            SafeImpactAnalysisSequenceReport(
+                SuiteType suiteType,
+                const TestRunSelection& selectedTests,
+                const TestRunSelection& discardedTests,
+                const AZStd::vector<AZStd::string>& draftedTests,
+                TestRunReport&& selectedTestRunReport,
+                TestRunReport&& discardedTestRunReport,
+                TestRunReport&& draftedTestRunReport);
+
+            // DraftingSequenceReport overrides ...
+            TestSequenceResult GetResult() const override;
+            AZStd::chrono::milliseconds GetDuration() const override;
+            size_t GetTotalNumPassingTests() const override;
+            size_t GetTotalNumFailingTests() const override;
+            size_t GetTotalNumTimedOutTests() const override;
+            size_t GetTotalNumUnexecutedTests() const override;
+
+            //! Returns the report for the discarded test runs.
+            const TestRunSelection GetDiscardedTests() const;
+
+            //! Returns the report for the discarded test runs.
+            TestRunReport GetDiscardedTestRunReport() const;
+
+        private:
+            TestRunSelection m_discardedTests;
+            TestRunReport m_discardedTestRunReport;
+        };
+    } // namespace Client
+} // namespace TestImpact

Code/Tools/TestImpactFramework/Runtime/Code/Include/TestImpactFramework/TestImpactClientTestRun.h

@@ -6,8 +6,9 @@
  *
  */
 
-#include <AzCore/std/string/string.h>
 #include <AzCore/std/chrono/chrono.h>
+#include <AzCore/std/containers/vector.h>
+#include <AzCore/std/string/string.h>
 
 #pragma once
 
@@ -25,18 +26,118 @@ namespace TestImpact
             AllTestsPass //!< The test run completed its run and all tests passed.
         };
 
+        //! Representation of a completed test run.
         class TestRun
         {
         public:
-            TestRun(const AZStd::string& name, TestRunResult result, AZStd::chrono::milliseconds duration);
+            //! Constructs the client facing representation of a given test target's run.
+            //! @param name The name of the test target.
+            //! @param commandString The command string used to execute this test target.
+            //! @param startTime The start time, relative to the sequence start, that this run started.
+            //! @param duration The duration that this test run took to complete.
+            //! @param result The result of the run.
+            TestRun(
+                const AZStd::string& name,
+                const AZStd::string& commandString,
+                AZStd::chrono::high_resolution_clock::time_point startTime,
+                AZStd::chrono::milliseconds duration,
+                TestRunResult result);
+
+            //! Returns the test target name.
             const AZStd::string& GetTargetName() const;
+
+            //! Returns the test run result.
             TestRunResult GetResult() const;
+
+            //! Returns the test run start time.
+            AZStd::chrono::high_resolution_clock::time_point GetStartTime() const;
+
+            //! Returns the end time, relative to the sequence start, that this run ended.
+            AZStd::chrono::high_resolution_clock::time_point GetEndTime() const;
+
+            //! Returns the duration that this test run took to complete.
             AZStd::chrono::milliseconds GetDuration() const;
 
+            //! Returns the command string used to execute this test target.
+            const AZStd::string& GetCommandString() const;
+
         private:
             AZStd::string m_targetName;
+            AZStd::string m_commandString;
             TestRunResult m_result;
+            AZStd::chrono::high_resolution_clock::time_point m_startTime;
             AZStd::chrono::milliseconds m_duration;
         };
+
+        //! Represents an individual test of a test target that failed.
+        class TestFailure
+        {
+        public:
+            TestFailure(const AZStd::string& testName, const AZStd::string& errorMessage);
+
+            //! Returns the name of the test that failed.
+            const AZStd::string& GetName() const;
+
+            //! Returns the error message of the test that failed.
+            const AZStd::string& GetErrorMessage() const;
+
+        private:
+            AZStd::string m_name;
+            AZStd::string m_errorMessage;
+        };
+
+        //! Represents a collection of tests that failed.
+        //! @note Only the failing tests are included in the collection.
+        class TestCaseFailure
+        {
+        public:
+            TestCaseFailure(const AZStd::string& testCaseName, AZStd::vector<TestFailure>&& testFailures);
+
+            //! Returns the name of the test case containing the failing tests.
+            const AZStd::string& GetName() const;
+
+            //! Returns the collection of tests in this test case that failed.
+            const AZStd::vector<TestFailure>& GetTestFailures() const;
+
+        private:
+            AZStd::string m_name;
+            AZStd::vector<TestFailure> m_testFailures;
+        };
+
+        //! Representation of a test run's failing tests.
+        class TestRunWithTestFailures
+            : public TestRun
+        {
+        public:
+            //! Constructs the client facing representation of a given test target's run.
+            //! @param name The name of the test target.
+            //! @param commandString The command string used to execute this test target.
+            //! @param startTime The start time, relative to the sequence start, that this run started.
+            //! @param duration The duration that this test run took to complete.
+            //! @param result The result of the run.
+            //! @param testFailures The failing tests for this test run.
+            TestRunWithTestFailures(
+                const AZStd::string& name,
+                const AZStd::string& commandString,
+                AZStd::chrono::high_resolution_clock::time_point startTime,
+                AZStd::chrono::milliseconds duration,
+                TestRunResult result,
+                AZStd::vector<TestCaseFailure>&& testFailures);
+
+            //! Constructs the client facing representation of a given test target's run.
+            //! @param testRun The test run this run is to be derived from.
+            //! @param testFailures The failing tests for this run.
+            TestRunWithTestFailures(TestRun&& testRun, AZStd::vector<TestCaseFailure>&& testFailures);
+
+            //! Returns the total number of failing tests in this run.
+            size_t GetNumTestFailures() const;
+
+            //! Returns the test cases in this run containing failing tests.
+            const AZStd::vector<TestCaseFailure>& GetTestCaseFailures() const;
+
+        private:
+            AZStd::vector<TestCaseFailure> m_testCaseFailures;
+            size_t m_numTestFailures = 0;
+        };
     } // namespace Client
 } // namespace TestImpact

Code/Tools/TestImpactFramework/Runtime/Code/Include/TestImpactFramework/TestImpactClientTestSelection.h

@@ -21,6 +21,7 @@ namespace TestImpact
         class TestRunSelection
         {
         public:
+            TestRunSelection() = default;
             TestRunSelection(const AZStd::vector<AZStd::string>& includedTests, const AZStd::vector<AZStd::string>& excludedTests);
             TestRunSelection(AZStd::vector<AZStd::string>&& includedTests, AZStd::vector<AZStd::string>&& excludedTests);
 

Code/Tools/TestImpactFramework/Runtime/Code/Include/TestImpactFramework/TestImpactRuntime.h

@@ -12,7 +12,7 @@
 #include <TestImpactFramework/TestImpactChangeList.h>
 #include <TestImpactFramework/TestImpactClientTestSelection.h>
 #include <TestImpactFramework/TestImpactClientTestRun.h>
-#include <TestImpactFramework/TestImpactClientFailureReport.h>
+#include <TestImpactFramework/TestImpactClientSequenceReport.h>
 #include <TestImpactFramework/TestImpactTestSequence.h>
 
 #include <AzCore/std/string/string.h>
@@ -34,10 +34,12 @@ namespace TestImpact
     class TestEngineInstrumentedRun;
 
     //! Callback for a test sequence that isn't using test impact analysis to determine selected tests.
+    //! @parm suiteType The test suite to select tests from.
     //! @param tests The tests that will be run for this sequence.
-    using TestSequenceStartCallback = AZStd::function<void(Client::TestRunSelection&& tests)>;
+    using TestSequenceStartCallback = AZStd::function<void(SuiteType suiteType, const Client::TestRunSelection& tests)>;
 
     //! Callback for a test sequence using test impact analysis.
+    //! @parm suiteType The test suite to select tests from.
     //! @param selectedTests The tests that have been selected for this run by test impact analysis.
     //! @param discardedTests The tests that have been rejected for this run by test impact analysis. 
     //! @param draftedTests The tests that have been drafted in for this run due to requirements outside of test impact analysis
@@ -46,11 +48,13 @@ namespace TestImpact
     //! These tests will be run with coverage instrumentation.
     //! @note discardedTests and draftedTests may contain overlapping tests.
     using ImpactAnalysisTestSequenceStartCallback = AZStd::function<void(
-        Client::TestRunSelection&& selectedTests,
-        AZStd::vector<AZStd::string>&& discardedTests,
-        AZStd::vector<AZStd::string>&& draftedTests)>;
+        SuiteType suiteType,
+        const Client::TestRunSelection& selectedTests,
+        const AZStd::vector<AZStd::string>& discardedTests,
+        const AZStd::vector<AZStd::string>& draftedTests)>;
 
     //! Callback for a test sequence using test impact analysis.
+    //! @parm suiteType The test suite to select tests from.
     //! @param selectedTests The tests that have been selected for this run by test impact analysis.
     //! @param discardedTests The tests that have been rejected for this run by test impact analysis.
     //! These tests will not be run without coverage instrumentation unless there is an entry in the draftedTests list.
@@ -59,30 +63,22 @@ namespace TestImpact
     //! to execute previously).
     //! @note discardedTests and draftedTests may contain overlapping tests.
     using SafeImpactAnalysisTestSequenceStartCallback = AZStd::function<void(
-        Client::TestRunSelection&& selectedTests,
-        Client::TestRunSelection&& discardedTests,
-        AZStd::vector<AZStd::string>&& draftedTests)>;
+        SuiteType suiteType,
+        const Client::TestRunSelection& selectedTests,
+        const Client::TestRunSelection& discardedTests,
+        const AZStd::vector<AZStd::string>& draftedTests)>;
 
     //! Callback for end of a test sequence.
-    //! @param failureReport The test runs that failed for any reason during this sequence.
-    //! @param duration The total duration of this test sequence.
-    using TestSequenceCompleteCallback = AZStd::function<void(
-        Client::SequenceFailure&& failureReport,
-        AZStd::chrono::milliseconds duration)>;
-
-    //! Callback for end of a test impact analysis test sequence.
-    //! @param selectedFailureReport The selected test runs that failed for any reason during this sequence.
-    //! @param discardedFailureReport The discarded test runs that failed for any reason during this sequence.
-    //! @param duration The total duration of this test sequence.
-    using SafeTestSequenceCompleteCallback = AZStd::function<void(
-        Client::SequenceFailure&& selectedFailureReport,
-        Client::SequenceFailure&& discardedFailureReport,
-        AZStd::chrono::milliseconds selectedDuration,
-        AZStd::chrono::milliseconds discardedDuration)>;
+    //! @tparam SequenceReportType The report type to be used for the sequence.
+    //! @param sequenceReport The completed sequence report.
+    template<typename SequenceReportType>
+    using TestSequenceCompleteCallback = AZStd::function<void(const SequenceReportType& sequenceReport)>;
 
     //! Callback for test runs that have completed for any reason.
-    //! @param selectedTests The test that has completed.
-    using TestRunCompleteCallback = AZStd::function<void(Client::TestRun&& selectedTests)>;
+    //! @param testRunMeta The test that has completed.
+    //! @param numTestRunsCompleted The number of test runs that have completed.
+    //! @param totalNumTestRuns The total number of test runs in the sequence.
+    using TestRunCompleteCallback = AZStd::function<void(Client::TestRun& testRun, size_t numTestRunsCompleted, size_t totalNumTestRuns)>;
 
     //! The API exposed to the client responsible for all test runs and persistent data management.
     class Runtime
@@ -108,19 +104,19 @@ namespace TestImpact
             AZStd::optional<size_t> maxConcurrency = AZStd::nullopt);
 
         ~Runtime();
-
+        
         //! Runs a test sequence where all tests with a matching suite in the suite filter and also not on the excluded list are selected.
         //! @param testTargetTimeout The maximum duration individual test targets may be in flight for (infinite if empty).
         //! @param globalTimeout The maximum duration the entire test sequence may run for (infinite if empty).
         //! @param testSequenceStartCallback The client function to be called after the test targets have been selected but prior to running the tests.
         //! @param testSequenceCompleteCallback The client function to be called after the test sequence has completed.
         //! @param testRunCompleteCallback The client function to be called after an individual test run has completed.
-        //! @returns
-        TestSequenceResult RegularTestSequence(
+        //! @returns The test run and sequence report for the selected test sequence.
+        Client::SequenceReport RegularTestSequence(
             AZStd::optional<AZStd::chrono::milliseconds> testTargetTimeout,
             AZStd::optional<AZStd::chrono::milliseconds> globalTimeout,
             AZStd::optional<TestSequenceStartCallback> testSequenceStartCallback,
-            AZStd::optional<TestSequenceCompleteCallback> testSequenceCompleteCallback,
+            AZStd::optional<TestSequenceCompleteCallback<Client::SequenceReport>> testSequenceCompleteCallback,
             AZStd::optional<TestRunCompleteCallback> testRunCompleteCallback);
 
         //! Runs a test sequence where tests are selected according to test impact analysis so long as they are not on the excluded list.
@@ -132,15 +128,15 @@ namespace TestImpact
         //! @param testSequenceStartCallback The client function to be called after the test targets have been selected but prior to running the tests.
         //! @param testSequenceCompleteCallback The client function to be called after the test sequence has completed.
         //! @param testRunCompleteCallback The client function to be called after an individual test run has completed.
-        //! @returns
-        TestSequenceResult ImpactAnalysisTestSequence(
+        //! @returns The test run and sequence report for the selected and drafted test sequences.
+        Client::ImpactAnalysisSequenceReport ImpactAnalysisTestSequence(
             const ChangeList& changeList,
             Policy::TestPrioritization testPrioritizationPolicy,
             Policy::DynamicDependencyMap dynamicDependencyMapPolicy,
             AZStd::optional<AZStd::chrono::milliseconds> testTargetTimeout,
             AZStd::optional<AZStd::chrono::milliseconds> globalTimeout,
             AZStd::optional<ImpactAnalysisTestSequenceStartCallback> testSequenceStartCallback,
-            AZStd::optional<TestSequenceCompleteCallback> testSequenceCompleteCallback,
+            AZStd::optional<TestSequenceCompleteCallback<Client::ImpactAnalysisSequenceReport>> testSequenceCompleteCallback,
             AZStd::optional<TestRunCompleteCallback> testRunCompleteCallback);
 
         //! Runs a test sequence as per the ImpactAnalysisTestSequence where the tests not selected are also run (albeit without instrumentation).
@@ -151,14 +147,14 @@ namespace TestImpact
         //! @param testSequenceStartCallback The client function to be called after the test targets have been selected but prior to running the tests.
         //! @param testSequenceCompleteCallback The client function to be called after the test sequence has completed.
         //! @param testRunCompleteCallback The client function to be called after an individual test run has completed.
-        //! @returns
-        AZStd::pair<TestSequenceResult, TestSequenceResult> SafeImpactAnalysisTestSequence(
+        //! @returns The test run and sequence report for the selected, discarded and drafted test sequences.
+        Client::SafeImpactAnalysisSequenceReport SafeImpactAnalysisTestSequence(
             const ChangeList& changeList,
             Policy::TestPrioritization testPrioritizationPolicy,
             AZStd::optional<AZStd::chrono::milliseconds> testTargetTimeout,
             AZStd::optional<AZStd::chrono::milliseconds> globalTimeout,
             AZStd::optional<SafeImpactAnalysisTestSequenceStartCallback> testSequenceStartCallback,
-            AZStd::optional<SafeTestSequenceCompleteCallback> testSequenceCompleteCallback,
+            AZStd::optional<TestSequenceCompleteCallback<Client::SafeImpactAnalysisSequenceReport>> testSequenceCompleteCallback,
             AZStd::optional<TestRunCompleteCallback> testRunCompleteCallback);
 
         //! Runs all tests not on the excluded list and uses their coverage data to seed the test impact analysis data (ant existing data will be overwritten).
@@ -167,12 +163,12 @@ namespace TestImpact
         //! @param testSequenceStartCallback The client function to be called after the test targets have been selected but prior to running the tests.
         //! @param testSequenceCompleteCallback The client function to be called after the test sequence has completed.
         //! @param testRunCompleteCallback The client function to be called after an individual test run has completed.
-        //! 
-        TestSequenceResult SeededTestSequence(
+        //! @returns The test run and sequence report for the selected test sequence.
+        Client::SequenceReport SeededTestSequence(
             AZStd::optional<AZStd::chrono::milliseconds> testTargetTimeout,
             AZStd::optional<AZStd::chrono::milliseconds> globalTimeout,
             AZStd::optional<TestSequenceStartCallback> testSequenceStartCallback,
-            AZStd::optional<TestSequenceCompleteCallback> testSequenceCompleteCallback,
+            AZStd::optional<TestSequenceCompleteCallback<Client::SequenceReport>> testSequenceCompleteCallback,
             AZStd::optional<TestRunCompleteCallback> testRunCompleteCallback);
 
         //! Returns true if the runtime has test impact analysis data (either preexisting or generated).
@@ -209,8 +205,8 @@ namespace TestImpact
         void UpdateAndSerializeDynamicDependencyMap(const AZStd::vector<TestEngineInstrumentedRun>& jobs);
 
         RuntimeConfig m_config;
-        SuiteType m_suiteFilter;
         RepoPath m_sparTIAFile;
+        SuiteType m_suiteFilter;
         Policy::ExecutionFailure m_executionFailurePolicy;
         Policy::FailedTestCoverage m_failedTestCoveragePolicy;
         Policy::TestFailure m_testFailurePolicy;

Code/Tools/TestImpactFramework/Runtime/Code/Source/TestEngine/TestImpactTestEngine.cpp

@@ -262,7 +262,7 @@ namespace TestImpact
         Policy::TestFailure testFailurePolicy,
         AZStd::optional<AZStd::chrono::milliseconds> testTargetTimeout,
         AZStd::optional<AZStd::chrono::milliseconds> globalTimeout,
-        AZStd::optional<TestEngineJobCompleteCallback> callback)
+        AZStd::optional<TestEngineJobCompleteCallback> callback) const
     {
         TestEngineJobMap<TestEnumerator::JobInfo::IdType> engineJobs;
         const auto jobInfos = m_testJobInfoGenerator->GenerateTestEnumerationJobInfos(testTargets, TestEnumerator::JobInfo::CachePolicy::Write);
@@ -285,7 +285,7 @@ namespace TestImpact
         [[maybe_unused]]Policy::TargetOutputCapture targetOutputCapture,
         AZStd::optional<AZStd::chrono::milliseconds> testTargetTimeout,
         AZStd::optional<AZStd::chrono::milliseconds> globalTimeout,
-        AZStd::optional<TestEngineJobCompleteCallback> callback)
+        AZStd::optional<TestEngineJobCompleteCallback> callback) const
     {
         DeleteArtifactXmls();
 
@@ -312,7 +312,7 @@ namespace TestImpact
         [[maybe_unused]]Policy::TargetOutputCapture targetOutputCapture,
         AZStd::optional<AZStd::chrono::milliseconds> testTargetTimeout,
         AZStd::optional<AZStd::chrono::milliseconds> globalTimeout,
-        AZStd::optional<TestEngineJobCompleteCallback> callback)
+        AZStd::optional<TestEngineJobCompleteCallback> callback) const
     {
         DeleteArtifactXmls();
 

Code/Tools/TestImpactFramework/Runtime/Code/Source/TestEngine/TestImpactTestEngine.h

@@ -69,7 +69,7 @@ namespace TestImpact
             Policy::TestFailure testFailurePolicy,
             AZStd::optional<AZStd::chrono::milliseconds> testTargetTimeout,
             AZStd::optional<AZStd::chrono::milliseconds> globalTimeout,
-            AZStd::optional<TestEngineJobCompleteCallback> callback);
+            AZStd::optional<TestEngineJobCompleteCallback> callback) const;
 
         //! Performs a test run without any instrumentation and, for each test target, returns the test run results and metrics about the run.
         //! @param testTargets The test targets to run.
@@ -89,7 +89,7 @@ namespace TestImpact
             Policy::TargetOutputCapture targetOutputCapture,
             AZStd::optional<AZStd::chrono::milliseconds> testTargetTimeout,
             AZStd::optional<AZStd::chrono::milliseconds> globalTimeout,
-            AZStd::optional<TestEngineJobCompleteCallback> callback);
+            AZStd::optional<TestEngineJobCompleteCallback> callback) const;
 
         //! Performs a test run with instrumentation and, for each test target, returns the test run results, coverage data and metrics about the run.
         //! @param testTargets The test targets to run.
@@ -111,7 +111,7 @@ namespace TestImpact
             Policy::TargetOutputCapture targetOutputCapture,
             AZStd::optional<AZStd::chrono::milliseconds> testTargetTimeout,
             AZStd::optional<AZStd::chrono::milliseconds> globalTimeout,
-            AZStd::optional<TestEngineJobCompleteCallback> callback);
+            AZStd::optional<TestEngineJobCompleteCallback> callback) const;
 
     private:
         //! Cleans up the artifacts directory of any artifacts from previous runs.

Code/Tools/TestImpactFramework/Runtime/Code/Source/TestImpactClientFailureReport.cpp

@@ -1,120 +0,0 @@
-/*
- * Copyright (c) Contributors to the Open 3D Engine Project.
- * For complete copyright and license terms please see the LICENSE at the root of this distribution.
- *
- * SPDX-License-Identifier: Apache-2.0 OR MIT
- *
- */
-
-#include <TestImpactFramework/TestImpactClientFailureReport.h>
-
-namespace TestImpact
-{
-    namespace Client
-    {
-        TargetFailure::TargetFailure(const AZStd::string& targetName)
-            : m_targetName(targetName)
-        {
-        }
-
-        const AZStd::string& TargetFailure::GetTargetName() const
-        {
-            return m_targetName;
-        }
-
-        ExecutionFailure::ExecutionFailure(const AZStd::string& targetName, const AZStd::string& command)
-            : TargetFailure(targetName)
-            , m_commandString(command)
-        {
-        }
-
-        const AZStd::string& ExecutionFailure::GetCommandString() const
-        {
-            return m_commandString;
-        }
-
-        TestFailure::TestFailure(const AZStd::string& testName, const AZStd::string& errorMessage)
-            : m_name(testName)
-            , m_errorMessage(errorMessage)
-        {
-        }
-
-        const AZStd::string& TestFailure::GetName() const
-        {
-            return m_name;
-        }
-
-        const AZStd::string& TestFailure::GetErrorMessage() const
-        {
-            return m_errorMessage;
-        }
-
-        TestCaseFailure::TestCaseFailure(const AZStd::string& testCaseName, AZStd::vector<TestFailure>&& testFailures)
-            : m_name(testCaseName)
-            , m_testFailures(AZStd::move(testFailures))
-        {
-        }
-
-        const AZStd::string& TestCaseFailure::GetName() const
-        {
-            return m_name;
-        }
-
-        const AZStd::vector<TestFailure>& TestCaseFailure::GetTestFailures() const
-        {
-            return m_testFailures;
-        }
-
-        TestRunFailure::TestRunFailure(const AZStd::string& targetName, AZStd::vector<TestCaseFailure>&& testFailures)
-            : TargetFailure(targetName)
-            , m_testCaseFailures(AZStd::move(testFailures))
-        {
-            for (const auto& testCase : m_testCaseFailures)
-            {
-                m_numTestFailures += testCase.GetTestFailures().size();
-            }
-        }
-
-        size_t TestRunFailure::GetNumTestFailures() const
-        {
-            return m_numTestFailures;
-        }
-
-        const AZStd::vector<TestCaseFailure>& TestRunFailure::GetTestCaseFailures() const
-        {
-            return m_testCaseFailures;
-        }
-
-        SequenceFailure::SequenceFailure(
-            AZStd::vector<ExecutionFailure>&& executionFailures,
-            AZStd::vector<TestRunFailure>&& testRunFailures,
-            AZStd::vector<TargetFailure>&& timedOutTests,
-            AZStd::vector<TargetFailure>&& unexecutionTests)
-            : m_executionFailures(AZStd::move(executionFailures))
-            , m_testRunFailures(testRunFailures)
-            , m_timedOutTests(AZStd::move(timedOutTests))
-            , m_unexecutedTests(AZStd::move(unexecutionTests))
-        {
-        }
-
-        const AZStd::vector<ExecutionFailure>& SequenceFailure::GetExecutionFailures() const
-        {
-            return m_executionFailures;
-        }
-
-        const AZStd::vector<TestRunFailure>& SequenceFailure::GetTestRunFailures() const
-        {
-            return m_testRunFailures;
-        }
-
-        const AZStd::vector<TargetFailure>& SequenceFailure::GetTimedOutTests() const
-        {
-            return m_timedOutTests;
-        }
-
-        const AZStd::vector<TargetFailure>& SequenceFailure::GetUnexecutedTests() const
-        {
-            return m_unexecutedTests;
-        }
-    } // namespace Client
-} // namespace TestImpact

Code/Tools/TestImpactFramework/Runtime/Code/Source/TestImpactClientSequenceReport.cpp

@@ -0,0 +1,312 @@
+/*
+ * Copyright (c) Contributors to the Open 3D Engine Project. For complete copyright and license terms please see the LICENSE at the root of this distribution.
+ * 
+ * SPDX-License-Identifier: Apache-2.0 OR MIT
+ *
+ */
+
+#include <TestImpactFramework/TestImpactClientSequenceReport.h>
+
+namespace TestImpact
+{
+    namespace Client
+    {
+        //! Calculates the final sequence result for a composite of multiple sequences.
+        TestSequenceResult CalculateMultiTestSequenceResult(const AZStd::vector<TestSequenceResult>& results)
+        {
+            // Order of precedence:
+            // 1. TestSequenceResult::Failure
+            // 2. TestSequenceResult::Timeout
+            // 3. TestSequenceResult::Success
+
+            if (const auto it = AZStd::find(results.begin(), results.end(), TestSequenceResult::Failure);
+                it != results.end())
+            {
+                return TestSequenceResult::Failure;
+            }
+            
+            if (const auto it = AZStd::find(results.begin(), results.end(), TestSequenceResult::Timeout);
+                it != results.end())
+            {
+                return TestSequenceResult::Timeout;
+            }
+
+            return TestSequenceResult::Success;
+        }
+
+        TestRunReport::TestRunReport(
+            TestSequenceResult result,
+            AZStd::chrono::high_resolution_clock::time_point startTime,
+            AZStd::chrono::milliseconds duration,
+            AZStd::vector<TestRun>&& passingTests,
+            AZStd::vector<TestRunWithTestFailures>&& failingTests,
+            AZStd::vector<TestRun>&& executionFailureTests,
+            AZStd::vector<TestRun>&& timedOutTests,
+            AZStd::vector<TestRun>&& unexecutedTests)
+            : m_startTime(startTime)
+            , m_result(result)
+            , m_duration(duration)
+            , m_passingTests(AZStd::move(passingTests))
+            , m_failingTests(AZStd::move(failingTests))
+            , m_executionFailureTests(AZStd::move(executionFailureTests))
+            , m_timedOutTests(AZStd::move(timedOutTests))
+            , m_unexecutedTests(AZStd::move(unexecutedTests))
+        {
+        }
+
+        TestSequenceResult TestRunReport::GetResult() const
+        {
+            return m_result;
+        }
+
+        AZStd::chrono::high_resolution_clock::time_point TestRunReport::GetStartTime() const
+        {
+            return m_startTime;
+        }
+
+        AZStd::chrono::high_resolution_clock::time_point TestRunReport::GetEndTime() const
+        {
+            return m_startTime + m_duration;
+        }
+
+        AZStd::chrono::milliseconds TestRunReport::GetDuration() const
+        {
+            return m_duration;
+        }
+
+        size_t TestRunReport::GetNumPassingTests() const
+        {
+            return m_passingTests.size();
+        }
+
+        size_t TestRunReport::GetNumFailingTests() const
+        {
+            return m_failingTests.size();
+        }
+
+        size_t TestRunReport::GetNumTimedOutTests() const
+        {
+            return m_timedOutTests.size();
+        }
+
+        size_t TestRunReport::GetNumUnexecutedTests() const
+        {
+            return m_unexecutedTests.size();
+        }
+
+        const AZStd::vector<TestRun>& TestRunReport::GetPassingTests() const
+        {
+            return m_passingTests;
+        }
+
+        const AZStd::vector<TestRunWithTestFailures>& TestRunReport::GetFailingTests() const
+        {
+            return m_failingTests;
+        }
+
+        const AZStd::vector<TestRun>& TestRunReport::GetExecutionFailureTests() const
+        {
+            return m_executionFailureTests;
+        }
+
+        const AZStd::vector<TestRun>& TestRunReport::GetTimedOutTests() const
+        {
+            return m_timedOutTests;
+        }
+
+        const AZStd::vector<TestRun>& TestRunReport::GetUnexecutedTests() const
+        {
+            return m_unexecutedTests;
+        }
+
+        SequenceReport::SequenceReport(SuiteType suiteType, const TestRunSelection& selectedTests, TestRunReport&& selectedTestRunReport)
+            : m_suite(suiteType)
+            , m_selectedTests(selectedTests)
+            , m_selectedTestRunReport(AZStd::move(selectedTestRunReport))
+        {
+        }
+
+        TestSequenceResult SequenceReport::GetResult() const
+        {
+            return m_selectedTestRunReport.GetResult();
+        }
+
+        AZStd::chrono::high_resolution_clock::time_point SequenceReport::GetStartTime() const
+        {
+            return m_selectedTestRunReport.GetStartTime();
+        }
+
+        AZStd::chrono::high_resolution_clock::time_point SequenceReport::GetEndTime() const
+        {
+            return GetStartTime() + GetDuration();
+        }
+
+        AZStd::chrono::milliseconds SequenceReport::GetDuration() const
+        {
+            return m_selectedTestRunReport.GetDuration();
+        }
+
+        TestRunSelection SequenceReport::GetSelectedTests() const
+        {
+            return m_selectedTests;
+        }
+
+        TestRunReport SequenceReport::GetSelectedTestRunReport() const
+        {
+            return m_selectedTestRunReport;
+        }
+
+        size_t SequenceReport::GetTotalNumPassingTests() const
+        {
+            return m_selectedTestRunReport.GetNumPassingTests();
+        }
+
+        size_t SequenceReport::GetTotalNumFailingTests() const
+        {
+            return m_selectedTestRunReport.GetNumFailingTests();
+        }
+
+        size_t SequenceReport::GetTotalNumTimedOutTests() const
+        {
+            return m_selectedTestRunReport.GetNumTimedOutTests();
+        }
+
+        size_t SequenceReport::GetTotalNumUnexecutedTests() const
+        {
+            return m_selectedTestRunReport.GetNumUnexecutedTests();
+        }
+
+        DraftingSequenceReport::DraftingSequenceReport(
+            SuiteType suiteType,
+            const TestRunSelection& selectedTests,
+            const AZStd::vector<AZStd::string>& draftedTests,
+            TestRunReport&& selectedTestRunReport,
+            TestRunReport&& draftedTestRunReport)
+            : SequenceReport(suiteType, selectedTests, AZStd::move(selectedTestRunReport))
+            , m_draftedTests(draftedTests)
+            , m_draftedTestRunReport(AZStd::move(draftedTestRunReport))
+        {
+        }
+
+        TestSequenceResult DraftingSequenceReport::GetResult() const
+        {
+            return CalculateMultiTestSequenceResult({SequenceReport::GetResult(), m_draftedTestRunReport.GetResult()});
+        }
+
+        AZStd::chrono::milliseconds DraftingSequenceReport::GetDuration() const
+        {
+            return SequenceReport::GetDuration() + m_draftedTestRunReport.GetDuration();
+        }
+
+        size_t DraftingSequenceReport::GetTotalNumPassingTests() const
+        {
+            return SequenceReport::GetTotalNumPassingTests() + m_draftedTestRunReport.GetNumPassingTests();
+        }
+
+        size_t DraftingSequenceReport::GetTotalNumFailingTests() const
+        {
+            return SequenceReport::GetTotalNumFailingTests() + m_draftedTestRunReport.GetNumFailingTests();
+        }
+
+        size_t DraftingSequenceReport::GetTotalNumTimedOutTests() const
+        {
+            return SequenceReport::GetTotalNumTimedOutTests() + m_draftedTestRunReport.GetNumTimedOutTests();
+        }
+
+        size_t DraftingSequenceReport::GetTotalNumUnexecutedTests() const
+        {
+            return SequenceReport::GetTotalNumUnexecutedTests() + m_draftedTestRunReport.GetNumUnexecutedTests();
+        }
+
+        const AZStd::vector<AZStd::string>& DraftingSequenceReport::GetDraftedTests() const
+        {
+            return m_draftedTests;
+        }
+
+        TestRunReport DraftingSequenceReport::GetDraftedTestRunReport() const
+        {
+            return m_draftedTestRunReport;
+        }
+
+        ImpactAnalysisSequenceReport::ImpactAnalysisSequenceReport(
+            SuiteType suiteType,
+            const TestRunSelection& selectedTests,
+            const AZStd::vector<AZStd::string>& discardedTests,
+            const AZStd::vector<AZStd::string>& draftedTests,
+            TestRunReport&& selectedTestRunReport,
+            TestRunReport&& draftedTestRunReport)
+            : DraftingSequenceReport(
+                  suiteType,
+                  selectedTests,
+                  draftedTests,
+                  AZStd::move(selectedTestRunReport),
+                  AZStd::move(draftedTestRunReport))
+            , m_discardedTests(discardedTests)
+        {
+        }
+
+        const AZStd::vector<AZStd::string>& ImpactAnalysisSequenceReport::GetDiscardedTests() const
+        {
+            return m_discardedTests;
+        }
+
+        SafeImpactAnalysisSequenceReport::SafeImpactAnalysisSequenceReport(
+            SuiteType suiteType,
+            const TestRunSelection& selectedTests,
+            const TestRunSelection& discardedTests,
+            const AZStd::vector<AZStd::string>& draftedTests,
+            TestRunReport&& selectedTestRunReport,
+            TestRunReport&& discardedTestRunReport,
+            TestRunReport&& draftedTestRunReport)
+            : DraftingSequenceReport(
+                  suiteType,
+                  selectedTests,
+                  draftedTests,
+                  AZStd::move(selectedTestRunReport),
+                  AZStd::move(draftedTestRunReport))
+            , m_discardedTests(discardedTests)
+            , m_discardedTestRunReport(AZStd::move(discardedTestRunReport))
+        {
+        }
+
+        TestSequenceResult SafeImpactAnalysisSequenceReport::GetResult() const
+        {
+            return CalculateMultiTestSequenceResult({ DraftingSequenceReport::GetResult(), m_discardedTestRunReport.GetResult() });
+        }
+
+        AZStd::chrono::milliseconds SafeImpactAnalysisSequenceReport::GetDuration() const
+        {
+            return DraftingSequenceReport::GetDuration() + m_discardedTestRunReport.GetDuration();
+        }
+
+        size_t SafeImpactAnalysisSequenceReport::GetTotalNumPassingTests() const
+        {
+            return DraftingSequenceReport::GetTotalNumPassingTests() + m_discardedTestRunReport.GetNumPassingTests();
+        }
+
+        size_t SafeImpactAnalysisSequenceReport::GetTotalNumFailingTests() const
+        {
+            return DraftingSequenceReport::GetTotalNumFailingTests() + m_discardedTestRunReport.GetNumFailingTests();
+        }
+
+        size_t SafeImpactAnalysisSequenceReport::GetTotalNumTimedOutTests() const
+        {
+            return DraftingSequenceReport::GetTotalNumTimedOutTests() + m_discardedTestRunReport.GetNumTimedOutTests();
+        }
+
+        size_t SafeImpactAnalysisSequenceReport::GetTotalNumUnexecutedTests() const
+        {
+            return DraftingSequenceReport::GetTotalNumUnexecutedTests() + m_discardedTestRunReport.GetNumUnexecutedTests();
+        }
+        
+        const TestRunSelection SafeImpactAnalysisSequenceReport::GetDiscardedTests() const
+        {
+            return m_discardedTests;
+        }
+
+        TestRunReport SafeImpactAnalysisSequenceReport::GetDiscardedTestRunReport() const
+        {
+            return m_discardedTestRunReport;
+        }
+    } // namespace Client
+} // namespace TestImpact

Code/Tools/TestImpactFramework/Runtime/Code/Source/TestImpactClientTestRun.cpp

@@ -7,14 +7,22 @@
  */
 
 #include <TestImpactFramework/TestImpactClientTestRun.h>
+
 namespace TestImpact
 {
     namespace Client
     {
-        TestRun::TestRun(const AZStd::string& name, TestRunResult result, AZStd::chrono::milliseconds duration)
+        TestRun::TestRun(
+            const AZStd::string& name,
+            const AZStd::string& commandString,
+            AZStd::chrono::high_resolution_clock::time_point startTime,
+            AZStd::chrono::milliseconds duration,
+            TestRunResult result)
             : m_targetName(name)
-            , m_result(result)
+            , m_commandString(commandString)
+            , m_startTime(startTime)
             , m_duration(duration)
+            , m_result(result)
         {
         }
 
@@ -23,6 +31,21 @@ namespace TestImpact
             return m_targetName;
         }
 
+        const AZStd::string& TestRun::GetCommandString() const
+        {
+            return m_commandString;
+        }
+
+        AZStd::chrono::high_resolution_clock::time_point TestRun::GetStartTime() const
+        {
+            return m_startTime;
+        }
+
+        AZStd::chrono::high_resolution_clock::time_point TestRun::GetEndTime() const
+        {
+            return m_startTime + m_duration;
+        }
+
         AZStd::chrono::milliseconds TestRun::GetDuration() const
         {
             return m_duration;
@@ -32,5 +55,78 @@ namespace TestImpact
         {
             return m_result;
         }
+
+        TestFailure::TestFailure(const AZStd::string& testName, const AZStd::string& errorMessage)
+            : m_name(testName)
+            , m_errorMessage(errorMessage)
+        {
+        }
+
+        const AZStd::string& TestFailure::GetName() const
+        {
+            return m_name;
+        }
+
+        const AZStd::string& TestFailure::GetErrorMessage() const
+        {
+            return m_errorMessage;
+        }
+
+        TestCaseFailure::TestCaseFailure(const AZStd::string& testCaseName, AZStd::vector<TestFailure>&& testFailures)
+            : m_name(testCaseName)
+            , m_testFailures(AZStd::move(testFailures))
+        {
+        }
+
+        const AZStd::string& TestCaseFailure::GetName() const
+        {
+            return m_name;
+        }
+
+        const AZStd::vector<TestFailure>& TestCaseFailure::GetTestFailures() const
+        {
+            return m_testFailures;
+        }
+
+        static size_t CalculateNumTestRunFailures(const AZStd::vector<TestCaseFailure>& testFailures)
+        {
+            size_t numTestFailures = 0;
+            for (const auto& testCase : testFailures)
+            {
+                numTestFailures += testCase.GetTestFailures().size();
+            }
+
+            return numTestFailures;
+        }
+
+        TestRunWithTestFailures::TestRunWithTestFailures(
+            const AZStd::string& name,
+            const AZStd::string& commandString,
+            AZStd::chrono::high_resolution_clock::time_point startTime,
+            AZStd::chrono::milliseconds duration,
+            TestRunResult result,
+            AZStd::vector<TestCaseFailure>&& testFailures)
+            : TestRun(name, commandString, startTime, duration, result)
+            , m_testCaseFailures(AZStd::move(testFailures))
+        {
+            m_numTestFailures = CalculateNumTestRunFailures(m_testCaseFailures);
+        }
+
+        TestRunWithTestFailures::TestRunWithTestFailures(TestRun&& testRun, AZStd::vector<TestCaseFailure>&& testFailures)
+            : TestRun(AZStd::move(testRun))
+            , m_testCaseFailures(AZStd::move(testFailures))
+        {
+            m_numTestFailures = CalculateNumTestRunFailures(m_testCaseFailures);
+        }
+
+        size_t TestRunWithTestFailures::GetNumTestFailures() const
+        {
+            return m_numTestFailures;
+        }
+
+        const AZStd::vector<TestCaseFailure>& TestRunWithTestFailures::GetTestCaseFailures() const
+        {
+            return m_testCaseFailures;
+        }
     } // namespace Client
 } // namespace TestImpact

Code/Tools/TestImpactFramework/Runtime/Code/Source/TestImpactRuntime.cpp

@@ -34,13 +34,33 @@ namespace TestImpact
             {
             }
 
-            //! Returns the time elapsed (in milliseconds) since the timer was instantiated
-            AZStd::chrono::milliseconds Elapsed()
+            //! Returns the time point that the timer was instantiated.
+            AZStd::chrono::high_resolution_clock::time_point GetStartTimePoint() const
+            {
+                return m_startTime;
+            }
+
+            //! Returns the time point that the timer was instantiated relative to the specified starting time point.
+            AZStd::chrono::high_resolution_clock::time_point GetStartTimePointRelative(const Timer& start) const
+            {
+                return AZStd::chrono::high_resolution_clock::time_point() +
+                    AZStd::chrono::duration_cast<AZStd::chrono::milliseconds>(m_startTime - start.GetStartTimePoint());
+            }
+
+            //! Returns the time elapsed (in milliseconds) since the timer was instantiated.
+            AZStd::chrono::milliseconds GetElapsedMs() const
             {
                 const auto endTime = AZStd::chrono::high_resolution_clock::now();
                 return AZStd::chrono::duration_cast<AZStd::chrono::milliseconds>(endTime - m_startTime);
             }
 
+            //! Returns the current time point relative to the time point the timer was instantiated.
+            AZStd::chrono::high_resolution_clock::time_point GetElapsedTimepoint() const
+            {
+                const auto endTime = AZStd::chrono::high_resolution_clock::now();
+                return m_startTime + AZStd::chrono::duration_cast<AZStd::chrono::milliseconds>(endTime - m_startTime);
+            }
+
         private:
             AZStd::chrono::high_resolution_clock::time_point m_startTime;
         };
@@ -49,8 +69,11 @@ namespace TestImpact
         class TestRunCompleteCallbackHandler
         {
         public:
-            TestRunCompleteCallbackHandler(AZStd::optional<TestRunCompleteCallback> testCompleteCallback)
-                : m_testCompleteCallback(testCompleteCallback)
+            TestRunCompleteCallbackHandler(
+                size_t totalTests,
+                AZStd::optional<TestRunCompleteCallback> testCompleteCallback)
+                : m_totalTests(totalTests)
+                , m_testCompleteCallback(testCompleteCallback)
             {
             }
 
@@ -58,19 +81,27 @@ namespace TestImpact
             {
                 if (m_testCompleteCallback.has_value())
                 {
-                    (*m_testCompleteCallback)
-                        (Client::TestRun(testJob.GetTestTarget()->GetName(), testJob.GetTestResult(), testJob.GetDuration()));
+                    Client::TestRun testRun(
+                        testJob.GetTestTarget()->GetName(),
+                        testJob.GetCommandString(),
+                        testJob.GetStartTime(),
+                        testJob.GetDuration(),
+                        testJob.GetTestResult());
+
+                    (*m_testCompleteCallback)(testRun, ++m_numTestsCompleted, m_totalTests);
                 }
             }
 
         private:
+            const size_t m_totalTests; //!< The total number of tests to run for the entire sequence.
+            size_t m_numTestsCompleted = 0; //!< The running total of tests that have completed.
             AZStd::optional<TestRunCompleteCallback> m_testCompleteCallback;
         };
     }
 
     //! Utility for concatenating two vectors.
     template<typename T>
-    AZStd::vector<T> ConcatenateVectors(const AZStd::vector<T>& v1, const AZStd::vector<T>& v2)
+    static AZStd::vector<T> ConcatenateVectors(const AZStd::vector<T>& v1, const AZStd::vector<T>& v2)
     {
         AZStd::vector<T> result;
         result.reserve(v1.size() + v2.size());
@@ -298,6 +329,7 @@ namespace TestImpact
                     continue;
                 }
 
+                // Add the sources covered by this test target to the coverage map
                 for (const auto& source : job.GetTestCoverge().value().GetSourcesCovered())
                 {
                     coverage[source.String()].insert(job.GetTestTarget()->GetName());
@@ -309,6 +341,7 @@ namespace TestImpact
         sourceCoveringTests.reserve(coverage.size());
         for (auto&& [source, testTargets] : coverage)
         {
+            // Check to see whether this source is inside the repo or not (not a perfect check but weeds out the obvious non-repo sources)
             if (const auto sourcePath = RepoPath(source);
                 sourcePath.IsRelativeTo(m_config.m_repo.m_root))
             {
@@ -353,17 +386,17 @@ namespace TestImpact
         }
     }
 
-    TestSequenceResult Runtime::RegularTestSequence(
+    Client::SequenceReport Runtime::RegularTestSequence(
         AZStd::optional<AZStd::chrono::milliseconds> testTargetTimeout,
         AZStd::optional<AZStd::chrono::milliseconds> globalTimeout,
         AZStd::optional<TestSequenceStartCallback> testSequenceStartCallback,
-        AZStd::optional<TestSequenceCompleteCallback> testSequenceEndCallback,
+        AZStd::optional<TestSequenceCompleteCallback<Client::SequenceReport>> testSequenceEndCallback,
         AZStd::optional<TestRunCompleteCallback> testCompleteCallback)
     {
-        Timer timer;
+        const Timer sequenceTimer;
         AZStd::vector<const TestTarget*> includedTestTargets;
         AZStd::vector<const TestTarget*> excludedTestTargets;
-
+        
         // Separate the test targets into those that are excluded by either the test filter or exclusion list and those that are not
         for (const auto& testTarget : m_dynamicDependencyMap->GetTestTargetList().GetTargets())
         {
@@ -378,12 +411,17 @@ namespace TestImpact
             }
         }
 
-        // Sequence start callback
+        // Extract the client facing representation of selected test targets
+        const Client::TestRunSelection selectedTests(ExtractTestTargetNames(includedTestTargets), ExtractTestTargetNames(excludedTestTargets));
+
+        // Inform the client that the sequence is about to start
         if (testSequenceStartCallback.has_value())
         {
-            (*testSequenceStartCallback)(Client::TestRunSelection(ExtractTestTargetNames(includedTestTargets), ExtractTestTargetNames(excludedTestTargets)));
+            (*testSequenceStartCallback)(m_suiteFilter, selectedTests);
         }
 
+        // Run the test targets and collect the test run results
+        const Timer testRunTimer;
         const auto [result, testJobs] = m_testEngine->RegularRun(
             includedTestTargets,
             m_testShardingPolicy,
@@ -392,27 +430,124 @@ namespace TestImpact
             m_targetOutputCapture,
             testTargetTimeout,
             globalTimeout,
-            TestRunCompleteCallbackHandler(testCompleteCallback));
+            TestRunCompleteCallbackHandler(includedTestTargets.size(), testCompleteCallback));
+        const auto testRunDuration = testRunTimer.GetElapsedMs();
 
+        // Generate the sequence report for the client
+        const auto sequenceReport = Client::SequenceReport(
+            m_suiteFilter,
+            selectedTests,
+            GenerateTestRunReport(result, testRunTimer.GetStartTimePointRelative(sequenceTimer), testRunDuration, testJobs));
+
+        // Inform the client that the sequence has ended
         if (testSequenceEndCallback.has_value())
         {
-            (*testSequenceEndCallback)(GenerateSequenceFailureReport(testJobs), timer.Elapsed());
+            (*testSequenceEndCallback)(sequenceReport);
         }
 
-        return result;
+        return sequenceReport;
     }
 
-    TestSequenceResult Runtime::ImpactAnalysisTestSequence(
+    //! Wrapper for the impact analysis test sequence to handle both the updating and non-updating policies through a common pathway.
+    //! @tparam TestRunnerFunctor The functor for running the specified tests.
+    //! @tparam TestJob The test engine job type returned by the functor.
+    //! @param suiteType The suite type used for this sequence.
+    //! @param timer The timer to use for the test run timings.
+    //! @param testRunner The test runner functor to use for each of the test runs.
+    //! @param includedSelectedTestTargets The subset of test targets that were selected to run and not also fully excluded from running.
+    //! @param excludedSelectedTestTargets The subset of test targets that were selected to run but were fully excluded running.
+    //! @param discardedTestTargets The subset of test targets that were discarded from the test selection and will not be run.
+    //! @param globalTimeout The maximum duration the entire test sequence may run for (infinite if empty).
+    //! @param testSequenceStartCallback The client function to be called after the test targets have been selected but prior to running the tests.
+    //! @param testSequenceCompleteCallback The client function to be called after the test sequence has completed.
+    //! @param testRunCompleteCallback The client function to be called after an individual test run has completed.
+    //! @param updateCoverage The function to call to update the dynamic dependency map with test coverage (if any).
+    template<typename TestRunnerFunctor, typename TestJob>
+    Client::ImpactAnalysisSequenceReport ImpactAnalysisTestSequenceWrapper(
+        SuiteType suiteType,
+        const Timer& sequenceTimer,
+        const TestRunnerFunctor& testRunner,
+        const AZStd::vector<const TestTarget*>& includedSelectedTestTargets,
+        const AZStd::vector<const TestTarget*>& excludedSelectedTestTargets,
+        const AZStd::vector<const TestTarget*>& discardedTestTargets,
+        const AZStd::vector<const TestTarget*>& draftedTestTargets,
+        const AZStd::optional<AZStd::chrono::milliseconds> globalTimeout,
+        AZStd::optional<ImpactAnalysisTestSequenceStartCallback> testSequenceStartCallback,
+        AZStd::optional<TestSequenceCompleteCallback<Client::ImpactAnalysisSequenceReport>> testSequenceEndCallback,
+        AZStd::optional<TestRunCompleteCallback> testCompleteCallback,
+        AZStd::optional<AZStd::function<void(const AZStd::vector<TestJob>& jobs)>> updateCoverage)
+    {
+        AZStd::optional<AZStd::chrono::milliseconds> sequenceTimeout = globalTimeout;
+
+        // Extract the client facing representation of selected, discarded and drafted test targets
+        const Client::TestRunSelection selectedTests(
+            ExtractTestTargetNames(includedSelectedTestTargets), ExtractTestTargetNames(excludedSelectedTestTargets));
+        const auto discardedTests = ExtractTestTargetNames(discardedTestTargets);
+        const auto draftedTests = ExtractTestTargetNames(draftedTestTargets);
+
+        // Inform the client that the sequence is about to start
+        if (testSequenceStartCallback.has_value())
+        {
+            (*testSequenceStartCallback)(suiteType, selectedTests, discardedTests, draftedTests);
+        }
+
+        // We share the test run complete handler between the selected and drafted test runs as to present them together as one
+        // continuous test sequence to the client rather than two discrete test runs
+        const size_t totalNumTestRuns = includedSelectedTestTargets.size() + draftedTestTargets.size();
+        TestRunCompleteCallbackHandler testRunCompleteHandler(totalNumTestRuns, testCompleteCallback);
+
+        // Run the selected test targets and collect the test run results
+        const Timer selectedTestRunTimer;
+        const auto [selectedResult, selectedTestJobs] = testRunner(includedSelectedTestTargets, testRunCompleteHandler, globalTimeout);
+        const auto selectedTestRunDuration = selectedTestRunTimer.GetElapsedMs();
+
+        // Carry the remaining global sequence time over to the drafted test run
+        if (globalTimeout.has_value())
+        {
+            const auto elapsed = selectedTestRunDuration;
+            sequenceTimeout = elapsed < globalTimeout.value() ? globalTimeout.value() - elapsed : AZStd::chrono::milliseconds(0);
+        }
+
+        // Run the drafted test targets and collect the test run results
+        Timer draftedTestRunTimer;
+        const auto [draftedResult, draftedTestJobs] = testRunner(draftedTestTargets, testRunCompleteHandler, globalTimeout);
+        const auto draftedTestRunDuration = draftedTestRunTimer.GetElapsedMs();
+
+        // Generate the sequence report for the client
+        const auto sequenceReport = Client::ImpactAnalysisSequenceReport(
+            suiteType,
+            selectedTests,
+            discardedTests,
+            draftedTests,
+            GenerateTestRunReport(selectedResult, selectedTestRunTimer.GetStartTimePointRelative(sequenceTimer), selectedTestRunDuration, selectedTestJobs),
+            GenerateTestRunReport(draftedResult, draftedTestRunTimer.GetStartTimePointRelative(sequenceTimer), draftedTestRunDuration, draftedTestJobs));
+
+        // Inform the client that the sequence has ended
+        if (testSequenceEndCallback.has_value())
+        {
+            (*testSequenceEndCallback)(sequenceReport);
+        }
+
+        // Update the dynamic dependency map with the latest coverage data (if any)
+        if (updateCoverage.has_value())
+        {
+            (*updateCoverage)(ConcatenateVectors(selectedTestJobs, draftedTestJobs));
+        }
+
+        return sequenceReport;
+    }
+
+    Client::ImpactAnalysisSequenceReport Runtime::ImpactAnalysisTestSequence(
         const ChangeList& changeList,
         Policy::TestPrioritization testPrioritizationPolicy,
         Policy::DynamicDependencyMap dynamicDependencyMapPolicy,
         AZStd::optional<AZStd::chrono::milliseconds> testTargetTimeout,
         AZStd::optional<AZStd::chrono::milliseconds> globalTimeout,
         AZStd::optional<ImpactAnalysisTestSequenceStartCallback> testSequenceStartCallback,
-        AZStd::optional<TestSequenceCompleteCallback> testSequenceEndCallback,
+        AZStd::optional<TestSequenceCompleteCallback<Client::ImpactAnalysisSequenceReport>> testSequenceEndCallback,
         AZStd::optional<TestRunCompleteCallback> testCompleteCallback)
     {
-        Timer timer;
+        const Timer sequenceTimer;
 
         // Draft in the test targets that have no coverage entries in the dynamic dependency map
         AZStd::vector<const TestTarget*> draftedTestTargets = m_dynamicDependencyMap->GetNotCoveringTests();
@@ -423,71 +558,94 @@ namespace TestImpact
         // The subset of selected test targets that are not on the configuration's exclude list and those that are
         auto [includedSelectedTestTargets, excludedSelectedTestTargets] = SelectTestTargetsByExcludeList(selectedTestTargets);
 
-        // We present to the client the included selected test targets and the drafted test targets as distinct sets but internally
-        // we consider the concatenated set of the two the actual set of tests to run
-        AZStd::vector<const TestTarget*> testTargetsToRun = ConcatenateVectors(includedSelectedTestTargets, draftedTestTargets);
-
-        if (testSequenceStartCallback.has_value())
+        // Functor for running instrumented test targets
+        const auto instrumentedTestRun =
+            [this, &testTargetTimeout](
+                const AZStd::vector<const TestTarget*>& testsTargets,
+                TestRunCompleteCallbackHandler& testRunCompleteHandler,
+                AZStd::optional<AZStd::chrono::milliseconds> globalTimeout)
         {
-            (*testSequenceStartCallback)(
-                Client::TestRunSelection(ExtractTestTargetNames(includedSelectedTestTargets), ExtractTestTargetNames(excludedSelectedTestTargets)),
-                ExtractTestTargetNames(discardedTestTargets),
-                ExtractTestTargetNames(draftedTestTargets));
-        }
+            return m_testEngine->InstrumentedRun(
+                testsTargets,
+                m_testShardingPolicy,
+                m_executionFailurePolicy,
+                m_integrationFailurePolicy,
+                m_testFailurePolicy,
+                m_targetOutputCapture,
+                testTargetTimeout,
+                globalTimeout,
+                AZStd::ref(testRunCompleteHandler));
+        };
+
+        // Functor for running uninstrumented test targets
+        const auto regularTestRun =
+            [this, &testTargetTimeout](
+                const AZStd::vector<const TestTarget*>& testsTargets,
+                 TestRunCompleteCallbackHandler& testRunCompleteHandler,
+                AZStd::optional<AZStd::chrono::milliseconds> globalTimeout)
+        {
+            return m_testEngine->RegularRun(
+                testsTargets,
+                m_testShardingPolicy,
+                m_executionFailurePolicy,
+                m_testFailurePolicy,
+                m_targetOutputCapture,
+                testTargetTimeout,
+                globalTimeout,
+                AZStd::ref(testRunCompleteHandler));
+        };
 
         if (dynamicDependencyMapPolicy == Policy::DynamicDependencyMap::Update)
         {
-            const auto [result, testJobs] = m_testEngine->InstrumentedRun(
-                testTargetsToRun,
-                m_testShardingPolicy,
-                m_executionFailurePolicy,
-                Policy::IntegrityFailure::Continue,
-                m_testFailurePolicy,
-                m_targetOutputCapture,
-                testTargetTimeout,
-                globalTimeout,
-                TestRunCompleteCallbackHandler(testCompleteCallback));
-
-            UpdateAndSerializeDynamicDependencyMap(testJobs);
-
-            if (testSequenceEndCallback.has_value())
+            AZStd::optional<AZStd::function<void(const AZStd::vector<TestEngineInstrumentedRun>& jobs)>> updateCoverage =
+                [this](const AZStd::vector<TestEngineInstrumentedRun>& jobs)
             {
-                (*testSequenceEndCallback)(GenerateSequenceFailureReport(testJobs), timer.Elapsed());
-            }
+                UpdateAndSerializeDynamicDependencyMap(jobs);
+            };
 
-            return result;
+            return ImpactAnalysisTestSequenceWrapper(
+                m_suiteFilter,
+                sequenceTimer,
+                instrumentedTestRun,
+                includedSelectedTestTargets,
+                excludedSelectedTestTargets,
+                discardedTestTargets,
+                draftedTestTargets,
+                globalTimeout,
+                testSequenceStartCallback,
+                testSequenceEndCallback,
+                testCompleteCallback,
+                updateCoverage);
         }
         else
         {
-            const auto [result, testJobs] = m_testEngine->RegularRun(
-                testTargetsToRun,
-                m_testShardingPolicy,
-                m_executionFailurePolicy,
-                m_testFailurePolicy,
-                m_targetOutputCapture,
-                testTargetTimeout,
+            return ImpactAnalysisTestSequenceWrapper(
+                m_suiteFilter,
+                sequenceTimer,
+                regularTestRun,
+                includedSelectedTestTargets,
+                excludedSelectedTestTargets,
+                discardedTestTargets,
+                draftedTestTargets,
                 globalTimeout,
-                TestRunCompleteCallbackHandler(testCompleteCallback));
-
-            if (testSequenceEndCallback.has_value())
-            {
-                (*testSequenceEndCallback)(GenerateSequenceFailureReport(testJobs), timer.Elapsed());
-            }
-
-            return result;
+                testSequenceStartCallback,
+                testSequenceEndCallback,
+                testCompleteCallback,
+                AZStd::optional<AZStd::function<void(const AZStd::vector<TestEngineRegularRun>& jobs)>>{ AZStd::nullopt });
         }
     }
 
-    AZStd::pair<TestSequenceResult, TestSequenceResult> Runtime::SafeImpactAnalysisTestSequence(
+    Client::SafeImpactAnalysisSequenceReport Runtime::SafeImpactAnalysisTestSequence(
         const ChangeList& changeList,
         Policy::TestPrioritization testPrioritizationPolicy,
         AZStd::optional<AZStd::chrono::milliseconds> testTargetTimeout,
         AZStd::optional<AZStd::chrono::milliseconds> globalTimeout,
         AZStd::optional<SafeImpactAnalysisTestSequenceStartCallback> testSequenceStartCallback,
-        AZStd::optional<SafeTestSequenceCompleteCallback> testSequenceEndCallback,
+        AZStd::optional<TestSequenceCompleteCallback<Client::SafeImpactAnalysisSequenceReport>> testSequenceEndCallback,
         AZStd::optional<TestRunCompleteCallback> testCompleteCallback)
     {
-        Timer timer;
+        const Timer sequenceTimer;
+        auto sequenceTimeout = globalTimeout;
 
         // Draft in the test targets that have no coverage entries in the dynamic dependency map
         AZStd::vector<const TestTarget*> draftedTestTargets = m_dynamicDependencyMap->GetNotCoveringTests();
@@ -501,40 +659,46 @@ namespace TestImpact
         // The subset of discarded test targets that are not on the configuration's exclude list and those that are
         auto [includedDiscardedTestTargets, excludedDiscardedTestTargets] = SelectTestTargetsByExcludeList(discardedTestTargets);
 
-        // We present to the client the included selected test targets and the drafted test targets as distinct sets but internally
-        // we consider the concatenated set of the two the actual set of tests to run
-        AZStd::vector<const TestTarget*> testTargetsToRun = ConcatenateVectors(includedSelectedTestTargets, draftedTestTargets);
+        // Extract the client facing representation of selected, discarded and drafted test targets
+        const Client::TestRunSelection selectedTests(
+            ExtractTestTargetNames(includedSelectedTestTargets), ExtractTestTargetNames(excludedSelectedTestTargets));
+        const Client::TestRunSelection discardedTests(ExtractTestTargetNames(includedDiscardedTestTargets), ExtractTestTargetNames(excludedDiscardedTestTargets));
+            const auto draftedTests = ExtractTestTargetNames(draftedTestTargets);
 
+        // Inform the client that the sequence is about to start
         if (testSequenceStartCallback.has_value())
         {
-            (*testSequenceStartCallback)(
-                Client::TestRunSelection(ExtractTestTargetNames(includedSelectedTestTargets), ExtractTestTargetNames(excludedSelectedTestTargets)),
-                Client::TestRunSelection(ExtractTestTargetNames(includedDiscardedTestTargets), ExtractTestTargetNames(excludedDiscardedTestTargets)),
-                ExtractTestTargetNames(draftedTestTargets));
+            (*testSequenceStartCallback)(m_suiteFilter, selectedTests, discardedTests, draftedTests);
         }
 
-        // Impact analysis run of the selected test targets
+        // We share the test run complete handler between the selected, discarded and drafted test runs as to present them together as one
+        // continuous test sequence to the client rather than three discrete test runs
+        const size_t totalNumTestRuns = includedSelectedTestTargets.size() + draftedTestTargets.size() + includedDiscardedTestTargets.size();
+        TestRunCompleteCallbackHandler testRunCompleteHandler(totalNumTestRuns, testCompleteCallback);
+
+        // Run the selected test targets and collect the test run results
+        const Timer selectedTestRunTimer;
         const auto [selectedResult, selectedTestJobs] = m_testEngine->InstrumentedRun(
-            testTargetsToRun,
+            includedSelectedTestTargets,
             m_testShardingPolicy,
             m_executionFailurePolicy,
-            Policy::IntegrityFailure::Continue,
+            m_integrationFailurePolicy,
             m_testFailurePolicy,
             m_targetOutputCapture,
             testTargetTimeout,
-            globalTimeout,
-            TestRunCompleteCallbackHandler(testCompleteCallback));
-        
-        const auto selectedDuraton = timer.Elapsed();
+            sequenceTimeout,
+            AZStd::ref(testRunCompleteHandler));
+        const auto selectedTestRunDuration = selectedTestRunTimer.GetElapsedMs();
 
         // Carry the remaining global sequence time over to the discarded test run
         if (globalTimeout.has_value())
         {
-            const auto elapsed = timer.Elapsed();
-            globalTimeout = elapsed < globalTimeout.value() ? globalTimeout.value() - elapsed : AZStd::chrono::milliseconds(0);
+            const auto elapsed = selectedTestRunDuration;
+            sequenceTimeout = elapsed < globalTimeout.value() ? globalTimeout.value() - elapsed : AZStd::chrono::milliseconds(0);
         }
 
-        // Regular run of the discarded test targets
+        // Run the discarded test targets and collect the test run results
+        const Timer discardedTestRunTimer;
         const auto [discardedResult, discardedTestJobs] = m_testEngine->RegularRun(
             includedDiscardedTestTargets,
             m_testShardingPolicy,
@@ -542,35 +706,63 @@ namespace TestImpact
             m_testFailurePolicy,
             m_targetOutputCapture,
             testTargetTimeout,
-            globalTimeout,
-            TestRunCompleteCallbackHandler(testCompleteCallback));
+            sequenceTimeout,
+            AZStd::ref(testRunCompleteHandler));
+        const auto discardedTestRunDuration = discardedTestRunTimer.GetElapsedMs();
 
-        const auto discardedDuraton = timer.Elapsed();
-
-        if (testSequenceEndCallback.has_value())
+        // Carry the remaining global sequence time over to the drafted test run
+        if (globalTimeout.has_value())
         {
-            (*testSequenceEndCallback)(
-                GenerateSequenceFailureReport(selectedTestJobs),
-                GenerateSequenceFailureReport(discardedTestJobs),
-                selectedDuraton,
-                discardedDuraton);
+            const auto elapsed = selectedTestRunDuration + discardedTestRunDuration;
+            sequenceTimeout = elapsed < globalTimeout.value() ? globalTimeout.value() - elapsed : AZStd::chrono::milliseconds(0);
         }
 
-        UpdateAndSerializeDynamicDependencyMap(selectedTestJobs);
-        return { selectedResult, discardedResult };
+        // Run the drafted test targets and collect the test run results
+        const Timer draftedTestRunTimer;
+        const auto [draftedResult, draftedTestJobs] = m_testEngine->InstrumentedRun(
+            draftedTestTargets,
+            m_testShardingPolicy,
+            m_executionFailurePolicy,
+            m_integrationFailurePolicy,
+            m_testFailurePolicy,
+            m_targetOutputCapture,
+            testTargetTimeout,
+            sequenceTimeout,
+            AZStd::ref(testRunCompleteHandler));
+        const auto draftedTestRunDuration = draftedTestRunTimer.GetElapsedMs();
+
+        // Generate the sequence report for the client
+        const auto sequenceReport = Client::SafeImpactAnalysisSequenceReport(
+            m_suiteFilter,
+            selectedTests,
+            discardedTests,
+            draftedTests,
+            GenerateTestRunReport(selectedResult, selectedTestRunTimer.GetStartTimePointRelative(sequenceTimer), selectedTestRunDuration, selectedTestJobs),
+            GenerateTestRunReport(discardedResult, discardedTestRunTimer.GetStartTimePointRelative(sequenceTimer), discardedTestRunDuration, discardedTestJobs),
+            GenerateTestRunReport(draftedResult, draftedTestRunTimer.GetStartTimePointRelative(sequenceTimer), draftedTestRunDuration, draftedTestJobs));
+
+        // Inform the client that the sequence has ended
+        if (testSequenceEndCallback.has_value())
+        {
+            (*testSequenceEndCallback)(sequenceReport);
+        }
+
+        UpdateAndSerializeDynamicDependencyMap(ConcatenateVectors(selectedTestJobs, draftedTestJobs));
+        return sequenceReport;
     }
 
-    TestSequenceResult Runtime::SeededTestSequence(
+    Client::SequenceReport Runtime::SeededTestSequence(
         AZStd::optional<AZStd::chrono::milliseconds> testTargetTimeout,
         AZStd::optional<AZStd::chrono::milliseconds> globalTimeout,
         AZStd::optional<TestSequenceStartCallback> testSequenceStartCallback,
-        AZStd::optional<TestSequenceCompleteCallback> testSequenceEndCallback,
+        AZStd::optional<TestSequenceCompleteCallback<Client::SequenceReport>> testSequenceEndCallback,
         AZStd::optional<TestRunCompleteCallback> testCompleteCallback)
     {
-        Timer timer;
+        const Timer sequenceTimer;
         AZStd::vector<const TestTarget*> includedTestTargets;
         AZStd::vector<const TestTarget*> excludedTestTargets;
 
+        // Separate the test targets into those that are excluded by either the test filter or exclusion list and those that are not
         for (const auto& testTarget : m_dynamicDependencyMap->GetTestTargetList().GetTargets())
         {
             if (!m_testTargetExcludeList.contains(&testTarget))
@@ -583,31 +775,44 @@ namespace TestImpact
             }
         }
 
+        // Extract the client facing representation of selected test targets
+        Client::TestRunSelection selectedTests(ExtractTestTargetNames(includedTestTargets), ExtractTestTargetNames(excludedTestTargets));
+
+        // Inform the client that the sequence is about to start
         if (testSequenceStartCallback.has_value())
         {
-            (*testSequenceStartCallback)(Client::TestRunSelection(ExtractTestTargetNames(includedTestTargets), ExtractTestTargetNames(excludedTestTargets)));
+            (*testSequenceStartCallback)(m_suiteFilter, selectedTests);
         }
 
+        // Run the test targets and collect the test run results
+        const Timer testRunTimer;
         const auto [result, testJobs] = m_testEngine->InstrumentedRun(
             includedTestTargets,
             m_testShardingPolicy,
             m_executionFailurePolicy,
-            Policy::IntegrityFailure::Continue,
+            m_integrationFailurePolicy,
             m_testFailurePolicy,
             m_targetOutputCapture,
             testTargetTimeout,
             globalTimeout,
-            TestRunCompleteCallbackHandler(testCompleteCallback));
+            TestRunCompleteCallbackHandler(includedTestTargets.size(), testCompleteCallback));
+        const auto testRunDuration = testRunTimer.GetElapsedMs();
 
+        // Generate the sequence report for the client
+        const auto sequenceReport = Client::SequenceReport(
+            m_suiteFilter,
+            selectedTests,
+            GenerateTestRunReport(result, testRunTimer.GetStartTimePointRelative(sequenceTimer), testRunDuration, testJobs));
+
+        // Inform the client that the sequence has ended
         if (testSequenceEndCallback.has_value())
         {
-            (*testSequenceEndCallback)(GenerateSequenceFailureReport(testJobs), timer.Elapsed());
+            (*testSequenceEndCallback)(sequenceReport);
         }
 
         ClearDynamicDependencyMapAndRemoveExistingFile();
         UpdateAndSerializeDynamicDependencyMap(testJobs);
-
-        return result;
+        return sequenceReport;
     }
 
     bool Runtime::HasImpactAnalysisData() const

Code/Tools/TestImpactFramework/Runtime/Code/Source/TestImpactRuntimeUtils.cpp

@@ -24,13 +24,13 @@ namespace TestImpact
         return TestTargetMetaMapFactory(masterTestListData, suiteFilter);
     }
 
-    AZStd::vector<TestImpact::BuildTargetDescriptor> ReadBuildTargetDescriptorFiles(const BuildTargetDescriptorConfig& buildTargetDescriptorConfig)
+    AZStd::vector<BuildTargetDescriptor> ReadBuildTargetDescriptorFiles(const BuildTargetDescriptorConfig& buildTargetDescriptorConfig)
     {
-        AZStd::vector<TestImpact::BuildTargetDescriptor> buildTargetDescriptors;
+        AZStd::vector<BuildTargetDescriptor> buildTargetDescriptors;
         for (const auto& buildTargetDescriptorFile : std::filesystem::directory_iterator(buildTargetDescriptorConfig.m_mappingDirectory.c_str()))
         {
             const auto buildTargetDescriptorContents = ReadFileContents<RuntimeException>(buildTargetDescriptorFile.path().string().c_str());
-            auto buildTargetDescriptor = TestImpact::BuildTargetDescriptorFactory(
+            auto buildTargetDescriptor = BuildTargetDescriptorFactory(
                 buildTargetDescriptorContents,
                 buildTargetDescriptorConfig.m_staticInclusionFilters,
                 buildTargetDescriptorConfig.m_inputInclusionFilters,
@@ -41,7 +41,7 @@ namespace TestImpact
         return buildTargetDescriptors;
     }
 
-    AZStd::unique_ptr<TestImpact::DynamicDependencyMap> ConstructDynamicDependencyMap(
+    AZStd::unique_ptr<DynamicDependencyMap> ConstructDynamicDependencyMap(
         SuiteType suiteFilter,
         const BuildTargetDescriptorConfig& buildTargetDescriptorConfig,
         const TestTargetMetaConfig& testTargetMetaConfig)
@@ -50,7 +50,7 @@ namespace TestImpact
         auto buildTargetDescriptors = ReadBuildTargetDescriptorFiles(buildTargetDescriptorConfig);
         auto buildTargets = CompileTargetDescriptors(AZStd::move(buildTargetDescriptors), AZStd::move(testTargetmetaMap));
         auto&& [productionTargets, testTargets] = buildTargets;
-        return AZStd::make_unique<TestImpact::DynamicDependencyMap>(AZStd::move(productionTargets), AZStd::move(testTargets));
+        return AZStd::make_unique<DynamicDependencyMap>(AZStd::move(productionTargets), AZStd::move(testTargets));
     }
 
     AZStd::unordered_set<const TestTarget*> ConstructTestTargetExcludeList(
@@ -68,7 +68,7 @@ namespace TestImpact
         return testTargetExcludeList;
     }
 
-    AZStd::vector<AZStd::string> ExtractTestTargetNames(const AZStd::vector<const TestTarget*> testTargets)
+    AZStd::vector<AZStd::string> ExtractTestTargetNames(const AZStd::vector<const TestTarget*>& testTargets)
     {
         AZStd::vector<AZStd::string> testNames;
         AZStd::transform(testTargets.begin(), testTargets.end(), AZStd::back_inserter(testNames), [](const TestTarget* testTarget)

Code/Tools/TestImpactFramework/Runtime/Code/Source/TestImpactRuntimeUtils.h

@@ -10,7 +10,7 @@
 
 #include <TestImpactFramework/TestImpactConfiguration.h>
 #include <TestImpactFramework/TestImpactClientTestSelection.h>
-#include <TestImpactFramework/TestImpactClientFailureReport.h>
+#include <TestImpactFramework/TestImpactClientSequenceReport.h>
 
 #include <Artifact/Static/TestImpactTestTargetMeta.h>
 #include <Artifact/Static/TestImpactBuildTargetDescriptor.h>
@@ -25,7 +25,7 @@
 namespace TestImpact
 {
     //! Construct a dynamic dependency map from the build target descriptors and test target metas.
-    AZStd::unique_ptr<TestImpact::DynamicDependencyMap> ConstructDynamicDependencyMap(
+    AZStd::unique_ptr<DynamicDependencyMap> ConstructDynamicDependencyMap(
         SuiteType suiteFilter,
         const BuildTargetDescriptorConfig& buildTargetDescriptorConfig,
         const TestTargetMetaConfig& testTargetMetaConfig);
@@ -36,16 +36,17 @@ namespace TestImpact
         const AZStd::vector<AZStd::string>& excludedTestTargets);
 
     //! Extracts the name information from the specified test targets.
-    AZStd::vector<AZStd::string> ExtractTestTargetNames(const AZStd::vector<const TestTarget*> testTargets);    
+    AZStd::vector<AZStd::string> ExtractTestTargetNames(const AZStd::vector<const TestTarget*>& testTargets);
 
     //! Generates a test run failure report from the specified test engine job information.
     //! @tparam TestJob The test engine job type.
     template<typename TestJob>
-    Client::TestRunFailure GenerateTestRunFailure(const TestJob& testJob)
+    AZStd::vector<Client::TestCaseFailure> GenerateTestCaseFailures(const TestJob& testJob)
     {
+        AZStd::vector<Client::TestCaseFailure> testCaseFailures;
+
         if (testJob.GetTestRun().has_value())
         {
-            AZStd::vector<Client::TestCaseFailure> testCaseFailures;
             for (const auto& testSuite : testJob.GetTestRun()->GetTestSuites())
             {
                 AZStd::vector<Client::TestFailure> testFailures;
@@ -56,57 +57,66 @@ namespace TestImpact
                         testFailures.push_back(Client::TestFailure(testCase.m_name, "No error message retrieved"));
                     }
                 }
-
+    
                 if (!testFailures.empty())
                 {
                     testCaseFailures.push_back(Client::TestCaseFailure(testSuite.m_name, AZStd::move(testFailures)));
                 }
             }
+        }
 
-            return Client::TestRunFailure(Client::TestRunFailure(testJob.GetTestTarget()->GetName(), AZStd::move(testCaseFailures)));
-        }
-        else
-        {
-            return Client::TestRunFailure(testJob.GetTestTarget()->GetName(), { });
-        }
+        return testCaseFailures;
     }
 
-    //! Generates a sequence failure report from the specified list of test engine jobs.
-    //! @tparam TestJob The test engine job type.
     template<typename TestJob>
-    Client::SequenceFailure GenerateSequenceFailureReport(const AZStd::vector<TestJob>& testJobs)
+    Client::TestRunReport GenerateTestRunReport(
+        TestSequenceResult result,
+        AZStd::chrono::high_resolution_clock::time_point startTime,
+        AZStd::chrono::milliseconds duration,
+        const AZStd::vector<TestJob>& testJobs)
     {
-        AZStd::vector<Client::ExecutionFailure> executionFailures;
-        AZStd::vector<Client::TestRunFailure> testRunFailures;
-        AZStd::vector<Client::TargetFailure> timedOutTestRuns;
-        AZStd::vector<Client::TargetFailure> unexecutedTestRuns;
-
+        AZStd::vector<Client::TestRun> passingTests;
+        AZStd::vector<Client::TestRunWithTestFailures> failingTests;
+        AZStd::vector<Client::TestRun> executionFailureTests;
+        AZStd::vector<Client::TestRun> timedOutTests;
+        AZStd::vector<Client::TestRun> unexecutedTests;
+        
         for (const auto& testJob : testJobs)
         {
+            // Test job start time relative to start time
+            const auto relativeStartTime =
+                AZStd::chrono::high_resolution_clock::time_point() +
+                AZStd::chrono::duration_cast<AZStd::chrono::milliseconds>(testJob.GetStartTime() - startTime);
+
+            Client::TestRun clientTestRun(
+                testJob.GetTestTarget()->GetName(), testJob.GetCommandString(), relativeStartTime, testJob.GetDuration(),
+                testJob.GetTestResult());
+
             switch (testJob.GetTestResult())
             {
             case Client::TestRunResult::FailedToExecute:
             {
-                executionFailures.push_back(Client::ExecutionFailure(testJob.GetTestTarget()->GetName(), testJob.GetCommandString()));
+                executionFailureTests.push_back(clientTestRun);
                 break;
             }
             case Client::TestRunResult::NotRun:
             {
-                unexecutedTestRuns.push_back(testJob.GetTestTarget()->GetName());
+                unexecutedTests.push_back(clientTestRun);
                 break;
             }
             case Client::TestRunResult::Timeout:
             {
-                timedOutTestRuns.push_back(testJob.GetTestTarget()->GetName());
+                timedOutTests.push_back(clientTestRun);
                 break;
             }
             case Client::TestRunResult::AllTestsPass:
             {
+                passingTests.push_back(clientTestRun);
                 break;
             }
             case Client::TestRunResult::TestFailures:
             {
-                testRunFailures.push_back(GenerateTestRunFailure(testJob));
+                failingTests.emplace_back(AZStd::move(clientTestRun), GenerateTestCaseFailures(testJob));
                 break;
             }
             default:
@@ -116,11 +126,15 @@ namespace TestImpact
             }
             }
         }
-
-        return Client::SequenceFailure(
-            AZStd::move(executionFailures),
-            AZStd::move(testRunFailures),
-            AZStd::move(timedOutTestRuns),
-            AZStd::move(unexecutedTestRuns));
+        
+        return Client::TestRunReport(
+            result,
+            startTime,
+            duration,
+            AZStd::move(passingTests),
+            AZStd::move(failingTests),
+            AZStd::move(executionFailureTests),
+            AZStd::move(timedOutTests),
+            AZStd::move(unexecutedTests));
     }
-}
+} // namespace TestImpact

Code/Tools/TestImpactFramework/Runtime/Code/testimpactframework_runtime_files.cmake

@@ -19,7 +19,7 @@ set(FILES
     Include/TestImpactFramework/TestImpactTestSequence.h
     Include/TestImpactFramework/TestImpactClientTestSelection.h
     Include/TestImpactFramework/TestImpactClientTestRun.h
-    Include/TestImpactFramework/TestImpactClientFailureReport.h
+    Include/TestImpactFramework/TestImpactClientSequenceReport.h
     Include/TestImpactFramework/TestImpactFileUtils.h
     Source/Artifact/TestImpactArtifactException.h
     Source/Artifact/Factory/TestImpactBuildTargetDescriptorFactory.cpp
@@ -123,7 +123,8 @@ set(FILES
     Source/TestImpactRuntimeUtils.h
     Source/TestImpactClientTestSelection.cpp
     Source/TestImpactClientTestRun.cpp
-    Source/TestImpactClientFailureReport.cpp
+    Source/TestImpactClientSequenceReport.cpp
     Source/TestImpactChangeListSerializer.cpp
+    Source/TestImpactChangeListSerializerInternal.h
     Source/TestImpactRepoPath.cpp
 )