From d7c1185dc23d42e7a535c5977aa1ee568d5f28ab Mon Sep 17 00:00:00 2001 From: John Date: Fri, 6 Aug 2021 20:09:53 +0100 Subject: [PATCH] Add previously approved changes. Signed-off-by: John --- .../Code/Source/TestImpactConsoleMain.cpp | 89 ++-- ...tImpactConsoleTestSequenceEventHandler.cpp | 145 +++---- ...estImpactConsoleTestSequenceEventHandler.h | 61 +-- .../TestImpactClientFailureReport.h | 125 ------ .../TestImpactClientSequenceReport.h | 240 ++++++++++ .../TestImpactClientTestRun.h | 105 ++++- .../TestImpactClientTestSelection.h | 1 + .../TestImpactFramework/TestImpactRuntime.h | 74 ++-- .../TestEngine/TestImpactTestEngine.cpp | 6 +- .../Source/TestEngine/TestImpactTestEngine.h | 6 +- .../Source/TestImpactClientFailureReport.cpp | 120 ----- .../Source/TestImpactClientSequenceReport.cpp | 312 +++++++++++++ .../Code/Source/TestImpactClientTestRun.cpp | 100 ++++- .../Runtime/Code/Source/TestImpactRuntime.cpp | 409 +++++++++++++----- .../Code/Source/TestImpactRuntimeUtils.cpp | 12 +- .../Code/Source/TestImpactRuntimeUtils.h | 76 ++-- .../testimpactframework_runtime_files.cmake | 5 +- 17 files changed, 1287 insertions(+), 599 deletions(-) delete mode 100644 Code/Tools/TestImpactFramework/Runtime/Code/Include/TestImpactFramework/TestImpactClientFailureReport.h create mode 100644 Code/Tools/TestImpactFramework/Runtime/Code/Include/TestImpactFramework/TestImpactClientSequenceReport.h delete mode 100644 Code/Tools/TestImpactFramework/Runtime/Code/Source/TestImpactClientFailureReport.cpp create mode 100644 Code/Tools/TestImpactFramework/Runtime/Code/Source/TestImpactClientSequenceReport.cpp diff --git a/Code/Tools/TestImpactFramework/Frontend/Console/Code/Source/TestImpactConsoleMain.cpp b/Code/Tools/TestImpactFramework/Frontend/Console/Code/Source/TestImpactConsoleMain.cpp index 5788da5b2b..8027649a44 100644 --- a/Code/Tools/TestImpactFramework/Frontend/Console/Code/Source/TestImpactConsoleMain.cpp +++ b/Code/Tools/TestImpactFramework/Frontend/Console/Code/Source/TestImpactConsoleMain.cpp @@ -77,7 +77,6 @@ namespace TestImpact //! Wrapper around impact analysis sequences to handle the case where the safe mode option is active. ReturnCode WrappedImpactAnalysisTestSequence( - TestSequenceEventHandler& sequenceEventHandler, const CommandLineOptions& options, Runtime& runtime, const AZStd::optional& changeList) @@ -94,44 +93,30 @@ namespace TestImpact { if (options.GetTestSequenceType() == TestSequenceType::ImpactAnalysis) { - auto [selectedResult, discardedResult] = runtime.SafeImpactAnalysisTestSequence( + auto safeImpactAnalysisSequenceReport = runtime.SafeImpactAnalysisTestSequence( changeList.value(), options.GetTestPrioritizationPolicy(), options.GetTestTargetTimeout(), options.GetGlobalTimeout(), - AZStd::ref(sequenceEventHandler), - AZStd::ref(sequenceEventHandler), - AZStd::ref(sequenceEventHandler)); - - // Handling the possible timeout and failure permutations of the selected and discarded test results is splitting hairs - // so apply the following, admittedly arbitrary, rules to determine what the composite test sequence result should be - if (selectedResult == TestSequenceResult::Success && discardedResult == TestSequenceResult::Success) - { - // Trivial case: both sequences succeeded - result = TestSequenceResult::Success; - } - else if (selectedResult == TestSequenceResult::Failure || discardedResult == TestSequenceResult::Failure) - { - // One sequence failed whilst the other sequence either succeeded or timed out - result = TestSequenceResult::Failure; - } - else - { - // One or both sequences timed out or failed - result = TestSequenceResult::Timeout; - } + SafeImpactAnalysisTestSequenceStartCallback, + SafeImpactAnalysisTestSequenceCompleteCallback, + TestRunCompleteCallback); + + result = safeImpactAnalysisSequenceReport.GetResult(); } else if (options.GetTestSequenceType() == TestSequenceType::ImpactAnalysisNoWrite) { // A no-write impact analysis sequence with safe mode enabled is functionally identical to a regular sequence type // due to a) the selected tests being run without instrumentation and b) the discarded tests also being run without // instrumentation - result = runtime.RegularTestSequence( + auto sequenceReport = runtime.RegularTestSequence( options.GetTestTargetTimeout(), options.GetGlobalTimeout(), - AZStd::ref(sequenceEventHandler), - AZStd::ref(sequenceEventHandler), - AZStd::ref(sequenceEventHandler)); + TestSequenceStartCallback, + TestSequenceCompleteCallback, + TestRunCompleteCallback); + + result = sequenceReport.GetResult(); } else { @@ -153,18 +138,20 @@ namespace TestImpact { throw(Exception("Unexpected sequence type")); } - - result = runtime.ImpactAnalysisTestSequence( + + auto impactAnalysisSequenceReport = runtime.ImpactAnalysisTestSequence( changeList.value(), options.GetTestPrioritizationPolicy(), dynamicDependencyMapPolicy, options.GetTestTargetTimeout(), options.GetGlobalTimeout(), - AZStd::ref(sequenceEventHandler), - AZStd::ref(sequenceEventHandler), - AZStd::ref(sequenceEventHandler)); - } + ImpactAnalysisTestSequenceStartCallback, + ImpactAnalysisTestSequenceCompleteCallback, + TestRunCompleteCallback); + result = impactAnalysisSequenceReport.GetResult(); + } + return GetReturnCodeForTestSequenceResult(result); }; @@ -217,53 +204,51 @@ namespace TestImpact std::cout << "Test impact analysis data for this repository was not found, seed or regular sequence fallbacks will be used.\n"; } - TestSequenceEventHandler sequenceEventHandler(options.GetSuiteFilter()); - switch (const auto type = options.GetTestSequenceType()) { case TestSequenceType::Regular: { - const auto result = runtime.RegularTestSequence( + const auto sequenceReport = runtime.RegularTestSequence( options.GetTestTargetTimeout(), options.GetGlobalTimeout(), - AZStd::ref(sequenceEventHandler), - AZStd::ref(sequenceEventHandler), - AZStd::ref(sequenceEventHandler)); + TestSequenceStartCallback, + TestSequenceCompleteCallback, + TestRunCompleteCallback); - return GetReturnCodeForTestSequenceResult(result); + return GetReturnCodeForTestSequenceResult(sequenceReport.GetResult()); } case TestSequenceType::Seed: { - const auto result = runtime.SeededTestSequence( + const auto sequenceReport = runtime.SeededTestSequence( options.GetTestTargetTimeout(), options.GetGlobalTimeout(), - AZStd::ref(sequenceEventHandler), - AZStd::ref(sequenceEventHandler), - AZStd::ref(sequenceEventHandler)); + TestSequenceStartCallback, + TestSequenceCompleteCallback, + TestRunCompleteCallback); - return GetReturnCodeForTestSequenceResult(result); + return GetReturnCodeForTestSequenceResult(sequenceReport.GetResult()); } case TestSequenceType::ImpactAnalysisNoWrite: case TestSequenceType::ImpactAnalysis: { - return WrappedImpactAnalysisTestSequence(sequenceEventHandler, options, runtime, changeList); + return WrappedImpactAnalysisTestSequence(options, runtime, changeList); } case TestSequenceType::ImpactAnalysisOrSeed: { if (runtime.HasImpactAnalysisData()) { - return WrappedImpactAnalysisTestSequence(sequenceEventHandler, options, runtime, changeList); + return WrappedImpactAnalysisTestSequence(options, runtime, changeList); } else { - const auto result = runtime.SeededTestSequence( + const auto sequenceReport = runtime.SeededTestSequence( options.GetTestTargetTimeout(), options.GetGlobalTimeout(), - AZStd::ref(sequenceEventHandler), - AZStd::ref(sequenceEventHandler), - AZStd::ref(sequenceEventHandler)); + TestSequenceStartCallback, + TestSequenceCompleteCallback, + TestRunCompleteCallback); - return GetReturnCodeForTestSequenceResult(result); + return GetReturnCodeForTestSequenceResult(sequenceReport.GetResult()); } } default: diff --git a/Code/Tools/TestImpactFramework/Frontend/Console/Code/Source/TestImpactConsoleTestSequenceEventHandler.cpp b/Code/Tools/TestImpactFramework/Frontend/Console/Code/Source/TestImpactConsoleTestSequenceEventHandler.cpp index dc29b2580c..da7052933c 100644 --- a/Code/Tools/TestImpactFramework/Frontend/Console/Code/Source/TestImpactConsoleTestSequenceEventHandler.cpp +++ b/Code/Tools/TestImpactFramework/Frontend/Console/Code/Source/TestImpactConsoleTestSequenceEventHandler.cpp @@ -32,72 +32,73 @@ namespace TestImpact std::cout << "Of which " << numExcludedTests << " tests have been excluded and " << numDraftedTests << " tests have been drafted.\n"; } - void FailureReport(const Client::SequenceFailure& failureReport, AZStd::chrono::milliseconds duration) + void FailureReport(const Client::TestRunReport& testRunReport) { - std::cout << "Sequence completed in " << (duration.count() / 1000.f) << "s with"; + std::cout << "Sequence completed in " << (testRunReport.GetDuration().count() / 1000.f) << "s with"; - if (!failureReport.GetExecutionFailures().empty() || - !failureReport.GetTestRunFailures().empty() || - !failureReport.GetTimedOutTests().empty() || - !failureReport.GetUnexecutedTests().empty()) + if (!testRunReport.GetExecutionFailureTests().empty() || + !testRunReport.GetFailingTests().empty() || + !testRunReport.GetTimedOutTests().empty() || + !testRunReport.GetUnexecutedTests().empty()) { std::cout << ":\n"; std::cout << SetColor(Foreground::White, Background::Red).c_str() - << failureReport.GetTestRunFailures().size() + << testRunReport.GetFailingTests().size() << ResetColor().c_str() << " test failures\n"; std::cout << SetColor(Foreground::White, Background::Red).c_str() - << failureReport.GetExecutionFailures().size() + << testRunReport.GetExecutionFailureTests().size() << ResetColor().c_str() << " execution failures\n"; std::cout << SetColor(Foreground::White, Background::Red).c_str() - << failureReport.GetTimedOutTests().size() + << testRunReport.GetTimedOutTests().size() << ResetColor().c_str() << " test timeouts\n"; std::cout << SetColor(Foreground::White, Background::Red).c_str() - << failureReport.GetUnexecutedTests().size() + << testRunReport.GetUnexecutedTests().size() << ResetColor().c_str() << " unexecuted tests\n"; - if (!failureReport.GetTestRunFailures().empty()) + if (!testRunReport.GetFailingTests().empty()) { std::cout << "\nTest failures:\n"; - for (const auto& testRunFailure : failureReport.GetTestRunFailures()) + for (const auto& testRunFailure : testRunReport.GetFailingTests()) { - std::cout << " " << testRunFailure.GetTargetName().c_str(); for (const auto& testCaseFailure : testRunFailure.GetTestCaseFailures()) { - std::cout << "." << testCaseFailure.GetName().c_str(); for (const auto& testFailure : testCaseFailure.GetTestFailures()) { - std::cout << "." << testFailure.GetName().c_str() << "\n"; + std::cout << " " + << testRunFailure.GetTargetName().c_str() + << "." << testCaseFailure.GetName().c_str() + << "." << testFailure.GetName().c_str() << "\n"; } } } } - if (!failureReport.GetExecutionFailures().empty()) + if (!testRunReport.GetExecutionFailureTests().empty()) { std::cout << "\nExecution failures:\n"; - for (const auto& executionFailure : failureReport.GetExecutionFailures()) + for (const auto& executionFailure : testRunReport.GetExecutionFailureTests()) { std::cout << " " << executionFailure.GetTargetName().c_str() << "\n"; std::cout << executionFailure.GetCommandString().c_str() << "\n"; } } - if (!failureReport.GetTimedOutTests().empty()) + if (!testRunReport.GetTimedOutTests().empty()) { std::cout << "\nTimed out tests:\n"; - for (const auto& testTimeout : failureReport.GetTimedOutTests()) + for (const auto& testTimeout : testRunReport.GetTimedOutTests()) { std::cout << " " << testTimeout.GetTargetName().c_str() << "\n"; } } - if (!failureReport.GetUnexecutedTests().empty()) + if (!testRunReport.GetUnexecutedTests().empty()) { std::cout << "\nUnexecuted tests:\n"; - for (const auto& unexecutedTest : failureReport.GetUnexecutedTests()) + for (const auto& unexecutedTest : testRunReport.GetUnexecutedTests()) { std::cout << " " << unexecutedTest.GetTargetName().c_str() << "\n"; } @@ -105,50 +106,42 @@ namespace TestImpact } else { - std::cout << SetColor(Foreground::White, Background::Green).c_str() << " \100% passes!\n" << ResetColor().c_str(); + std::cout << SetColor(Foreground::White, Background::Green).c_str() << " \100% passes!\n" << ResetColor().c_str() << "\n"; } } } - TestSequenceEventHandler::TestSequenceEventHandler(SuiteType suiteFilter) - : m_suiteFilter(suiteFilter) + void TestSequenceStartCallback(SuiteType suiteType, const Client::TestRunSelection& selectedTests) { + Output::TestSuiteFilter(suiteType); + std::cout << selectedTests.GetNumIncludedTestRuns() << " tests selected, " << selectedTests.GetNumExcludedTestRuns() + << " excluded.\n"; } - // TestSequenceStartCallback - void TestSequenceEventHandler::operator()(Client::TestRunSelection&& selectedTests) + void TestSequenceCompleteCallback(SuiteType suiteType, const Client::TestRunSelection& selectedTests) { - ClearState(); - m_numTests = selectedTests.GetNumIncludedTestRuns(); - - Output::TestSuiteFilter(m_suiteFilter); + Output::TestSuiteFilter(suiteType); std::cout << selectedTests.GetNumIncludedTestRuns() << " tests selected, " << selectedTests.GetNumExcludedTestRuns() << " excluded.\n"; } - // ImpactAnalysisTestSequenceStartCallback - void TestSequenceEventHandler::operator()( - Client::TestRunSelection&& selectedTests, - AZStd::vector&& discardedTests, - AZStd::vector&& draftedTests) + void ImpactAnalysisTestSequenceStartCallback( + SuiteType suiteType, + const Client::TestRunSelection& selectedTests, + const AZStd::vector& discardedTests, + const AZStd::vector& draftedTests) { - ClearState(); - m_numTests = selectedTests.GetNumIncludedTestRuns() + draftedTests.size(); - - Output::TestSuiteFilter(m_suiteFilter); + Output::TestSuiteFilter(suiteType); Output::ImpactAnalysisTestSelection( selectedTests.GetTotalNumTests(), discardedTests.size(), selectedTests.GetNumExcludedTestRuns(), draftedTests.size()); } - // SafeImpactAnalysisTestSequenceStartCallback - void TestSequenceEventHandler::operator()( - Client::TestRunSelection&& selectedTests, - Client::TestRunSelection&& discardedTests, - AZStd::vector&& draftedTests) + void SafeImpactAnalysisTestSequenceStartCallback( + SuiteType suiteType, + const Client::TestRunSelection& selectedTests, + const Client::TestRunSelection& discardedTests, + const AZStd::vector& draftedTests) { - ClearState(); - m_numTests = selectedTests.GetNumIncludedTestRuns() + draftedTests.size(); - - Output::TestSuiteFilter(m_suiteFilter); + Output::TestSuiteFilter(suiteType); Output::ImpactAnalysisTestSelection( selectedTests.GetTotalNumTests(), discardedTests.GetTotalNumTests(), @@ -156,40 +149,45 @@ namespace TestImpact draftedTests.size()); } - // TestSequenceCompleteCallback - void TestSequenceEventHandler::operator()( - Client::SequenceFailure&& failureReport, - AZStd::chrono::milliseconds duration) + void TestSequenceCompleteCallback(const Client::SequenceReport& sequenceReport) { - Output::FailureReport(failureReport, duration); + Output::FailureReport(sequenceReport.GetSelectedTestRunReport()); std::cout << "Updating and serializing the test impact analysis data, this may take a moment...\n"; } - // SafeTestSequenceCompleteCallback - void TestSequenceEventHandler::operator()( - Client::SequenceFailure&& selectedFailureReport, - Client::SequenceFailure&& discardedFailureReport, - AZStd::chrono::milliseconds selectedDuration, - AZStd::chrono::milliseconds discaredDuration) + void ImpactAnalysisTestSequenceCompleteCallback(const Client::ImpactAnalysisSequenceReport& sequenceReport) { std::cout << "Selected test run:\n"; - Output::FailureReport(selectedFailureReport, selectedDuration); + Output::FailureReport(sequenceReport.GetSelectedTestRunReport()); - std::cout << "Discarded test run:\n"; - Output::FailureReport(discardedFailureReport, discaredDuration); + std::cout << "Drafted test run:\n"; + Output::FailureReport(sequenceReport.GetDraftedTestRunReport()); std::cout << "Updating and serializing the test impact analysis data, this may take a moment...\n"; } - // TestRunCompleteCallback - void TestSequenceEventHandler::operator()([[maybe_unused]] Client::TestRun&& test) + void SafeImpactAnalysisTestSequenceCompleteCallback(const Client::SafeImpactAnalysisSequenceReport& sequenceReport) { - m_numTestsComplete++; - const auto progress = AZStd::string::format("(%03u/%03u)", m_numTestsComplete, m_numTests, test.GetTargetName().c_str()); + std::cout << "Selected test run:\n"; + Output::FailureReport(sequenceReport.GetSelectedTestRunReport()); + + std::cout << "Discarded test run:\n"; + Output::FailureReport(sequenceReport.GetDiscardedTestRunReport()); + + std::cout << "Drafted test run:\n"; + Output::FailureReport(sequenceReport.GetDraftedTestRunReport()); + + std::cout << "Updating and serializing the test impact analysis data, this may take a moment...\n"; + } + + void TestRunCompleteCallback(const Client::TestRun& testRun, size_t numTestRunsCompleted, size_t totalNumTestRuns) + { + const auto progress = + AZStd::string::format("(%03u/%03u)", numTestRunsCompleted, totalNumTestRuns, testRun.GetTargetName().c_str()); AZStd::string result; - switch (test.GetResult()) + switch (testRun.GetResult()) { case Client::TestRunResult::AllTestsPass: { @@ -216,15 +214,14 @@ namespace TestImpact result = SetColorForString(Foreground::White, Background::Magenta, "TIME"); break; } + default: + { + AZ_Error("TestRunCompleteCallback", false, "Unexpected test result to handle: %u", aznumeric_cast(testRun.GetResult())); + } } - std::cout << progress.c_str() << " " << result.c_str() << " " << test.GetTargetName().c_str() << " (" << (test.GetDuration().count() / 1000.f) << "s)\n"; - } - - void TestSequenceEventHandler::ClearState() - { - m_numTests = 0; - m_numTestsComplete = 0; + std::cout << progress.c_str() << " " << result.c_str() << " " << testRun.GetTargetName().c_str() << " (" + << (testRun.GetDuration().count() / 1000.f) << "s)\n"; } } // namespace Console } // namespace TestImpact diff --git a/Code/Tools/TestImpactFramework/Frontend/Console/Code/Source/TestImpactConsoleTestSequenceEventHandler.h b/Code/Tools/TestImpactFramework/Frontend/Console/Code/Source/TestImpactConsoleTestSequenceEventHandler.h index ea570f6fc8..ff757b2d5a 100644 --- a/Code/Tools/TestImpactFramework/Frontend/Console/Code/Source/TestImpactConsoleTestSequenceEventHandler.h +++ b/Code/Tools/TestImpactFramework/Frontend/Console/Code/Source/TestImpactConsoleTestSequenceEventHandler.h @@ -8,7 +8,7 @@ #include #include -#include +#include #include #include @@ -21,48 +21,33 @@ namespace TestImpact { namespace Console { - //! Event handler for all test sequence types. - class TestSequenceEventHandler - { - public: - explicit TestSequenceEventHandler(SuiteType suiteFilter); + //! Handler for TestSequenceStartCallback event. + void TestSequenceStartCallback(SuiteType suiteType, const Client::TestRunSelection& selectedTests); - //! TestSequenceStartCallback. - void operator()(Client::TestRunSelection&& selectedTests); + //! Handler for TestSequenceStartCallback event. + void ImpactAnalysisTestSequenceStartCallback( + SuiteType suiteType, + const Client::TestRunSelection& selectedTests, + const AZStd::vector& discardedTests, + const AZStd::vector& draftedTests); - //! ImpactAnalysisTestSequenceStartCallback. - void operator()( - Client::TestRunSelection&& selectedTests, - AZStd::vector&& discardedTests, - AZStd::vector&& draftedTests); + //! Handler for SafeImpactAnalysisTestSequenceStartCallback event. + void SafeImpactAnalysisTestSequenceStartCallback( + SuiteType suiteType, + const Client::TestRunSelection& selectedTests, + const Client::TestRunSelection& discardedTests, + const AZStd::vector& draftedTests); - //! SafeImpactAnalysisTestSequenceStartCallback. - void operator()( - Client::TestRunSelection&& selectedTests, - Client::TestRunSelection&& discardedTests, - AZStd::vector&& draftedTests); + //! Handler for TestSequenceCompleteCallback event. + void TestSequenceCompleteCallback(const Client::SequenceReport& sequenceReport); - //! TestSequenceCompleteCallback. - void operator()( - Client::SequenceFailure&& failureReport, - AZStd::chrono::milliseconds duration); + //! Handler for ImpactAnalysisTestSequenceCompleteCallback event. + void ImpactAnalysisTestSequenceCompleteCallback(const Client::ImpactAnalysisSequenceReport& sequenceReport); - //! SafeTestSequenceCompleteCallback. - void operator()( - Client::SequenceFailure&& selectedFailureReport, - Client::SequenceFailure&& discardedFailureReport, - AZStd::chrono::milliseconds selectedDuration, - AZStd::chrono::milliseconds discaredDuration); + //! Handler for SafeImpactAnalysisTestSequenceCompleteCallback event. + void SafeImpactAnalysisTestSequenceCompleteCallback(const Client::SafeImpactAnalysisSequenceReport& sequenceReport); - //! TestRunCompleteCallback. - void operator()(Client::TestRun&& test); - - private: - void ClearState(); - - SuiteType m_suiteFilter; - size_t m_numTests = 0; - size_t m_numTestsComplete = 0; - }; + //! Handler for TestRunCompleteCallback event. + void TestRunCompleteCallback(const Client::TestRun& testRun, size_t numTestRunsCompleted, size_t totalNumTestRuns); } // namespace Console } // namespace TestImpact diff --git a/Code/Tools/TestImpactFramework/Runtime/Code/Include/TestImpactFramework/TestImpactClientFailureReport.h b/Code/Tools/TestImpactFramework/Runtime/Code/Include/TestImpactFramework/TestImpactClientFailureReport.h deleted file mode 100644 index 063a3fa642..0000000000 --- a/Code/Tools/TestImpactFramework/Runtime/Code/Include/TestImpactFramework/TestImpactClientFailureReport.h +++ /dev/null @@ -1,125 +0,0 @@ -/* - * Copyright (c) Contributors to the Open 3D Engine Project. - * For complete copyright and license terms please see the LICENSE at the root of this distribution. - * - * SPDX-License-Identifier: Apache-2.0 OR MIT - * - */ - -#pragma once - -#include -#include - -namespace TestImpact -{ - namespace Client - { - //! Represents a test target that failed, either due to failing to execute, completing in an abnormal state or completing with failing tests. - class TargetFailure - { - public: - TargetFailure(const AZStd::string& targetName); - - //! Returns the name of the test target this failure pertains to. - const AZStd::string& GetTargetName() const; - private: - AZStd::string m_targetName; - }; - - //! Represents a test target that failed to execute. - class ExecutionFailure - : public TargetFailure - { - public: - ExecutionFailure(const AZStd::string& targetName, const AZStd::string& command); - - //! Returns the command string used to execute this test target. - const AZStd::string& GetCommandString() const; - private: - AZStd::string m_commandString; - }; - - //! Represents an individual test of a test target that failed. - class TestFailure - { - public: - TestFailure(const AZStd::string& testName, const AZStd::string& errorMessage); - - //! Returns the name of the test that failed. - const AZStd::string& GetName() const; - - //! Returns the error message of the test that failed. - const AZStd::string& GetErrorMessage() const; - - private: - AZStd::string m_name; - AZStd::string m_errorMessage; - }; - - //! Represents a collection of tests that failed. - //! @note Only the failing tests are included in the collection. - class TestCaseFailure - { - public: - TestCaseFailure(const AZStd::string& testCaseName, AZStd::vector&& testFailures); - - //! Returns the name of the test case containing the failing tests. - const AZStd::string& GetName() const; - - //! Returns the collection of tests in this test case that failed. - const AZStd::vector& GetTestFailures() const; - - private: - AZStd::string m_name; - AZStd::vector m_testFailures; - }; - - //! Represents a test target that launched successfully but contains failing tests. - class TestRunFailure - : public TargetFailure - { - public: - TestRunFailure(const AZStd::string& targetName, AZStd::vector&& testFailures); - - //! Returns the total number of failing tests in this run. - size_t GetNumTestFailures() const; - - //! Returns the test cases in this run containing failing tests. - const AZStd::vector& GetTestCaseFailures() const; - - private: - AZStd::vector m_testCaseFailures; - size_t m_numTestFailures = 0; - }; - - //! Base class for reporting failing test sequences. - class SequenceFailure - { - public: - SequenceFailure( - AZStd::vector&& executionFailures, - AZStd::vector&& testRunFailures, - AZStd::vector&& timedOutTests, - AZStd::vector&& unexecutedTests); - - //! Returns the test targets in this sequence that failed to execute. - const AZStd::vector& GetExecutionFailures() const; - - //! Returns the test targets that contain failing tests. - const AZStd::vector& GetTestRunFailures() const; - - //! Returns the test targets in this sequence that were terminated for exceeding their allotted runtime. - const AZStd::vector& GetTimedOutTests() const; - - //! Returns the test targets in this sequence that were not executed due to the sequence terminating prematurely. - const AZStd::vector& GetUnexecutedTests() const; - - private: - AZStd::vector m_executionFailures; - AZStd::vector m_testRunFailures; - AZStd::vector m_timedOutTests; - AZStd::vector m_unexecutedTests; - }; - } // namespace Client -} // namespace TestImpact diff --git a/Code/Tools/TestImpactFramework/Runtime/Code/Include/TestImpactFramework/TestImpactClientSequenceReport.h b/Code/Tools/TestImpactFramework/Runtime/Code/Include/TestImpactFramework/TestImpactClientSequenceReport.h new file mode 100644 index 0000000000..73f3827fae --- /dev/null +++ b/Code/Tools/TestImpactFramework/Runtime/Code/Include/TestImpactFramework/TestImpactClientSequenceReport.h @@ -0,0 +1,240 @@ +/* + * Copyright (c) Contributors to the Open 3D Engine Project. For complete copyright and license terms please see the LICENSE at the root of this distribution. + * + * SPDX-License-Identifier: Apache-2.0 OR MIT + * + */ + +#pragma once + +#include +#include +#include + +namespace TestImpact +{ + namespace Client + { + //! Report detailing the result and duration of a given set of test runs along with the details of each individual test run. + class TestRunReport + { + public: + //! Constructs the report for the given set of test runs that were run together in the same set. + //! @param result The result of this set of test runs. + //! @param startTime The time point his set of test runs started. + //! @param duration The duration this set of test runs took to complete. + //! @param passingTests The set of test runs that executed successfully with no failing tests. + //! @param failing tests The set of test runs that executed successfully but had one or more failing tests. + //! @param executionFailureTests The set of test runs that failed to execute. + //! @param timedOutTests The set of test runs that executed successfully but were terminated prematurely due to timing out. + //! @param unexecutedTests The set of test runs that were queued up for execution but did not get the opportunity to execute. + TestRunReport( + TestSequenceResult result, + AZStd::chrono::high_resolution_clock::time_point startTime, + AZStd::chrono::milliseconds duration, + AZStd::vector&& passingTests, + AZStd::vector&& failingTests, + AZStd::vector&& executionFailureTests, + AZStd::vector&& timedOutTests, + AZStd::vector&& unexecutedTests); + + //! Returns the result of this sequence of test runs. + TestSequenceResult GetResult() const; + + //! Returns the time this sequence of test runs started relative to T0. + AZStd::chrono::high_resolution_clock::time_point GetStartTime() const; + + //! Returns the time this sequence of test runs ended relative to T0. + AZStd::chrono::high_resolution_clock::time_point GetEndTime() const; + + //! Returns the duration this sequence of test runs took to complete. + AZStd::chrono::milliseconds GetDuration() const; + + //! Returns the number of passing test runs. + size_t GetNumPassingTests() const; + + //! Returns the number of failing test runs. + size_t GetNumFailingTests() const; + + //! Returns the number of timed out test runs. + size_t GetNumTimedOutTests() const; + + //! Returns the number of unexecuted test runs. + size_t GetNumUnexecutedTests() const; + + //! Returns the set of test runs that executed successfully with no failing tests. + const AZStd::vector& GetPassingTests() const; + + //! Returns the set of test runs that executed successfully but had one or more failing tests. + const AZStd::vector& GetFailingTests() const; + + //! Returns the set of test runs that failed to execute. + const AZStd::vector& GetExecutionFailureTests() const; + + //! Returns the set of test runs that executed successfully but were terminated prematurely due to timing out. + const AZStd::vector& GetTimedOutTests() const; + + //! Returns the set of test runs that were queued up for execution but did not get the opportunity to execute. + const AZStd::vector& GetUnexecutedTests() const; + private: + TestSequenceResult m_result; + AZStd::chrono::high_resolution_clock::time_point m_startTime; + AZStd::chrono::milliseconds m_duration; + AZStd::vector m_passingTests; + AZStd::vector m_failingTests; + AZStd::vector m_executionFailureTests; + AZStd::vector m_timedOutTests; + AZStd::vector m_unexecutedTests; + }; + + //! Report detailing a test run sequence of selected tests. + class SequenceReport + { + public: + //! Constructs the report for a sequence of selected tests. + //! @param suiteType The suite from which the tests have been selected from. + //! @param selectedTests The target names of the selected tests. + //! @param selectedTestRunReport The report for the set of selected test runs. + SequenceReport(SuiteType suiteType, const TestRunSelection& selectedTests, TestRunReport&& selectedTestRunReport); + + //! Returns the tests selected for running in the sequence. + TestRunSelection GetSelectedTests() const; + + //! Returns the report for the selected test runs. + TestRunReport GetSelectedTestRunReport() const; + + //! Returns the start time of the sequence. + AZStd::chrono::high_resolution_clock::time_point GetStartTime() const; + + //! Returns the end time of the sequence. + AZStd::chrono::high_resolution_clock::time_point GetEndTime() const; + + //! Returns the result of the sequence. + virtual TestSequenceResult GetResult() const; + + //! Returns the entire duration the sequence took from start to finish. + virtual AZStd::chrono::milliseconds GetDuration() const; + + //! Get the total number of tests in the sequence that passed. + virtual size_t GetTotalNumPassingTests() const; + + //! Get the total number of tests in the sequence that contain one or more test failures. + virtual size_t GetTotalNumFailingTests() const; + + //! Get the total number of tests in the sequence that timed out whilst in flight. + virtual size_t GetTotalNumTimedOutTests() const; + + //! Get the total number of tests in the sequence that were queued for execution but did not get the oppurtunity to execute. + virtual size_t GetTotalNumUnexecutedTests() const; + + private: + SuiteType m_suite; + TestRunSelection m_selectedTests; + TestRunReport m_selectedTestRunReport; + }; + + //! Report detailing a test run sequence of selected and drafted tests. + class DraftingSequenceReport + : public SequenceReport + { + public: + //! Constructs the report for a sequence of selected and drafted tests. + //! @param suiteType The suite from which the tests have been selected from. + //! @param selectedTests The target names of the selected tests. + //! @param draftedTests The target names of the drafted tests. + //! @param selectedTestRunReport The report for the set of selected test runs. + //! @param draftedTestRunReport The report for the set of drafted test runs. + DraftingSequenceReport( + SuiteType suiteType, + const TestRunSelection& selectedTests, + const AZStd::vector& draftedTests, + TestRunReport&& selectedTestRunReport, + TestRunReport&& draftedTestRunReport); + + // SequenceReport overrides ... + TestSequenceResult GetResult() const override; + AZStd::chrono::milliseconds GetDuration() const override; + size_t GetTotalNumPassingTests() const override; + size_t GetTotalNumFailingTests() const override; + size_t GetTotalNumTimedOutTests() const override; + size_t GetTotalNumUnexecutedTests() const override; + + //! Returns the tests drafted for running in the sequence. + const AZStd::vector& GetDraftedTests() const; + + //! Returns the report for the drafted test runs. + TestRunReport GetDraftedTestRunReport() const; + + private: + AZStd::vector m_draftedTests; + TestRunReport m_draftedTestRunReport; + }; + + //! Report detailing an impact analysis sequence of selected, discarded and drafted tests. + class ImpactAnalysisSequenceReport + : public DraftingSequenceReport + { + public: + //! Constructs the report for a sequence of selected and drafted tests. + //! @param suiteType The suite from which the tests have been selected from. + //! @param selectedTests The target names of the selected tests. + //! @param discardedTests The target names of the discarded tests. + //! @param draftedTests The target names of the drafted tests. + //! @param selectedTestRunReport The report for the set of selected test runs. + //! @param draftedTestRunReport The report for the set of drafted test runs. + ImpactAnalysisSequenceReport( + SuiteType suiteType, + const TestRunSelection& selectedTests, + const AZStd::vector& discardedTests, + const AZStd::vector& draftedTests, + TestRunReport&& selectedTestRunReport, + TestRunReport&& draftedTestRunReport); + + //! Returns the tests discarded from running in the sequence. + const AZStd::vector& GetDiscardedTests() const; + private: + AZStd::vector m_discardedTests; + }; + + //! Report detailing an impact analysis sequence of selected, discarded and drafted tests. + class SafeImpactAnalysisSequenceReport + : public DraftingSequenceReport + { + public: + //! Constructs the report for a sequence of selected and drafted tests. + //! @param suiteType The suite from which the tests have been selected from. + //! @param selectedTests The target names of the selected tests. + //! @param discardedTests The target names of the discarded tests. + //! @param draftedTests The target names of the drafted tests. + //! @param selectedTestRunReport The report for the set of selected test runs. + //! @param discardedTestRunReport The report for the set of discarded test runs. + //! @param draftedTestRunReport The report for the set of drafted test runs. + SafeImpactAnalysisSequenceReport( + SuiteType suiteType, + const TestRunSelection& selectedTests, + const TestRunSelection& discardedTests, + const AZStd::vector& draftedTests, + TestRunReport&& selectedTestRunReport, + TestRunReport&& discardedTestRunReport, + TestRunReport&& draftedTestRunReport); + + // DraftingSequenceReport overrides ... + TestSequenceResult GetResult() const override; + AZStd::chrono::milliseconds GetDuration() const override; + size_t GetTotalNumPassingTests() const override; + size_t GetTotalNumFailingTests() const override; + size_t GetTotalNumTimedOutTests() const override; + size_t GetTotalNumUnexecutedTests() const override; + + //! Returns the report for the discarded test runs. + const TestRunSelection GetDiscardedTests() const; + + //! Returns the report for the discarded test runs. + TestRunReport GetDiscardedTestRunReport() const; + + private: + TestRunSelection m_discardedTests; + TestRunReport m_discardedTestRunReport; + }; + } // namespace Client +} // namespace TestImpact diff --git a/Code/Tools/TestImpactFramework/Runtime/Code/Include/TestImpactFramework/TestImpactClientTestRun.h b/Code/Tools/TestImpactFramework/Runtime/Code/Include/TestImpactFramework/TestImpactClientTestRun.h index f8a2707b96..4b7715bf1d 100644 --- a/Code/Tools/TestImpactFramework/Runtime/Code/Include/TestImpactFramework/TestImpactClientTestRun.h +++ b/Code/Tools/TestImpactFramework/Runtime/Code/Include/TestImpactFramework/TestImpactClientTestRun.h @@ -6,8 +6,9 @@ * */ -#include #include +#include +#include #pragma once @@ -25,18 +26,118 @@ namespace TestImpact AllTestsPass //!< The test run completed its run and all tests passed. }; + //! Representation of a completed test run. class TestRun { public: - TestRun(const AZStd::string& name, TestRunResult result, AZStd::chrono::milliseconds duration); + //! Constructs the client facing representation of a given test target's run. + //! @param name The name of the test target. + //! @param commandString The command string used to execute this test target. + //! @param startTime The start time, relative to the sequence start, that this run started. + //! @param duration The duration that this test run took to complete. + //! @param result The result of the run. + TestRun( + const AZStd::string& name, + const AZStd::string& commandString, + AZStd::chrono::high_resolution_clock::time_point startTime, + AZStd::chrono::milliseconds duration, + TestRunResult result); + + //! Returns the test target name. const AZStd::string& GetTargetName() const; + + //! Returns the test run result. TestRunResult GetResult() const; + + //! Returns the test run start time. + AZStd::chrono::high_resolution_clock::time_point GetStartTime() const; + + //! Returns the end time, relative to the sequence start, that this run ended. + AZStd::chrono::high_resolution_clock::time_point GetEndTime() const; + + //! Returns the duration that this test run took to complete. AZStd::chrono::milliseconds GetDuration() const; + //! Returns the command string used to execute this test target. + const AZStd::string& GetCommandString() const; + private: AZStd::string m_targetName; + AZStd::string m_commandString; TestRunResult m_result; + AZStd::chrono::high_resolution_clock::time_point m_startTime; AZStd::chrono::milliseconds m_duration; }; + + //! Represents an individual test of a test target that failed. + class TestFailure + { + public: + TestFailure(const AZStd::string& testName, const AZStd::string& errorMessage); + + //! Returns the name of the test that failed. + const AZStd::string& GetName() const; + + //! Returns the error message of the test that failed. + const AZStd::string& GetErrorMessage() const; + + private: + AZStd::string m_name; + AZStd::string m_errorMessage; + }; + + //! Represents a collection of tests that failed. + //! @note Only the failing tests are included in the collection. + class TestCaseFailure + { + public: + TestCaseFailure(const AZStd::string& testCaseName, AZStd::vector&& testFailures); + + //! Returns the name of the test case containing the failing tests. + const AZStd::string& GetName() const; + + //! Returns the collection of tests in this test case that failed. + const AZStd::vector& GetTestFailures() const; + + private: + AZStd::string m_name; + AZStd::vector m_testFailures; + }; + + //! Representation of a test run's failing tests. + class TestRunWithTestFailures + : public TestRun + { + public: + //! Constructs the client facing representation of a given test target's run. + //! @param name The name of the test target. + //! @param commandString The command string used to execute this test target. + //! @param startTime The start time, relative to the sequence start, that this run started. + //! @param duration The duration that this test run took to complete. + //! @param result The result of the run. + //! @param testFailures The failing tests for this test run. + TestRunWithTestFailures( + const AZStd::string& name, + const AZStd::string& commandString, + AZStd::chrono::high_resolution_clock::time_point startTime, + AZStd::chrono::milliseconds duration, + TestRunResult result, + AZStd::vector&& testFailures); + + //! Constructs the client facing representation of a given test target's run. + //! @param testRun The test run this run is to be derived from. + //! @param testFailures The failing tests for this run. + TestRunWithTestFailures(TestRun&& testRun, AZStd::vector&& testFailures); + + //! Returns the total number of failing tests in this run. + size_t GetNumTestFailures() const; + + //! Returns the test cases in this run containing failing tests. + const AZStd::vector& GetTestCaseFailures() const; + + private: + AZStd::vector m_testCaseFailures; + size_t m_numTestFailures = 0; + }; } // namespace Client } // namespace TestImpact diff --git a/Code/Tools/TestImpactFramework/Runtime/Code/Include/TestImpactFramework/TestImpactClientTestSelection.h b/Code/Tools/TestImpactFramework/Runtime/Code/Include/TestImpactFramework/TestImpactClientTestSelection.h index af084ee57b..7d2169e680 100644 --- a/Code/Tools/TestImpactFramework/Runtime/Code/Include/TestImpactFramework/TestImpactClientTestSelection.h +++ b/Code/Tools/TestImpactFramework/Runtime/Code/Include/TestImpactFramework/TestImpactClientTestSelection.h @@ -21,6 +21,7 @@ namespace TestImpact class TestRunSelection { public: + TestRunSelection() = default; TestRunSelection(const AZStd::vector& includedTests, const AZStd::vector& excludedTests); TestRunSelection(AZStd::vector&& includedTests, AZStd::vector&& excludedTests); diff --git a/Code/Tools/TestImpactFramework/Runtime/Code/Include/TestImpactFramework/TestImpactRuntime.h b/Code/Tools/TestImpactFramework/Runtime/Code/Include/TestImpactFramework/TestImpactRuntime.h index 506aeef53e..69feb48749 100644 --- a/Code/Tools/TestImpactFramework/Runtime/Code/Include/TestImpactFramework/TestImpactRuntime.h +++ b/Code/Tools/TestImpactFramework/Runtime/Code/Include/TestImpactFramework/TestImpactRuntime.h @@ -12,7 +12,7 @@ #include #include #include -#include +#include #include #include @@ -34,10 +34,12 @@ namespace TestImpact class TestEngineInstrumentedRun; //! Callback for a test sequence that isn't using test impact analysis to determine selected tests. + //! @parm suiteType The test suite to select tests from. //! @param tests The tests that will be run for this sequence. - using TestSequenceStartCallback = AZStd::function; + using TestSequenceStartCallback = AZStd::function; //! Callback for a test sequence using test impact analysis. + //! @parm suiteType The test suite to select tests from. //! @param selectedTests The tests that have been selected for this run by test impact analysis. //! @param discardedTests The tests that have been rejected for this run by test impact analysis. //! @param draftedTests The tests that have been drafted in for this run due to requirements outside of test impact analysis @@ -46,11 +48,13 @@ namespace TestImpact //! These tests will be run with coverage instrumentation. //! @note discardedTests and draftedTests may contain overlapping tests. using ImpactAnalysisTestSequenceStartCallback = AZStd::function&& discardedTests, - AZStd::vector&& draftedTests)>; + SuiteType suiteType, + const Client::TestRunSelection& selectedTests, + const AZStd::vector& discardedTests, + const AZStd::vector& draftedTests)>; //! Callback for a test sequence using test impact analysis. + //! @parm suiteType The test suite to select tests from. //! @param selectedTests The tests that have been selected for this run by test impact analysis. //! @param discardedTests The tests that have been rejected for this run by test impact analysis. //! These tests will not be run without coverage instrumentation unless there is an entry in the draftedTests list. @@ -59,30 +63,22 @@ namespace TestImpact //! to execute previously). //! @note discardedTests and draftedTests may contain overlapping tests. using SafeImpactAnalysisTestSequenceStartCallback = AZStd::function&& draftedTests)>; + SuiteType suiteType, + const Client::TestRunSelection& selectedTests, + const Client::TestRunSelection& discardedTests, + const AZStd::vector& draftedTests)>; //! Callback for end of a test sequence. - //! @param failureReport The test runs that failed for any reason during this sequence. - //! @param duration The total duration of this test sequence. - using TestSequenceCompleteCallback = AZStd::function; - - //! Callback for end of a test impact analysis test sequence. - //! @param selectedFailureReport The selected test runs that failed for any reason during this sequence. - //! @param discardedFailureReport The discarded test runs that failed for any reason during this sequence. - //! @param duration The total duration of this test sequence. - using SafeTestSequenceCompleteCallback = AZStd::function; + //! @tparam SequenceReportType The report type to be used for the sequence. + //! @param sequenceReport The completed sequence report. + template + using TestSequenceCompleteCallback = AZStd::function; //! Callback for test runs that have completed for any reason. - //! @param selectedTests The test that has completed. - using TestRunCompleteCallback = AZStd::function; + //! @param testRunMeta The test that has completed. + //! @param numTestRunsCompleted The number of test runs that have completed. + //! @param totalNumTestRuns The total number of test runs in the sequence. + using TestRunCompleteCallback = AZStd::function; //! The API exposed to the client responsible for all test runs and persistent data management. class Runtime @@ -108,19 +104,19 @@ namespace TestImpact AZStd::optional maxConcurrency = AZStd::nullopt); ~Runtime(); - + //! Runs a test sequence where all tests with a matching suite in the suite filter and also not on the excluded list are selected. //! @param testTargetTimeout The maximum duration individual test targets may be in flight for (infinite if empty). //! @param globalTimeout The maximum duration the entire test sequence may run for (infinite if empty). //! @param testSequenceStartCallback The client function to be called after the test targets have been selected but prior to running the tests. //! @param testSequenceCompleteCallback The client function to be called after the test sequence has completed. //! @param testRunCompleteCallback The client function to be called after an individual test run has completed. - //! @returns - TestSequenceResult RegularTestSequence( + //! @returns The test run and sequence report for the selected test sequence. + Client::SequenceReport RegularTestSequence( AZStd::optional testTargetTimeout, AZStd::optional globalTimeout, AZStd::optional testSequenceStartCallback, - AZStd::optional testSequenceCompleteCallback, + AZStd::optional> testSequenceCompleteCallback, AZStd::optional testRunCompleteCallback); //! Runs a test sequence where tests are selected according to test impact analysis so long as they are not on the excluded list. @@ -132,15 +128,15 @@ namespace TestImpact //! @param testSequenceStartCallback The client function to be called after the test targets have been selected but prior to running the tests. //! @param testSequenceCompleteCallback The client function to be called after the test sequence has completed. //! @param testRunCompleteCallback The client function to be called after an individual test run has completed. - //! @returns - TestSequenceResult ImpactAnalysisTestSequence( + //! @returns The test run and sequence report for the selected and drafted test sequences. + Client::ImpactAnalysisSequenceReport ImpactAnalysisTestSequence( const ChangeList& changeList, Policy::TestPrioritization testPrioritizationPolicy, Policy::DynamicDependencyMap dynamicDependencyMapPolicy, AZStd::optional testTargetTimeout, AZStd::optional globalTimeout, AZStd::optional testSequenceStartCallback, - AZStd::optional testSequenceCompleteCallback, + AZStd::optional> testSequenceCompleteCallback, AZStd::optional testRunCompleteCallback); //! Runs a test sequence as per the ImpactAnalysisTestSequence where the tests not selected are also run (albeit without instrumentation). @@ -151,14 +147,14 @@ namespace TestImpact //! @param testSequenceStartCallback The client function to be called after the test targets have been selected but prior to running the tests. //! @param testSequenceCompleteCallback The client function to be called after the test sequence has completed. //! @param testRunCompleteCallback The client function to be called after an individual test run has completed. - //! @returns - AZStd::pair SafeImpactAnalysisTestSequence( + //! @returns The test run and sequence report for the selected, discarded and drafted test sequences. + Client::SafeImpactAnalysisSequenceReport SafeImpactAnalysisTestSequence( const ChangeList& changeList, Policy::TestPrioritization testPrioritizationPolicy, AZStd::optional testTargetTimeout, AZStd::optional globalTimeout, AZStd::optional testSequenceStartCallback, - AZStd::optional testSequenceCompleteCallback, + AZStd::optional> testSequenceCompleteCallback, AZStd::optional testRunCompleteCallback); //! Runs all tests not on the excluded list and uses their coverage data to seed the test impact analysis data (ant existing data will be overwritten). @@ -167,12 +163,12 @@ namespace TestImpact //! @param testSequenceStartCallback The client function to be called after the test targets have been selected but prior to running the tests. //! @param testSequenceCompleteCallback The client function to be called after the test sequence has completed. //! @param testRunCompleteCallback The client function to be called after an individual test run has completed. - //! - TestSequenceResult SeededTestSequence( + //! @returns The test run and sequence report for the selected test sequence. + Client::SequenceReport SeededTestSequence( AZStd::optional testTargetTimeout, AZStd::optional globalTimeout, AZStd::optional testSequenceStartCallback, - AZStd::optional testSequenceCompleteCallback, + AZStd::optional> testSequenceCompleteCallback, AZStd::optional testRunCompleteCallback); //! Returns true if the runtime has test impact analysis data (either preexisting or generated). @@ -209,8 +205,8 @@ namespace TestImpact void UpdateAndSerializeDynamicDependencyMap(const AZStd::vector& jobs); RuntimeConfig m_config; - SuiteType m_suiteFilter; RepoPath m_sparTIAFile; + SuiteType m_suiteFilter; Policy::ExecutionFailure m_executionFailurePolicy; Policy::FailedTestCoverage m_failedTestCoveragePolicy; Policy::TestFailure m_testFailurePolicy; diff --git a/Code/Tools/TestImpactFramework/Runtime/Code/Source/TestEngine/TestImpactTestEngine.cpp b/Code/Tools/TestImpactFramework/Runtime/Code/Source/TestEngine/TestImpactTestEngine.cpp index 61634a8bd3..226301d444 100644 --- a/Code/Tools/TestImpactFramework/Runtime/Code/Source/TestEngine/TestImpactTestEngine.cpp +++ b/Code/Tools/TestImpactFramework/Runtime/Code/Source/TestEngine/TestImpactTestEngine.cpp @@ -262,7 +262,7 @@ namespace TestImpact Policy::TestFailure testFailurePolicy, AZStd::optional testTargetTimeout, AZStd::optional globalTimeout, - AZStd::optional callback) + AZStd::optional callback) const { TestEngineJobMap engineJobs; const auto jobInfos = m_testJobInfoGenerator->GenerateTestEnumerationJobInfos(testTargets, TestEnumerator::JobInfo::CachePolicy::Write); @@ -285,7 +285,7 @@ namespace TestImpact [[maybe_unused]]Policy::TargetOutputCapture targetOutputCapture, AZStd::optional testTargetTimeout, AZStd::optional globalTimeout, - AZStd::optional callback) + AZStd::optional callback) const { DeleteArtifactXmls(); @@ -312,7 +312,7 @@ namespace TestImpact [[maybe_unused]]Policy::TargetOutputCapture targetOutputCapture, AZStd::optional testTargetTimeout, AZStd::optional globalTimeout, - AZStd::optional callback) + AZStd::optional callback) const { DeleteArtifactXmls(); diff --git a/Code/Tools/TestImpactFramework/Runtime/Code/Source/TestEngine/TestImpactTestEngine.h b/Code/Tools/TestImpactFramework/Runtime/Code/Source/TestEngine/TestImpactTestEngine.h index f1bb77f06f..6b097f241f 100644 --- a/Code/Tools/TestImpactFramework/Runtime/Code/Source/TestEngine/TestImpactTestEngine.h +++ b/Code/Tools/TestImpactFramework/Runtime/Code/Source/TestEngine/TestImpactTestEngine.h @@ -69,7 +69,7 @@ namespace TestImpact Policy::TestFailure testFailurePolicy, AZStd::optional testTargetTimeout, AZStd::optional globalTimeout, - AZStd::optional callback); + AZStd::optional callback) const; //! Performs a test run without any instrumentation and, for each test target, returns the test run results and metrics about the run. //! @param testTargets The test targets to run. @@ -89,7 +89,7 @@ namespace TestImpact Policy::TargetOutputCapture targetOutputCapture, AZStd::optional testTargetTimeout, AZStd::optional globalTimeout, - AZStd::optional callback); + AZStd::optional callback) const; //! Performs a test run with instrumentation and, for each test target, returns the test run results, coverage data and metrics about the run. //! @param testTargets The test targets to run. @@ -111,7 +111,7 @@ namespace TestImpact Policy::TargetOutputCapture targetOutputCapture, AZStd::optional testTargetTimeout, AZStd::optional globalTimeout, - AZStd::optional callback); + AZStd::optional callback) const; private: //! Cleans up the artifacts directory of any artifacts from previous runs. diff --git a/Code/Tools/TestImpactFramework/Runtime/Code/Source/TestImpactClientFailureReport.cpp b/Code/Tools/TestImpactFramework/Runtime/Code/Source/TestImpactClientFailureReport.cpp deleted file mode 100644 index 000ded2aa8..0000000000 --- a/Code/Tools/TestImpactFramework/Runtime/Code/Source/TestImpactClientFailureReport.cpp +++ /dev/null @@ -1,120 +0,0 @@ -/* - * Copyright (c) Contributors to the Open 3D Engine Project. - * For complete copyright and license terms please see the LICENSE at the root of this distribution. - * - * SPDX-License-Identifier: Apache-2.0 OR MIT - * - */ - -#include - -namespace TestImpact -{ - namespace Client - { - TargetFailure::TargetFailure(const AZStd::string& targetName) - : m_targetName(targetName) - { - } - - const AZStd::string& TargetFailure::GetTargetName() const - { - return m_targetName; - } - - ExecutionFailure::ExecutionFailure(const AZStd::string& targetName, const AZStd::string& command) - : TargetFailure(targetName) - , m_commandString(command) - { - } - - const AZStd::string& ExecutionFailure::GetCommandString() const - { - return m_commandString; - } - - TestFailure::TestFailure(const AZStd::string& testName, const AZStd::string& errorMessage) - : m_name(testName) - , m_errorMessage(errorMessage) - { - } - - const AZStd::string& TestFailure::GetName() const - { - return m_name; - } - - const AZStd::string& TestFailure::GetErrorMessage() const - { - return m_errorMessage; - } - - TestCaseFailure::TestCaseFailure(const AZStd::string& testCaseName, AZStd::vector&& testFailures) - : m_name(testCaseName) - , m_testFailures(AZStd::move(testFailures)) - { - } - - const AZStd::string& TestCaseFailure::GetName() const - { - return m_name; - } - - const AZStd::vector& TestCaseFailure::GetTestFailures() const - { - return m_testFailures; - } - - TestRunFailure::TestRunFailure(const AZStd::string& targetName, AZStd::vector&& testFailures) - : TargetFailure(targetName) - , m_testCaseFailures(AZStd::move(testFailures)) - { - for (const auto& testCase : m_testCaseFailures) - { - m_numTestFailures += testCase.GetTestFailures().size(); - } - } - - size_t TestRunFailure::GetNumTestFailures() const - { - return m_numTestFailures; - } - - const AZStd::vector& TestRunFailure::GetTestCaseFailures() const - { - return m_testCaseFailures; - } - - SequenceFailure::SequenceFailure( - AZStd::vector&& executionFailures, - AZStd::vector&& testRunFailures, - AZStd::vector&& timedOutTests, - AZStd::vector&& unexecutionTests) - : m_executionFailures(AZStd::move(executionFailures)) - , m_testRunFailures(testRunFailures) - , m_timedOutTests(AZStd::move(timedOutTests)) - , m_unexecutedTests(AZStd::move(unexecutionTests)) - { - } - - const AZStd::vector& SequenceFailure::GetExecutionFailures() const - { - return m_executionFailures; - } - - const AZStd::vector& SequenceFailure::GetTestRunFailures() const - { - return m_testRunFailures; - } - - const AZStd::vector& SequenceFailure::GetTimedOutTests() const - { - return m_timedOutTests; - } - - const AZStd::vector& SequenceFailure::GetUnexecutedTests() const - { - return m_unexecutedTests; - } - } // namespace Client -} // namespace TestImpact diff --git a/Code/Tools/TestImpactFramework/Runtime/Code/Source/TestImpactClientSequenceReport.cpp b/Code/Tools/TestImpactFramework/Runtime/Code/Source/TestImpactClientSequenceReport.cpp new file mode 100644 index 0000000000..ba3a479fdf --- /dev/null +++ b/Code/Tools/TestImpactFramework/Runtime/Code/Source/TestImpactClientSequenceReport.cpp @@ -0,0 +1,312 @@ +/* + * Copyright (c) Contributors to the Open 3D Engine Project. For complete copyright and license terms please see the LICENSE at the root of this distribution. + * + * SPDX-License-Identifier: Apache-2.0 OR MIT + * + */ + +#include + +namespace TestImpact +{ + namespace Client + { + //! Calculates the final sequence result for a composite of multiple sequences. + TestSequenceResult CalculateMultiTestSequenceResult(const AZStd::vector& results) + { + // Order of precedence: + // 1. TestSequenceResult::Failure + // 2. TestSequenceResult::Timeout + // 3. TestSequenceResult::Success + + if (const auto it = AZStd::find(results.begin(), results.end(), TestSequenceResult::Failure); + it != results.end()) + { + return TestSequenceResult::Failure; + } + + if (const auto it = AZStd::find(results.begin(), results.end(), TestSequenceResult::Timeout); + it != results.end()) + { + return TestSequenceResult::Timeout; + } + + return TestSequenceResult::Success; + } + + TestRunReport::TestRunReport( + TestSequenceResult result, + AZStd::chrono::high_resolution_clock::time_point startTime, + AZStd::chrono::milliseconds duration, + AZStd::vector&& passingTests, + AZStd::vector&& failingTests, + AZStd::vector&& executionFailureTests, + AZStd::vector&& timedOutTests, + AZStd::vector&& unexecutedTests) + : m_startTime(startTime) + , m_result(result) + , m_duration(duration) + , m_passingTests(AZStd::move(passingTests)) + , m_failingTests(AZStd::move(failingTests)) + , m_executionFailureTests(AZStd::move(executionFailureTests)) + , m_timedOutTests(AZStd::move(timedOutTests)) + , m_unexecutedTests(AZStd::move(unexecutedTests)) + { + } + + TestSequenceResult TestRunReport::GetResult() const + { + return m_result; + } + + AZStd::chrono::high_resolution_clock::time_point TestRunReport::GetStartTime() const + { + return m_startTime; + } + + AZStd::chrono::high_resolution_clock::time_point TestRunReport::GetEndTime() const + { + return m_startTime + m_duration; + } + + AZStd::chrono::milliseconds TestRunReport::GetDuration() const + { + return m_duration; + } + + size_t TestRunReport::GetNumPassingTests() const + { + return m_passingTests.size(); + } + + size_t TestRunReport::GetNumFailingTests() const + { + return m_failingTests.size(); + } + + size_t TestRunReport::GetNumTimedOutTests() const + { + return m_timedOutTests.size(); + } + + size_t TestRunReport::GetNumUnexecutedTests() const + { + return m_unexecutedTests.size(); + } + + const AZStd::vector& TestRunReport::GetPassingTests() const + { + return m_passingTests; + } + + const AZStd::vector& TestRunReport::GetFailingTests() const + { + return m_failingTests; + } + + const AZStd::vector& TestRunReport::GetExecutionFailureTests() const + { + return m_executionFailureTests; + } + + const AZStd::vector& TestRunReport::GetTimedOutTests() const + { + return m_timedOutTests; + } + + const AZStd::vector& TestRunReport::GetUnexecutedTests() const + { + return m_unexecutedTests; + } + + SequenceReport::SequenceReport(SuiteType suiteType, const TestRunSelection& selectedTests, TestRunReport&& selectedTestRunReport) + : m_suite(suiteType) + , m_selectedTests(selectedTests) + , m_selectedTestRunReport(AZStd::move(selectedTestRunReport)) + { + } + + TestSequenceResult SequenceReport::GetResult() const + { + return m_selectedTestRunReport.GetResult(); + } + + AZStd::chrono::high_resolution_clock::time_point SequenceReport::GetStartTime() const + { + return m_selectedTestRunReport.GetStartTime(); + } + + AZStd::chrono::high_resolution_clock::time_point SequenceReport::GetEndTime() const + { + return GetStartTime() + GetDuration(); + } + + AZStd::chrono::milliseconds SequenceReport::GetDuration() const + { + return m_selectedTestRunReport.GetDuration(); + } + + TestRunSelection SequenceReport::GetSelectedTests() const + { + return m_selectedTests; + } + + TestRunReport SequenceReport::GetSelectedTestRunReport() const + { + return m_selectedTestRunReport; + } + + size_t SequenceReport::GetTotalNumPassingTests() const + { + return m_selectedTestRunReport.GetNumPassingTests(); + } + + size_t SequenceReport::GetTotalNumFailingTests() const + { + return m_selectedTestRunReport.GetNumFailingTests(); + } + + size_t SequenceReport::GetTotalNumTimedOutTests() const + { + return m_selectedTestRunReport.GetNumTimedOutTests(); + } + + size_t SequenceReport::GetTotalNumUnexecutedTests() const + { + return m_selectedTestRunReport.GetNumUnexecutedTests(); + } + + DraftingSequenceReport::DraftingSequenceReport( + SuiteType suiteType, + const TestRunSelection& selectedTests, + const AZStd::vector& draftedTests, + TestRunReport&& selectedTestRunReport, + TestRunReport&& draftedTestRunReport) + : SequenceReport(suiteType, selectedTests, AZStd::move(selectedTestRunReport)) + , m_draftedTests(draftedTests) + , m_draftedTestRunReport(AZStd::move(draftedTestRunReport)) + { + } + + TestSequenceResult DraftingSequenceReport::GetResult() const + { + return CalculateMultiTestSequenceResult({SequenceReport::GetResult(), m_draftedTestRunReport.GetResult()}); + } + + AZStd::chrono::milliseconds DraftingSequenceReport::GetDuration() const + { + return SequenceReport::GetDuration() + m_draftedTestRunReport.GetDuration(); + } + + size_t DraftingSequenceReport::GetTotalNumPassingTests() const + { + return SequenceReport::GetTotalNumPassingTests() + m_draftedTestRunReport.GetNumPassingTests(); + } + + size_t DraftingSequenceReport::GetTotalNumFailingTests() const + { + return SequenceReport::GetTotalNumFailingTests() + m_draftedTestRunReport.GetNumFailingTests(); + } + + size_t DraftingSequenceReport::GetTotalNumTimedOutTests() const + { + return SequenceReport::GetTotalNumTimedOutTests() + m_draftedTestRunReport.GetNumTimedOutTests(); + } + + size_t DraftingSequenceReport::GetTotalNumUnexecutedTests() const + { + return SequenceReport::GetTotalNumUnexecutedTests() + m_draftedTestRunReport.GetNumUnexecutedTests(); + } + + const AZStd::vector& DraftingSequenceReport::GetDraftedTests() const + { + return m_draftedTests; + } + + TestRunReport DraftingSequenceReport::GetDraftedTestRunReport() const + { + return m_draftedTestRunReport; + } + + ImpactAnalysisSequenceReport::ImpactAnalysisSequenceReport( + SuiteType suiteType, + const TestRunSelection& selectedTests, + const AZStd::vector& discardedTests, + const AZStd::vector& draftedTests, + TestRunReport&& selectedTestRunReport, + TestRunReport&& draftedTestRunReport) + : DraftingSequenceReport( + suiteType, + selectedTests, + draftedTests, + AZStd::move(selectedTestRunReport), + AZStd::move(draftedTestRunReport)) + , m_discardedTests(discardedTests) + { + } + + const AZStd::vector& ImpactAnalysisSequenceReport::GetDiscardedTests() const + { + return m_discardedTests; + } + + SafeImpactAnalysisSequenceReport::SafeImpactAnalysisSequenceReport( + SuiteType suiteType, + const TestRunSelection& selectedTests, + const TestRunSelection& discardedTests, + const AZStd::vector& draftedTests, + TestRunReport&& selectedTestRunReport, + TestRunReport&& discardedTestRunReport, + TestRunReport&& draftedTestRunReport) + : DraftingSequenceReport( + suiteType, + selectedTests, + draftedTests, + AZStd::move(selectedTestRunReport), + AZStd::move(draftedTestRunReport)) + , m_discardedTests(discardedTests) + , m_discardedTestRunReport(AZStd::move(discardedTestRunReport)) + { + } + + TestSequenceResult SafeImpactAnalysisSequenceReport::GetResult() const + { + return CalculateMultiTestSequenceResult({ DraftingSequenceReport::GetResult(), m_discardedTestRunReport.GetResult() }); + } + + AZStd::chrono::milliseconds SafeImpactAnalysisSequenceReport::GetDuration() const + { + return DraftingSequenceReport::GetDuration() + m_discardedTestRunReport.GetDuration(); + } + + size_t SafeImpactAnalysisSequenceReport::GetTotalNumPassingTests() const + { + return DraftingSequenceReport::GetTotalNumPassingTests() + m_discardedTestRunReport.GetNumPassingTests(); + } + + size_t SafeImpactAnalysisSequenceReport::GetTotalNumFailingTests() const + { + return DraftingSequenceReport::GetTotalNumFailingTests() + m_discardedTestRunReport.GetNumFailingTests(); + } + + size_t SafeImpactAnalysisSequenceReport::GetTotalNumTimedOutTests() const + { + return DraftingSequenceReport::GetTotalNumTimedOutTests() + m_discardedTestRunReport.GetNumTimedOutTests(); + } + + size_t SafeImpactAnalysisSequenceReport::GetTotalNumUnexecutedTests() const + { + return DraftingSequenceReport::GetTotalNumUnexecutedTests() + m_discardedTestRunReport.GetNumUnexecutedTests(); + } + + const TestRunSelection SafeImpactAnalysisSequenceReport::GetDiscardedTests() const + { + return m_discardedTests; + } + + TestRunReport SafeImpactAnalysisSequenceReport::GetDiscardedTestRunReport() const + { + return m_discardedTestRunReport; + } + } // namespace Client +} // namespace TestImpact diff --git a/Code/Tools/TestImpactFramework/Runtime/Code/Source/TestImpactClientTestRun.cpp b/Code/Tools/TestImpactFramework/Runtime/Code/Source/TestImpactClientTestRun.cpp index 582927baa5..0a258c5dac 100644 --- a/Code/Tools/TestImpactFramework/Runtime/Code/Source/TestImpactClientTestRun.cpp +++ b/Code/Tools/TestImpactFramework/Runtime/Code/Source/TestImpactClientTestRun.cpp @@ -7,14 +7,22 @@ */ #include + namespace TestImpact { namespace Client { - TestRun::TestRun(const AZStd::string& name, TestRunResult result, AZStd::chrono::milliseconds duration) + TestRun::TestRun( + const AZStd::string& name, + const AZStd::string& commandString, + AZStd::chrono::high_resolution_clock::time_point startTime, + AZStd::chrono::milliseconds duration, + TestRunResult result) : m_targetName(name) - , m_result(result) + , m_commandString(commandString) + , m_startTime(startTime) , m_duration(duration) + , m_result(result) { } @@ -23,6 +31,21 @@ namespace TestImpact return m_targetName; } + const AZStd::string& TestRun::GetCommandString() const + { + return m_commandString; + } + + AZStd::chrono::high_resolution_clock::time_point TestRun::GetStartTime() const + { + return m_startTime; + } + + AZStd::chrono::high_resolution_clock::time_point TestRun::GetEndTime() const + { + return m_startTime + m_duration; + } + AZStd::chrono::milliseconds TestRun::GetDuration() const { return m_duration; @@ -32,5 +55,78 @@ namespace TestImpact { return m_result; } + + TestFailure::TestFailure(const AZStd::string& testName, const AZStd::string& errorMessage) + : m_name(testName) + , m_errorMessage(errorMessage) + { + } + + const AZStd::string& TestFailure::GetName() const + { + return m_name; + } + + const AZStd::string& TestFailure::GetErrorMessage() const + { + return m_errorMessage; + } + + TestCaseFailure::TestCaseFailure(const AZStd::string& testCaseName, AZStd::vector&& testFailures) + : m_name(testCaseName) + , m_testFailures(AZStd::move(testFailures)) + { + } + + const AZStd::string& TestCaseFailure::GetName() const + { + return m_name; + } + + const AZStd::vector& TestCaseFailure::GetTestFailures() const + { + return m_testFailures; + } + + static size_t CalculateNumTestRunFailures(const AZStd::vector& testFailures) + { + size_t numTestFailures = 0; + for (const auto& testCase : testFailures) + { + numTestFailures += testCase.GetTestFailures().size(); + } + + return numTestFailures; + } + + TestRunWithTestFailures::TestRunWithTestFailures( + const AZStd::string& name, + const AZStd::string& commandString, + AZStd::chrono::high_resolution_clock::time_point startTime, + AZStd::chrono::milliseconds duration, + TestRunResult result, + AZStd::vector&& testFailures) + : TestRun(name, commandString, startTime, duration, result) + , m_testCaseFailures(AZStd::move(testFailures)) + { + m_numTestFailures = CalculateNumTestRunFailures(m_testCaseFailures); + } + + TestRunWithTestFailures::TestRunWithTestFailures(TestRun&& testRun, AZStd::vector&& testFailures) + : TestRun(AZStd::move(testRun)) + , m_testCaseFailures(AZStd::move(testFailures)) + { + m_numTestFailures = CalculateNumTestRunFailures(m_testCaseFailures); + } + + size_t TestRunWithTestFailures::GetNumTestFailures() const + { + return m_numTestFailures; + } + + const AZStd::vector& TestRunWithTestFailures::GetTestCaseFailures() const + { + return m_testCaseFailures; + } } // namespace Client } // namespace TestImpact diff --git a/Code/Tools/TestImpactFramework/Runtime/Code/Source/TestImpactRuntime.cpp b/Code/Tools/TestImpactFramework/Runtime/Code/Source/TestImpactRuntime.cpp index 57b6db1e8e..478b63e8b2 100644 --- a/Code/Tools/TestImpactFramework/Runtime/Code/Source/TestImpactRuntime.cpp +++ b/Code/Tools/TestImpactFramework/Runtime/Code/Source/TestImpactRuntime.cpp @@ -34,13 +34,33 @@ namespace TestImpact { } - //! Returns the time elapsed (in milliseconds) since the timer was instantiated - AZStd::chrono::milliseconds Elapsed() + //! Returns the time point that the timer was instantiated. + AZStd::chrono::high_resolution_clock::time_point GetStartTimePoint() const + { + return m_startTime; + } + + //! Returns the time point that the timer was instantiated relative to the specified starting time point. + AZStd::chrono::high_resolution_clock::time_point GetStartTimePointRelative(const Timer& start) const + { + return AZStd::chrono::high_resolution_clock::time_point() + + AZStd::chrono::duration_cast(m_startTime - start.GetStartTimePoint()); + } + + //! Returns the time elapsed (in milliseconds) since the timer was instantiated. + AZStd::chrono::milliseconds GetElapsedMs() const { const auto endTime = AZStd::chrono::high_resolution_clock::now(); return AZStd::chrono::duration_cast(endTime - m_startTime); } + //! Returns the current time point relative to the time point the timer was instantiated. + AZStd::chrono::high_resolution_clock::time_point GetElapsedTimepoint() const + { + const auto endTime = AZStd::chrono::high_resolution_clock::now(); + return m_startTime + AZStd::chrono::duration_cast(endTime - m_startTime); + } + private: AZStd::chrono::high_resolution_clock::time_point m_startTime; }; @@ -49,8 +69,11 @@ namespace TestImpact class TestRunCompleteCallbackHandler { public: - TestRunCompleteCallbackHandler(AZStd::optional testCompleteCallback) - : m_testCompleteCallback(testCompleteCallback) + TestRunCompleteCallbackHandler( + size_t totalTests, + AZStd::optional testCompleteCallback) + : m_totalTests(totalTests) + , m_testCompleteCallback(testCompleteCallback) { } @@ -58,19 +81,27 @@ namespace TestImpact { if (m_testCompleteCallback.has_value()) { - (*m_testCompleteCallback) - (Client::TestRun(testJob.GetTestTarget()->GetName(), testJob.GetTestResult(), testJob.GetDuration())); + Client::TestRun testRun( + testJob.GetTestTarget()->GetName(), + testJob.GetCommandString(), + testJob.GetStartTime(), + testJob.GetDuration(), + testJob.GetTestResult()); + + (*m_testCompleteCallback)(testRun, ++m_numTestsCompleted, m_totalTests); } } private: + const size_t m_totalTests; //!< The total number of tests to run for the entire sequence. + size_t m_numTestsCompleted = 0; //!< The running total of tests that have completed. AZStd::optional m_testCompleteCallback; }; } //! Utility for concatenating two vectors. template - AZStd::vector ConcatenateVectors(const AZStd::vector& v1, const AZStd::vector& v2) + static AZStd::vector ConcatenateVectors(const AZStd::vector& v1, const AZStd::vector& v2) { AZStd::vector result; result.reserve(v1.size() + v2.size()); @@ -298,6 +329,7 @@ namespace TestImpact continue; } + // Add the sources covered by this test target to the coverage map for (const auto& source : job.GetTestCoverge().value().GetSourcesCovered()) { coverage[source.String()].insert(job.GetTestTarget()->GetName()); @@ -309,6 +341,7 @@ namespace TestImpact sourceCoveringTests.reserve(coverage.size()); for (auto&& [source, testTargets] : coverage) { + // Check to see whether this source is inside the repo or not (not a perfect check but weeds out the obvious non-repo sources) if (const auto sourcePath = RepoPath(source); sourcePath.IsRelativeTo(m_config.m_repo.m_root)) { @@ -353,17 +386,17 @@ namespace TestImpact } } - TestSequenceResult Runtime::RegularTestSequence( + Client::SequenceReport Runtime::RegularTestSequence( AZStd::optional testTargetTimeout, AZStd::optional globalTimeout, AZStd::optional testSequenceStartCallback, - AZStd::optional testSequenceEndCallback, + AZStd::optional> testSequenceEndCallback, AZStd::optional testCompleteCallback) { - Timer timer; + const Timer sequenceTimer; AZStd::vector includedTestTargets; AZStd::vector excludedTestTargets; - + // Separate the test targets into those that are excluded by either the test filter or exclusion list and those that are not for (const auto& testTarget : m_dynamicDependencyMap->GetTestTargetList().GetTargets()) { @@ -378,12 +411,17 @@ namespace TestImpact } } - // Sequence start callback + // Extract the client facing representation of selected test targets + const Client::TestRunSelection selectedTests(ExtractTestTargetNames(includedTestTargets), ExtractTestTargetNames(excludedTestTargets)); + + // Inform the client that the sequence is about to start if (testSequenceStartCallback.has_value()) { - (*testSequenceStartCallback)(Client::TestRunSelection(ExtractTestTargetNames(includedTestTargets), ExtractTestTargetNames(excludedTestTargets))); + (*testSequenceStartCallback)(m_suiteFilter, selectedTests); } + // Run the test targets and collect the test run results + const Timer testRunTimer; const auto [result, testJobs] = m_testEngine->RegularRun( includedTestTargets, m_testShardingPolicy, @@ -392,27 +430,124 @@ namespace TestImpact m_targetOutputCapture, testTargetTimeout, globalTimeout, - TestRunCompleteCallbackHandler(testCompleteCallback)); + TestRunCompleteCallbackHandler(includedTestTargets.size(), testCompleteCallback)); + const auto testRunDuration = testRunTimer.GetElapsedMs(); + // Generate the sequence report for the client + const auto sequenceReport = Client::SequenceReport( + m_suiteFilter, + selectedTests, + GenerateTestRunReport(result, testRunTimer.GetStartTimePointRelative(sequenceTimer), testRunDuration, testJobs)); + + // Inform the client that the sequence has ended if (testSequenceEndCallback.has_value()) { - (*testSequenceEndCallback)(GenerateSequenceFailureReport(testJobs), timer.Elapsed()); + (*testSequenceEndCallback)(sequenceReport); } - return result; + return sequenceReport; } - TestSequenceResult Runtime::ImpactAnalysisTestSequence( + //! Wrapper for the impact analysis test sequence to handle both the updating and non-updating policies through a common pathway. + //! @tparam TestRunnerFunctor The functor for running the specified tests. + //! @tparam TestJob The test engine job type returned by the functor. + //! @param suiteType The suite type used for this sequence. + //! @param timer The timer to use for the test run timings. + //! @param testRunner The test runner functor to use for each of the test runs. + //! @param includedSelectedTestTargets The subset of test targets that were selected to run and not also fully excluded from running. + //! @param excludedSelectedTestTargets The subset of test targets that were selected to run but were fully excluded running. + //! @param discardedTestTargets The subset of test targets that were discarded from the test selection and will not be run. + //! @param globalTimeout The maximum duration the entire test sequence may run for (infinite if empty). + //! @param testSequenceStartCallback The client function to be called after the test targets have been selected but prior to running the tests. + //! @param testSequenceCompleteCallback The client function to be called after the test sequence has completed. + //! @param testRunCompleteCallback The client function to be called after an individual test run has completed. + //! @param updateCoverage The function to call to update the dynamic dependency map with test coverage (if any). + template + Client::ImpactAnalysisSequenceReport ImpactAnalysisTestSequenceWrapper( + SuiteType suiteType, + const Timer& sequenceTimer, + const TestRunnerFunctor& testRunner, + const AZStd::vector& includedSelectedTestTargets, + const AZStd::vector& excludedSelectedTestTargets, + const AZStd::vector& discardedTestTargets, + const AZStd::vector& draftedTestTargets, + const AZStd::optional globalTimeout, + AZStd::optional testSequenceStartCallback, + AZStd::optional> testSequenceEndCallback, + AZStd::optional testCompleteCallback, + AZStd::optional& jobs)>> updateCoverage) + { + AZStd::optional sequenceTimeout = globalTimeout; + + // Extract the client facing representation of selected, discarded and drafted test targets + const Client::TestRunSelection selectedTests( + ExtractTestTargetNames(includedSelectedTestTargets), ExtractTestTargetNames(excludedSelectedTestTargets)); + const auto discardedTests = ExtractTestTargetNames(discardedTestTargets); + const auto draftedTests = ExtractTestTargetNames(draftedTestTargets); + + // Inform the client that the sequence is about to start + if (testSequenceStartCallback.has_value()) + { + (*testSequenceStartCallback)(suiteType, selectedTests, discardedTests, draftedTests); + } + + // We share the test run complete handler between the selected and drafted test runs as to present them together as one + // continuous test sequence to the client rather than two discrete test runs + const size_t totalNumTestRuns = includedSelectedTestTargets.size() + draftedTestTargets.size(); + TestRunCompleteCallbackHandler testRunCompleteHandler(totalNumTestRuns, testCompleteCallback); + + // Run the selected test targets and collect the test run results + const Timer selectedTestRunTimer; + const auto [selectedResult, selectedTestJobs] = testRunner(includedSelectedTestTargets, testRunCompleteHandler, globalTimeout); + const auto selectedTestRunDuration = selectedTestRunTimer.GetElapsedMs(); + + // Carry the remaining global sequence time over to the drafted test run + if (globalTimeout.has_value()) + { + const auto elapsed = selectedTestRunDuration; + sequenceTimeout = elapsed < globalTimeout.value() ? globalTimeout.value() - elapsed : AZStd::chrono::milliseconds(0); + } + + // Run the drafted test targets and collect the test run results + Timer draftedTestRunTimer; + const auto [draftedResult, draftedTestJobs] = testRunner(draftedTestTargets, testRunCompleteHandler, globalTimeout); + const auto draftedTestRunDuration = draftedTestRunTimer.GetElapsedMs(); + + // Generate the sequence report for the client + const auto sequenceReport = Client::ImpactAnalysisSequenceReport( + suiteType, + selectedTests, + discardedTests, + draftedTests, + GenerateTestRunReport(selectedResult, selectedTestRunTimer.GetStartTimePointRelative(sequenceTimer), selectedTestRunDuration, selectedTestJobs), + GenerateTestRunReport(draftedResult, draftedTestRunTimer.GetStartTimePointRelative(sequenceTimer), draftedTestRunDuration, draftedTestJobs)); + + // Inform the client that the sequence has ended + if (testSequenceEndCallback.has_value()) + { + (*testSequenceEndCallback)(sequenceReport); + } + + // Update the dynamic dependency map with the latest coverage data (if any) + if (updateCoverage.has_value()) + { + (*updateCoverage)(ConcatenateVectors(selectedTestJobs, draftedTestJobs)); + } + + return sequenceReport; + } + + Client::ImpactAnalysisSequenceReport Runtime::ImpactAnalysisTestSequence( const ChangeList& changeList, Policy::TestPrioritization testPrioritizationPolicy, Policy::DynamicDependencyMap dynamicDependencyMapPolicy, AZStd::optional testTargetTimeout, AZStd::optional globalTimeout, AZStd::optional testSequenceStartCallback, - AZStd::optional testSequenceEndCallback, + AZStd::optional> testSequenceEndCallback, AZStd::optional testCompleteCallback) { - Timer timer; + const Timer sequenceTimer; // Draft in the test targets that have no coverage entries in the dynamic dependency map AZStd::vector draftedTestTargets = m_dynamicDependencyMap->GetNotCoveringTests(); @@ -423,71 +558,94 @@ namespace TestImpact // The subset of selected test targets that are not on the configuration's exclude list and those that are auto [includedSelectedTestTargets, excludedSelectedTestTargets] = SelectTestTargetsByExcludeList(selectedTestTargets); - // We present to the client the included selected test targets and the drafted test targets as distinct sets but internally - // we consider the concatenated set of the two the actual set of tests to run - AZStd::vector testTargetsToRun = ConcatenateVectors(includedSelectedTestTargets, draftedTestTargets); - - if (testSequenceStartCallback.has_value()) + // Functor for running instrumented test targets + const auto instrumentedTestRun = + [this, &testTargetTimeout]( + const AZStd::vector& testsTargets, + TestRunCompleteCallbackHandler& testRunCompleteHandler, + AZStd::optional globalTimeout) { - (*testSequenceStartCallback)( - Client::TestRunSelection(ExtractTestTargetNames(includedSelectedTestTargets), ExtractTestTargetNames(excludedSelectedTestTargets)), - ExtractTestTargetNames(discardedTestTargets), - ExtractTestTargetNames(draftedTestTargets)); - } + return m_testEngine->InstrumentedRun( + testsTargets, + m_testShardingPolicy, + m_executionFailurePolicy, + m_integrationFailurePolicy, + m_testFailurePolicy, + m_targetOutputCapture, + testTargetTimeout, + globalTimeout, + AZStd::ref(testRunCompleteHandler)); + }; + + // Functor for running uninstrumented test targets + const auto regularTestRun = + [this, &testTargetTimeout]( + const AZStd::vector& testsTargets, + TestRunCompleteCallbackHandler& testRunCompleteHandler, + AZStd::optional globalTimeout) + { + return m_testEngine->RegularRun( + testsTargets, + m_testShardingPolicy, + m_executionFailurePolicy, + m_testFailurePolicy, + m_targetOutputCapture, + testTargetTimeout, + globalTimeout, + AZStd::ref(testRunCompleteHandler)); + }; if (dynamicDependencyMapPolicy == Policy::DynamicDependencyMap::Update) { - const auto [result, testJobs] = m_testEngine->InstrumentedRun( - testTargetsToRun, - m_testShardingPolicy, - m_executionFailurePolicy, - Policy::IntegrityFailure::Continue, - m_testFailurePolicy, - m_targetOutputCapture, - testTargetTimeout, - globalTimeout, - TestRunCompleteCallbackHandler(testCompleteCallback)); - - UpdateAndSerializeDynamicDependencyMap(testJobs); - - if (testSequenceEndCallback.has_value()) + AZStd::optional& jobs)>> updateCoverage = + [this](const AZStd::vector& jobs) { - (*testSequenceEndCallback)(GenerateSequenceFailureReport(testJobs), timer.Elapsed()); - } + UpdateAndSerializeDynamicDependencyMap(jobs); + }; - return result; + return ImpactAnalysisTestSequenceWrapper( + m_suiteFilter, + sequenceTimer, + instrumentedTestRun, + includedSelectedTestTargets, + excludedSelectedTestTargets, + discardedTestTargets, + draftedTestTargets, + globalTimeout, + testSequenceStartCallback, + testSequenceEndCallback, + testCompleteCallback, + updateCoverage); } else { - const auto [result, testJobs] = m_testEngine->RegularRun( - testTargetsToRun, - m_testShardingPolicy, - m_executionFailurePolicy, - m_testFailurePolicy, - m_targetOutputCapture, - testTargetTimeout, + return ImpactAnalysisTestSequenceWrapper( + m_suiteFilter, + sequenceTimer, + regularTestRun, + includedSelectedTestTargets, + excludedSelectedTestTargets, + discardedTestTargets, + draftedTestTargets, globalTimeout, - TestRunCompleteCallbackHandler(testCompleteCallback)); - - if (testSequenceEndCallback.has_value()) - { - (*testSequenceEndCallback)(GenerateSequenceFailureReport(testJobs), timer.Elapsed()); - } - - return result; + testSequenceStartCallback, + testSequenceEndCallback, + testCompleteCallback, + AZStd::optional& jobs)>>{ AZStd::nullopt }); } } - AZStd::pair Runtime::SafeImpactAnalysisTestSequence( + Client::SafeImpactAnalysisSequenceReport Runtime::SafeImpactAnalysisTestSequence( const ChangeList& changeList, Policy::TestPrioritization testPrioritizationPolicy, AZStd::optional testTargetTimeout, AZStd::optional globalTimeout, AZStd::optional testSequenceStartCallback, - AZStd::optional testSequenceEndCallback, + AZStd::optional> testSequenceEndCallback, AZStd::optional testCompleteCallback) { - Timer timer; + const Timer sequenceTimer; + auto sequenceTimeout = globalTimeout; // Draft in the test targets that have no coverage entries in the dynamic dependency map AZStd::vector draftedTestTargets = m_dynamicDependencyMap->GetNotCoveringTests(); @@ -501,40 +659,46 @@ namespace TestImpact // The subset of discarded test targets that are not on the configuration's exclude list and those that are auto [includedDiscardedTestTargets, excludedDiscardedTestTargets] = SelectTestTargetsByExcludeList(discardedTestTargets); - // We present to the client the included selected test targets and the drafted test targets as distinct sets but internally - // we consider the concatenated set of the two the actual set of tests to run - AZStd::vector testTargetsToRun = ConcatenateVectors(includedSelectedTestTargets, draftedTestTargets); + // Extract the client facing representation of selected, discarded and drafted test targets + const Client::TestRunSelection selectedTests( + ExtractTestTargetNames(includedSelectedTestTargets), ExtractTestTargetNames(excludedSelectedTestTargets)); + const Client::TestRunSelection discardedTests(ExtractTestTargetNames(includedDiscardedTestTargets), ExtractTestTargetNames(excludedDiscardedTestTargets)); + const auto draftedTests = ExtractTestTargetNames(draftedTestTargets); + // Inform the client that the sequence is about to start if (testSequenceStartCallback.has_value()) { - (*testSequenceStartCallback)( - Client::TestRunSelection(ExtractTestTargetNames(includedSelectedTestTargets), ExtractTestTargetNames(excludedSelectedTestTargets)), - Client::TestRunSelection(ExtractTestTargetNames(includedDiscardedTestTargets), ExtractTestTargetNames(excludedDiscardedTestTargets)), - ExtractTestTargetNames(draftedTestTargets)); + (*testSequenceStartCallback)(m_suiteFilter, selectedTests, discardedTests, draftedTests); } - // Impact analysis run of the selected test targets + // We share the test run complete handler between the selected, discarded and drafted test runs as to present them together as one + // continuous test sequence to the client rather than three discrete test runs + const size_t totalNumTestRuns = includedSelectedTestTargets.size() + draftedTestTargets.size() + includedDiscardedTestTargets.size(); + TestRunCompleteCallbackHandler testRunCompleteHandler(totalNumTestRuns, testCompleteCallback); + + // Run the selected test targets and collect the test run results + const Timer selectedTestRunTimer; const auto [selectedResult, selectedTestJobs] = m_testEngine->InstrumentedRun( - testTargetsToRun, + includedSelectedTestTargets, m_testShardingPolicy, m_executionFailurePolicy, - Policy::IntegrityFailure::Continue, + m_integrationFailurePolicy, m_testFailurePolicy, m_targetOutputCapture, testTargetTimeout, - globalTimeout, - TestRunCompleteCallbackHandler(testCompleteCallback)); - - const auto selectedDuraton = timer.Elapsed(); + sequenceTimeout, + AZStd::ref(testRunCompleteHandler)); + const auto selectedTestRunDuration = selectedTestRunTimer.GetElapsedMs(); // Carry the remaining global sequence time over to the discarded test run if (globalTimeout.has_value()) { - const auto elapsed = timer.Elapsed(); - globalTimeout = elapsed < globalTimeout.value() ? globalTimeout.value() - elapsed : AZStd::chrono::milliseconds(0); + const auto elapsed = selectedTestRunDuration; + sequenceTimeout = elapsed < globalTimeout.value() ? globalTimeout.value() - elapsed : AZStd::chrono::milliseconds(0); } - // Regular run of the discarded test targets + // Run the discarded test targets and collect the test run results + const Timer discardedTestRunTimer; const auto [discardedResult, discardedTestJobs] = m_testEngine->RegularRun( includedDiscardedTestTargets, m_testShardingPolicy, @@ -542,35 +706,63 @@ namespace TestImpact m_testFailurePolicy, m_targetOutputCapture, testTargetTimeout, - globalTimeout, - TestRunCompleteCallbackHandler(testCompleteCallback)); + sequenceTimeout, + AZStd::ref(testRunCompleteHandler)); + const auto discardedTestRunDuration = discardedTestRunTimer.GetElapsedMs(); - const auto discardedDuraton = timer.Elapsed(); - - if (testSequenceEndCallback.has_value()) + // Carry the remaining global sequence time over to the drafted test run + if (globalTimeout.has_value()) { - (*testSequenceEndCallback)( - GenerateSequenceFailureReport(selectedTestJobs), - GenerateSequenceFailureReport(discardedTestJobs), - selectedDuraton, - discardedDuraton); + const auto elapsed = selectedTestRunDuration + discardedTestRunDuration; + sequenceTimeout = elapsed < globalTimeout.value() ? globalTimeout.value() - elapsed : AZStd::chrono::milliseconds(0); } - UpdateAndSerializeDynamicDependencyMap(selectedTestJobs); - return { selectedResult, discardedResult }; + // Run the drafted test targets and collect the test run results + const Timer draftedTestRunTimer; + const auto [draftedResult, draftedTestJobs] = m_testEngine->InstrumentedRun( + draftedTestTargets, + m_testShardingPolicy, + m_executionFailurePolicy, + m_integrationFailurePolicy, + m_testFailurePolicy, + m_targetOutputCapture, + testTargetTimeout, + sequenceTimeout, + AZStd::ref(testRunCompleteHandler)); + const auto draftedTestRunDuration = draftedTestRunTimer.GetElapsedMs(); + + // Generate the sequence report for the client + const auto sequenceReport = Client::SafeImpactAnalysisSequenceReport( + m_suiteFilter, + selectedTests, + discardedTests, + draftedTests, + GenerateTestRunReport(selectedResult, selectedTestRunTimer.GetStartTimePointRelative(sequenceTimer), selectedTestRunDuration, selectedTestJobs), + GenerateTestRunReport(discardedResult, discardedTestRunTimer.GetStartTimePointRelative(sequenceTimer), discardedTestRunDuration, discardedTestJobs), + GenerateTestRunReport(draftedResult, draftedTestRunTimer.GetStartTimePointRelative(sequenceTimer), draftedTestRunDuration, draftedTestJobs)); + + // Inform the client that the sequence has ended + if (testSequenceEndCallback.has_value()) + { + (*testSequenceEndCallback)(sequenceReport); + } + + UpdateAndSerializeDynamicDependencyMap(ConcatenateVectors(selectedTestJobs, draftedTestJobs)); + return sequenceReport; } - TestSequenceResult Runtime::SeededTestSequence( + Client::SequenceReport Runtime::SeededTestSequence( AZStd::optional testTargetTimeout, AZStd::optional globalTimeout, AZStd::optional testSequenceStartCallback, - AZStd::optional testSequenceEndCallback, + AZStd::optional> testSequenceEndCallback, AZStd::optional testCompleteCallback) { - Timer timer; + const Timer sequenceTimer; AZStd::vector includedTestTargets; AZStd::vector excludedTestTargets; + // Separate the test targets into those that are excluded by either the test filter or exclusion list and those that are not for (const auto& testTarget : m_dynamicDependencyMap->GetTestTargetList().GetTargets()) { if (!m_testTargetExcludeList.contains(&testTarget)) @@ -583,31 +775,44 @@ namespace TestImpact } } + // Extract the client facing representation of selected test targets + Client::TestRunSelection selectedTests(ExtractTestTargetNames(includedTestTargets), ExtractTestTargetNames(excludedTestTargets)); + + // Inform the client that the sequence is about to start if (testSequenceStartCallback.has_value()) { - (*testSequenceStartCallback)(Client::TestRunSelection(ExtractTestTargetNames(includedTestTargets), ExtractTestTargetNames(excludedTestTargets))); + (*testSequenceStartCallback)(m_suiteFilter, selectedTests); } + // Run the test targets and collect the test run results + const Timer testRunTimer; const auto [result, testJobs] = m_testEngine->InstrumentedRun( includedTestTargets, m_testShardingPolicy, m_executionFailurePolicy, - Policy::IntegrityFailure::Continue, + m_integrationFailurePolicy, m_testFailurePolicy, m_targetOutputCapture, testTargetTimeout, globalTimeout, - TestRunCompleteCallbackHandler(testCompleteCallback)); + TestRunCompleteCallbackHandler(includedTestTargets.size(), testCompleteCallback)); + const auto testRunDuration = testRunTimer.GetElapsedMs(); + // Generate the sequence report for the client + const auto sequenceReport = Client::SequenceReport( + m_suiteFilter, + selectedTests, + GenerateTestRunReport(result, testRunTimer.GetStartTimePointRelative(sequenceTimer), testRunDuration, testJobs)); + + // Inform the client that the sequence has ended if (testSequenceEndCallback.has_value()) { - (*testSequenceEndCallback)(GenerateSequenceFailureReport(testJobs), timer.Elapsed()); + (*testSequenceEndCallback)(sequenceReport); } ClearDynamicDependencyMapAndRemoveExistingFile(); UpdateAndSerializeDynamicDependencyMap(testJobs); - - return result; + return sequenceReport; } bool Runtime::HasImpactAnalysisData() const diff --git a/Code/Tools/TestImpactFramework/Runtime/Code/Source/TestImpactRuntimeUtils.cpp b/Code/Tools/TestImpactFramework/Runtime/Code/Source/TestImpactRuntimeUtils.cpp index 10a77a2e58..0c61c5f426 100644 --- a/Code/Tools/TestImpactFramework/Runtime/Code/Source/TestImpactRuntimeUtils.cpp +++ b/Code/Tools/TestImpactFramework/Runtime/Code/Source/TestImpactRuntimeUtils.cpp @@ -24,13 +24,13 @@ namespace TestImpact return TestTargetMetaMapFactory(masterTestListData, suiteFilter); } - AZStd::vector ReadBuildTargetDescriptorFiles(const BuildTargetDescriptorConfig& buildTargetDescriptorConfig) + AZStd::vector ReadBuildTargetDescriptorFiles(const BuildTargetDescriptorConfig& buildTargetDescriptorConfig) { - AZStd::vector buildTargetDescriptors; + AZStd::vector buildTargetDescriptors; for (const auto& buildTargetDescriptorFile : std::filesystem::directory_iterator(buildTargetDescriptorConfig.m_mappingDirectory.c_str())) { const auto buildTargetDescriptorContents = ReadFileContents(buildTargetDescriptorFile.path().string().c_str()); - auto buildTargetDescriptor = TestImpact::BuildTargetDescriptorFactory( + auto buildTargetDescriptor = BuildTargetDescriptorFactory( buildTargetDescriptorContents, buildTargetDescriptorConfig.m_staticInclusionFilters, buildTargetDescriptorConfig.m_inputInclusionFilters, @@ -41,7 +41,7 @@ namespace TestImpact return buildTargetDescriptors; } - AZStd::unique_ptr ConstructDynamicDependencyMap( + AZStd::unique_ptr ConstructDynamicDependencyMap( SuiteType suiteFilter, const BuildTargetDescriptorConfig& buildTargetDescriptorConfig, const TestTargetMetaConfig& testTargetMetaConfig) @@ -50,7 +50,7 @@ namespace TestImpact auto buildTargetDescriptors = ReadBuildTargetDescriptorFiles(buildTargetDescriptorConfig); auto buildTargets = CompileTargetDescriptors(AZStd::move(buildTargetDescriptors), AZStd::move(testTargetmetaMap)); auto&& [productionTargets, testTargets] = buildTargets; - return AZStd::make_unique(AZStd::move(productionTargets), AZStd::move(testTargets)); + return AZStd::make_unique(AZStd::move(productionTargets), AZStd::move(testTargets)); } AZStd::unordered_set ConstructTestTargetExcludeList( @@ -68,7 +68,7 @@ namespace TestImpact return testTargetExcludeList; } - AZStd::vector ExtractTestTargetNames(const AZStd::vector testTargets) + AZStd::vector ExtractTestTargetNames(const AZStd::vector& testTargets) { AZStd::vector testNames; AZStd::transform(testTargets.begin(), testTargets.end(), AZStd::back_inserter(testNames), [](const TestTarget* testTarget) diff --git a/Code/Tools/TestImpactFramework/Runtime/Code/Source/TestImpactRuntimeUtils.h b/Code/Tools/TestImpactFramework/Runtime/Code/Source/TestImpactRuntimeUtils.h index 608888726e..33e15bfd20 100644 --- a/Code/Tools/TestImpactFramework/Runtime/Code/Source/TestImpactRuntimeUtils.h +++ b/Code/Tools/TestImpactFramework/Runtime/Code/Source/TestImpactRuntimeUtils.h @@ -10,7 +10,7 @@ #include #include -#include +#include #include #include @@ -25,7 +25,7 @@ namespace TestImpact { //! Construct a dynamic dependency map from the build target descriptors and test target metas. - AZStd::unique_ptr ConstructDynamicDependencyMap( + AZStd::unique_ptr ConstructDynamicDependencyMap( SuiteType suiteFilter, const BuildTargetDescriptorConfig& buildTargetDescriptorConfig, const TestTargetMetaConfig& testTargetMetaConfig); @@ -36,16 +36,17 @@ namespace TestImpact const AZStd::vector& excludedTestTargets); //! Extracts the name information from the specified test targets. - AZStd::vector ExtractTestTargetNames(const AZStd::vector testTargets); + AZStd::vector ExtractTestTargetNames(const AZStd::vector& testTargets); //! Generates a test run failure report from the specified test engine job information. //! @tparam TestJob The test engine job type. template - Client::TestRunFailure GenerateTestRunFailure(const TestJob& testJob) + AZStd::vector GenerateTestCaseFailures(const TestJob& testJob) { + AZStd::vector testCaseFailures; + if (testJob.GetTestRun().has_value()) { - AZStd::vector testCaseFailures; for (const auto& testSuite : testJob.GetTestRun()->GetTestSuites()) { AZStd::vector testFailures; @@ -56,57 +57,66 @@ namespace TestImpact testFailures.push_back(Client::TestFailure(testCase.m_name, "No error message retrieved")); } } - + if (!testFailures.empty()) { testCaseFailures.push_back(Client::TestCaseFailure(testSuite.m_name, AZStd::move(testFailures))); } } + } - return Client::TestRunFailure(Client::TestRunFailure(testJob.GetTestTarget()->GetName(), AZStd::move(testCaseFailures))); - } - else - { - return Client::TestRunFailure(testJob.GetTestTarget()->GetName(), { }); - } + return testCaseFailures; } - //! Generates a sequence failure report from the specified list of test engine jobs. - //! @tparam TestJob The test engine job type. template - Client::SequenceFailure GenerateSequenceFailureReport(const AZStd::vector& testJobs) + Client::TestRunReport GenerateTestRunReport( + TestSequenceResult result, + AZStd::chrono::high_resolution_clock::time_point startTime, + AZStd::chrono::milliseconds duration, + const AZStd::vector& testJobs) { - AZStd::vector executionFailures; - AZStd::vector testRunFailures; - AZStd::vector timedOutTestRuns; - AZStd::vector unexecutedTestRuns; - + AZStd::vector passingTests; + AZStd::vector failingTests; + AZStd::vector executionFailureTests; + AZStd::vector timedOutTests; + AZStd::vector unexecutedTests; + for (const auto& testJob : testJobs) { + // Test job start time relative to start time + const auto relativeStartTime = + AZStd::chrono::high_resolution_clock::time_point() + + AZStd::chrono::duration_cast(testJob.GetStartTime() - startTime); + + Client::TestRun clientTestRun( + testJob.GetTestTarget()->GetName(), testJob.GetCommandString(), relativeStartTime, testJob.GetDuration(), + testJob.GetTestResult()); + switch (testJob.GetTestResult()) { case Client::TestRunResult::FailedToExecute: { - executionFailures.push_back(Client::ExecutionFailure(testJob.GetTestTarget()->GetName(), testJob.GetCommandString())); + executionFailureTests.push_back(clientTestRun); break; } case Client::TestRunResult::NotRun: { - unexecutedTestRuns.push_back(testJob.GetTestTarget()->GetName()); + unexecutedTests.push_back(clientTestRun); break; } case Client::TestRunResult::Timeout: { - timedOutTestRuns.push_back(testJob.GetTestTarget()->GetName()); + timedOutTests.push_back(clientTestRun); break; } case Client::TestRunResult::AllTestsPass: { + passingTests.push_back(clientTestRun); break; } case Client::TestRunResult::TestFailures: { - testRunFailures.push_back(GenerateTestRunFailure(testJob)); + failingTests.emplace_back(AZStd::move(clientTestRun), GenerateTestCaseFailures(testJob)); break; } default: @@ -116,11 +126,15 @@ namespace TestImpact } } } - - return Client::SequenceFailure( - AZStd::move(executionFailures), - AZStd::move(testRunFailures), - AZStd::move(timedOutTestRuns), - AZStd::move(unexecutedTestRuns)); + + return Client::TestRunReport( + result, + startTime, + duration, + AZStd::move(passingTests), + AZStd::move(failingTests), + AZStd::move(executionFailureTests), + AZStd::move(timedOutTests), + AZStd::move(unexecutedTests)); } -} +} // namespace TestImpact diff --git a/Code/Tools/TestImpactFramework/Runtime/Code/testimpactframework_runtime_files.cmake b/Code/Tools/TestImpactFramework/Runtime/Code/testimpactframework_runtime_files.cmake index 090bb68fcc..e28bb5ba7b 100644 --- a/Code/Tools/TestImpactFramework/Runtime/Code/testimpactframework_runtime_files.cmake +++ b/Code/Tools/TestImpactFramework/Runtime/Code/testimpactframework_runtime_files.cmake @@ -19,7 +19,7 @@ set(FILES Include/TestImpactFramework/TestImpactTestSequence.h Include/TestImpactFramework/TestImpactClientTestSelection.h Include/TestImpactFramework/TestImpactClientTestRun.h - Include/TestImpactFramework/TestImpactClientFailureReport.h + Include/TestImpactFramework/TestImpactClientSequenceReport.h Include/TestImpactFramework/TestImpactFileUtils.h Source/Artifact/TestImpactArtifactException.h Source/Artifact/Factory/TestImpactBuildTargetDescriptorFactory.cpp @@ -123,7 +123,8 @@ set(FILES Source/TestImpactRuntimeUtils.h Source/TestImpactClientTestSelection.cpp Source/TestImpactClientTestRun.cpp - Source/TestImpactClientFailureReport.cpp + Source/TestImpactClientSequenceReport.cpp Source/TestImpactChangeListSerializer.cpp + Source/TestImpactChangeListSerializerInternal.h Source/TestImpactRepoPath.cpp )