birkeh/o3de
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Add previously approved changes.

Browse Source 
Signed-off-by: John <jonawals@amazon.com>
This commit is contained in:
John
2021-08-06 20:09:53 +01:00
parent ca3bc3d343
commit d7c1185dc2
17 changed files with 1287 additions and 599 deletions
Download Patch File Download Diff File Expand all files Collapse all files
Code/Tools/TestImpactFramework/Frontend/Console/Code/Source/TestImpactConsoleMain.cpp
+37 -52
View File
@@ -77,7 +77,6 @@ namespace TestImpact
//! Wrapper around impact analysis sequences to handle the case where the safe mode option is active.
ReturnCode WrappedImpactAnalysisTestSequence(
TestSequenceEventHandler& sequenceEventHandler,
const CommandLineOptions& options,
Runtime& runtime,
const AZStd::optional<ChangeList>& changeList)
@@ -94,44 +93,30 @@ namespace TestImpact
{
if (options.GetTestSequenceType() == TestSequenceType::ImpactAnalysis)
{
auto [selectedResult, discardedResult] = runtime.SafeImpactAnalysisTestSequence(
auto safeImpactAnalysisSequenceReport = runtime.SafeImpactAnalysisTestSequence(
changeList.value(),
options.GetTestPrioritizationPolicy(),
options.GetTestTargetTimeout(),
options.GetGlobalTimeout(),
AZStd::ref(sequenceEventHandler),
AZStd::ref(sequenceEventHandler),
AZStd::ref(sequenceEventHandler));
// Handling the possible timeout and failure permutations of the selected and discarded test results is splitting hairs
// so apply the following, admittedly arbitrary, rules to determine what the composite test sequence result should be
if (selectedResult == TestSequenceResult::Success && discardedResult == TestSequenceResult::Success)
{
// Trivial case: both sequences succeeded
result = TestSequenceResult::Success;
}
else if (selectedResult == TestSequenceResult::Failure || discardedResult == TestSequenceResult::Failure)
{
// One sequence failed whilst the other sequence either succeeded or timed out
result = TestSequenceResult::Failure;
}
else
{
// One or both sequences timed out or failed
result = TestSequenceResult::Timeout;
}
SafeImpactAnalysisTestSequenceStartCallback,
SafeImpactAnalysisTestSequenceCompleteCallback,
TestRunCompleteCallback);
result = safeImpactAnalysisSequenceReport.GetResult();
}
else if (options.GetTestSequenceType() == TestSequenceType::ImpactAnalysisNoWrite)
{
// A no-write impact analysis sequence with safe mode enabled is functionally identical to a regular sequence type
// due to a) the selected tests being run without instrumentation and b) the discarded tests also being run without
// instrumentation
result = runtime.RegularTestSequence(
auto sequenceReport = runtime.RegularTestSequence(
options.GetTestTargetTimeout(),
options.GetGlobalTimeout(),
AZStd::ref(sequenceEventHandler),
AZStd::ref(sequenceEventHandler),
AZStd::ref(sequenceEventHandler));
TestSequenceStartCallback,
TestSequenceCompleteCallback,
TestRunCompleteCallback);
result = sequenceReport.GetResult();
}
else
{
@@ -153,18 +138,20 @@ namespace TestImpact
{
throw(Exception("Unexpected sequence type"));
}
result = runtime.ImpactAnalysisTestSequence(
auto impactAnalysisSequenceReport = runtime.ImpactAnalysisTestSequence(
changeList.value(),
options.GetTestPrioritizationPolicy(),
dynamicDependencyMapPolicy,
options.GetTestTargetTimeout(),
options.GetGlobalTimeout(),
AZStd::ref(sequenceEventHandler),
AZStd::ref(sequenceEventHandler),
AZStd::ref(sequenceEventHandler));
}
ImpactAnalysisTestSequenceStartCallback,
ImpactAnalysisTestSequenceCompleteCallback,
TestRunCompleteCallback);
result = impactAnalysisSequenceReport.GetResult();
}
return GetReturnCodeForTestSequenceResult(result);
};
@@ -217,53 +204,51 @@ namespace TestImpact
std::cout << "Test impact analysis data for this repository was not found, seed or regular sequence fallbacks will be used.\n";
}
TestSequenceEventHandler sequenceEventHandler(options.GetSuiteFilter());
switch (const auto type = options.GetTestSequenceType())
{
case TestSequenceType::Regular:
{
const auto result = runtime.RegularTestSequence(
const auto sequenceReport = runtime.RegularTestSequence(
options.GetTestTargetTimeout(),
options.GetGlobalTimeout(),
AZStd::ref(sequenceEventHandler),
AZStd::ref(sequenceEventHandler),
AZStd::ref(sequenceEventHandler));
TestSequenceStartCallback,
TestSequenceCompleteCallback,
TestRunCompleteCallback);
return GetReturnCodeForTestSequenceResult(result);
return GetReturnCodeForTestSequenceResult(sequenceReport.GetResult());
}
case TestSequenceType::Seed:
{
const auto result = runtime.SeededTestSequence(
const auto sequenceReport = runtime.SeededTestSequence(
options.GetTestTargetTimeout(),
options.GetGlobalTimeout(),
AZStd::ref(sequenceEventHandler),
AZStd::ref(sequenceEventHandler),
AZStd::ref(sequenceEventHandler));
TestSequenceStartCallback,
TestSequenceCompleteCallback,
TestRunCompleteCallback);
return GetReturnCodeForTestSequenceResult(result);
return GetReturnCodeForTestSequenceResult(sequenceReport.GetResult());
}
case TestSequenceType::ImpactAnalysisNoWrite:
case TestSequenceType::ImpactAnalysis:
{
return WrappedImpactAnalysisTestSequence(sequenceEventHandler, options, runtime, changeList);
return WrappedImpactAnalysisTestSequence(options, runtime, changeList);
}
case TestSequenceType::ImpactAnalysisOrSeed:
{
if (runtime.HasImpactAnalysisData())
{
return WrappedImpactAnalysisTestSequence(sequenceEventHandler, options, runtime, changeList);
return WrappedImpactAnalysisTestSequence(options, runtime, changeList);
}
else
{
const auto result = runtime.SeededTestSequence(
const auto sequenceReport = runtime.SeededTestSequence(
options.GetTestTargetTimeout(),
options.GetGlobalTimeout(),
AZStd::ref(sequenceEventHandler),
AZStd::ref(sequenceEventHandler),
AZStd::ref(sequenceEventHandler));
TestSequenceStartCallback,
TestSequenceCompleteCallback,
TestRunCompleteCallback);
return GetReturnCodeForTestSequenceResult(result);
return GetReturnCodeForTestSequenceResult(sequenceReport.GetResult());
}
}
default:
Code/Tools/TestImpactFramework/Frontend/Console/Code/Source/TestImpactConsoleTestSequenceEventHandler.cpp
+71 -74
View File
@@ -32,72 +32,73 @@ namespace TestImpact
std::cout << "Of which " << numExcludedTests << " tests have been excluded and " << numDraftedTests << " tests have been drafted.\n";
}
void FailureReport(const Client::SequenceFailure& failureReport, AZStd::chrono::milliseconds duration)
void FailureReport(const Client::TestRunReport& testRunReport)
{
std::cout << "Sequence completed in " << (duration.count() / 1000.f) << "s with";
std::cout << "Sequence completed in " << (testRunReport.GetDuration().count() / 1000.f) << "s with";
if (!failureReport.GetExecutionFailures().empty() ||
!failureReport.GetTestRunFailures().empty() ||
!failureReport.GetTimedOutTests().empty() ||
!failureReport.GetUnexecutedTests().empty())
if (!testRunReport.GetExecutionFailureTests().empty() ||
!testRunReport.GetFailingTests().empty() ||
!testRunReport.GetTimedOutTests().empty() ||
!testRunReport.GetUnexecutedTests().empty())
{
std::cout << ":\n";
std::cout << SetColor(Foreground::White, Background::Red).c_str()
<< failureReport.GetTestRunFailures().size()
<< testRunReport.GetFailingTests().size()
<< ResetColor().c_str() << " test failures\n";
std::cout << SetColor(Foreground::White, Background::Red).c_str()
<< failureReport.GetExecutionFailures().size()
<< testRunReport.GetExecutionFailureTests().size()
<< ResetColor().c_str() << " execution failures\n";
std::cout << SetColor(Foreground::White, Background::Red).c_str()
<< failureReport.GetTimedOutTests().size()
<< testRunReport.GetTimedOutTests().size()
<< ResetColor().c_str() << " test timeouts\n";
std::cout << SetColor(Foreground::White, Background::Red).c_str()
<< failureReport.GetUnexecutedTests().size()
<< testRunReport.GetUnexecutedTests().size()
<< ResetColor().c_str() << " unexecuted tests\n";
if (!failureReport.GetTestRunFailures().empty())
if (!testRunReport.GetFailingTests().empty())
{
std::cout << "\nTest failures:\n";
for (const auto& testRunFailure : failureReport.GetTestRunFailures())
for (const auto& testRunFailure : testRunReport.GetFailingTests())
{
std::cout << " " << testRunFailure.GetTargetName().c_str();
for (const auto& testCaseFailure : testRunFailure.GetTestCaseFailures())
{
std::cout << "." << testCaseFailure.GetName().c_str();
for (const auto& testFailure : testCaseFailure.GetTestFailures())
{
std::cout << "." << testFailure.GetName().c_str() << "\n";
std::cout << " "
<< testRunFailure.GetTargetName().c_str()
<< "." << testCaseFailure.GetName().c_str()
<< "." << testFailure.GetName().c_str() << "\n";
}
}
}
}
if (!failureReport.GetExecutionFailures().empty())
if (!testRunReport.GetExecutionFailureTests().empty())
{
std::cout << "\nExecution failures:\n";
for (const auto& executionFailure : failureReport.GetExecutionFailures())
for (const auto& executionFailure : testRunReport.GetExecutionFailureTests())
{
std::cout << " " << executionFailure.GetTargetName().c_str() << "\n";
std::cout << executionFailure.GetCommandString().c_str() << "\n";
}
}
if (!failureReport.GetTimedOutTests().empty())
if (!testRunReport.GetTimedOutTests().empty())
{
std::cout << "\nTimed out tests:\n";
for (const auto& testTimeout : failureReport.GetTimedOutTests())
for (const auto& testTimeout : testRunReport.GetTimedOutTests())
{
std::cout << " " << testTimeout.GetTargetName().c_str() << "\n";
}
}
if (!failureReport.GetUnexecutedTests().empty())
if (!testRunReport.GetUnexecutedTests().empty())
{
std::cout << "\nUnexecuted tests:\n";
for (const auto& unexecutedTest : failureReport.GetUnexecutedTests())
for (const auto& unexecutedTest : testRunReport.GetUnexecutedTests())
{
std::cout << " " << unexecutedTest.GetTargetName().c_str() << "\n";
}
@@ -105,50 +106,42 @@ namespace TestImpact
}
else
{
std::cout << SetColor(Foreground::White, Background::Green).c_str() << " \100% passes!\n" << ResetColor().c_str();
std::cout << SetColor(Foreground::White, Background::Green).c_str() << " \100% passes!\n" << ResetColor().c_str() << "\n";
}
}
}
TestSequenceEventHandler::TestSequenceEventHandler(SuiteType suiteFilter)
: m_suiteFilter(suiteFilter)
void TestSequenceStartCallback(SuiteType suiteType, const Client::TestRunSelection& selectedTests)
{
Output::TestSuiteFilter(suiteType);
std::cout << selectedTests.GetNumIncludedTestRuns() << " tests selected, " << selectedTests.GetNumExcludedTestRuns()
<< " excluded.\n";
}
// TestSequenceStartCallback
void TestSequenceEventHandler::operator()(Client::TestRunSelection&& selectedTests)
void TestSequenceCompleteCallback(SuiteType suiteType, const Client::TestRunSelection& selectedTests)
{
ClearState();
m_numTests = selectedTests.GetNumIncludedTestRuns();
Output::TestSuiteFilter(m_suiteFilter);
Output::TestSuiteFilter(suiteType);
std::cout << selectedTests.GetNumIncludedTestRuns() << " tests selected, " << selectedTests.GetNumExcludedTestRuns() << " excluded.\n";
}
// ImpactAnalysisTestSequenceStartCallback
void TestSequenceEventHandler::operator()(
Client::TestRunSelection&& selectedTests,
AZStd::vector<AZStd::string>&& discardedTests,
AZStd::vector<AZStd::string>&& draftedTests)
void ImpactAnalysisTestSequenceStartCallback(
SuiteType suiteType,
const Client::TestRunSelection& selectedTests,
const AZStd::vector<AZStd::string>& discardedTests,
const AZStd::vector<AZStd::string>& draftedTests)
{
ClearState();
m_numTests = selectedTests.GetNumIncludedTestRuns() + draftedTests.size();
Output::TestSuiteFilter(m_suiteFilter);
Output::TestSuiteFilter(suiteType);
Output::ImpactAnalysisTestSelection(
selectedTests.GetTotalNumTests(), discardedTests.size(), selectedTests.GetNumExcludedTestRuns(), draftedTests.size());
}
// SafeImpactAnalysisTestSequenceStartCallback
void TestSequenceEventHandler::operator()(
Client::TestRunSelection&& selectedTests,
Client::TestRunSelection&& discardedTests,
AZStd::vector<AZStd::string>&& draftedTests)
void SafeImpactAnalysisTestSequenceStartCallback(
SuiteType suiteType,
const Client::TestRunSelection& selectedTests,
const Client::TestRunSelection& discardedTests,
const AZStd::vector<AZStd::string>& draftedTests)
{
ClearState();
m_numTests = selectedTests.GetNumIncludedTestRuns() + draftedTests.size();
Output::TestSuiteFilter(m_suiteFilter);
Output::TestSuiteFilter(suiteType);
Output::ImpactAnalysisTestSelection(
selectedTests.GetTotalNumTests(),
discardedTests.GetTotalNumTests(),
@@ -156,40 +149,45 @@ namespace TestImpact
draftedTests.size());
}
// TestSequenceCompleteCallback
void TestSequenceEventHandler::operator()(
Client::SequenceFailure&& failureReport,
AZStd::chrono::milliseconds duration)
void TestSequenceCompleteCallback(const Client::SequenceReport& sequenceReport)
{
Output::FailureReport(failureReport, duration);
Output::FailureReport(sequenceReport.GetSelectedTestRunReport());
std::cout << "Updating and serializing the test impact analysis data, this may take a moment...\n";
}
// SafeTestSequenceCompleteCallback
void TestSequenceEventHandler::operator()(
Client::SequenceFailure&& selectedFailureReport,
Client::SequenceFailure&& discardedFailureReport,
AZStd::chrono::milliseconds selectedDuration,
AZStd::chrono::milliseconds discaredDuration)
void ImpactAnalysisTestSequenceCompleteCallback(const Client::ImpactAnalysisSequenceReport& sequenceReport)
{
std::cout << "Selected test run:\n";
Output::FailureReport(selectedFailureReport, selectedDuration);
Output::FailureReport(sequenceReport.GetSelectedTestRunReport());
std::cout << "Discarded test run:\n";
Output::FailureReport(discardedFailureReport, discaredDuration);
std::cout << "Drafted test run:\n";
Output::FailureReport(sequenceReport.GetDraftedTestRunReport());
std::cout << "Updating and serializing the test impact analysis data, this may take a moment...\n";
}
// TestRunCompleteCallback
void TestSequenceEventHandler::operator()([[maybe_unused]] Client::TestRun&& test)
void SafeImpactAnalysisTestSequenceCompleteCallback(const Client::SafeImpactAnalysisSequenceReport& sequenceReport)
{
m_numTestsComplete++;
const auto progress = AZStd::string::format("(%03u/%03u)", m_numTestsComplete, m_numTests, test.GetTargetName().c_str());
std::cout << "Selected test run:\n";
Output::FailureReport(sequenceReport.GetSelectedTestRunReport());
std::cout << "Discarded test run:\n";
Output::FailureReport(sequenceReport.GetDiscardedTestRunReport());
std::cout << "Drafted test run:\n";
Output::FailureReport(sequenceReport.GetDraftedTestRunReport());
std::cout << "Updating and serializing the test impact analysis data, this may take a moment...\n";
}
void TestRunCompleteCallback(const Client::TestRun& testRun, size_t numTestRunsCompleted, size_t totalNumTestRuns)
{
const auto progress =
AZStd::string::format("(%03u/%03u)", numTestRunsCompleted, totalNumTestRuns, testRun.GetTargetName().c_str());
AZStd::string result;
switch (test.GetResult())
switch (testRun.GetResult())
{
case Client::TestRunResult::AllTestsPass:
{
@@ -216,15 +214,14 @@ namespace TestImpact
result = SetColorForString(Foreground::White, Background::Magenta, "TIME");
break;
}
default:
{
AZ_Error("TestRunCompleteCallback", false, "Unexpected test result to handle: %u", aznumeric_cast<AZ::u32>(testRun.GetResult()));
}
}
std::cout << progress.c_str() << " " << result.c_str() << " " << test.GetTargetName().c_str() << " (" << (test.GetDuration().count() / 1000.f) << "s)\n";
}
void TestSequenceEventHandler::ClearState()
{
m_numTests = 0;
m_numTestsComplete = 0;
std::cout << progress.c_str() << " " << result.c_str() << " " << testRun.GetTargetName().c_str() << " ("
<< (testRun.GetDuration().count() / 1000.f) << "s)\n";
}
} // namespace Console
} // namespace TestImpact
Code/Tools/TestImpactFramework/Frontend/Console/Code/Source/TestImpactConsoleTestSequenceEventHandler.h
+23 -38
View File
@@ -8,7 +8,7 @@
#include <TestImpactFramework/TestImpactTestSequence.h>
#include <TestImpactFramework/TestImpactClientTestSelection.h>
#include <TestImpactFramework/TestImpactClientFailureReport.h>
#include <TestImpactFramework/TestImpactClientSequenceReport.h>
#include <TestImpactFramework/TestImpactClientTestRun.h>
#include <AzCore/std/chrono/chrono.h>
@@ -21,48 +21,33 @@ namespace TestImpact
{
namespace Console
{
//! Event handler for all test sequence types.
class TestSequenceEventHandler
{
public:
explicit TestSequenceEventHandler(SuiteType suiteFilter);
//! Handler for TestSequenceStartCallback event.
void TestSequenceStartCallback(SuiteType suiteType, const Client::TestRunSelection& selectedTests);
//! TestSequenceStartCallback.
void operator()(Client::TestRunSelection&& selectedTests);
//! Handler for TestSequenceStartCallback event.
void ImpactAnalysisTestSequenceStartCallback(
SuiteType suiteType,
const Client::TestRunSelection& selectedTests,
const AZStd::vector<AZStd::string>& discardedTests,
const AZStd::vector<AZStd::string>& draftedTests);
//! ImpactAnalysisTestSequenceStartCallback.
void operator()(
Client::TestRunSelection&& selectedTests,
AZStd::vector<AZStd::string>&& discardedTests,
AZStd::vector<AZStd::string>&& draftedTests);
//! Handler for SafeImpactAnalysisTestSequenceStartCallback event.
void SafeImpactAnalysisTestSequenceStartCallback(
SuiteType suiteType,
const Client::TestRunSelection& selectedTests,
const Client::TestRunSelection& discardedTests,
const AZStd::vector<AZStd::string>& draftedTests);
//! SafeImpactAnalysisTestSequenceStartCallback.
void operator()(
Client::TestRunSelection&& selectedTests,
Client::TestRunSelection&& discardedTests,
AZStd::vector<AZStd::string>&& draftedTests);
//! Handler for TestSequenceCompleteCallback event.
void TestSequenceCompleteCallback(const Client::SequenceReport& sequenceReport);
//! TestSequenceCompleteCallback.
void operator()(
Client::SequenceFailure&& failureReport,
AZStd::chrono::milliseconds duration);
//! Handler for ImpactAnalysisTestSequenceCompleteCallback event.
void ImpactAnalysisTestSequenceCompleteCallback(const Client::ImpactAnalysisSequenceReport& sequenceReport);
//! SafeTestSequenceCompleteCallback.
void operator()(
Client::SequenceFailure&& selectedFailureReport,
Client::SequenceFailure&& discardedFailureReport,
AZStd::chrono::milliseconds selectedDuration,
AZStd::chrono::milliseconds discaredDuration);
//! Handler for SafeImpactAnalysisTestSequenceCompleteCallback event.
void SafeImpactAnalysisTestSequenceCompleteCallback(const Client::SafeImpactAnalysisSequenceReport& sequenceReport);
//! TestRunCompleteCallback.
void operator()(Client::TestRun&& test);
private:
void ClearState();
SuiteType m_suiteFilter;
size_t m_numTests = 0;
size_t m_numTestsComplete = 0;
};
//! Handler for TestRunCompleteCallback event.
void TestRunCompleteCallback(const Client::TestRun& testRun, size_t numTestRunsCompleted, size_t totalNumTestRuns);
} // namespace Console
} // namespace TestImpact