Add changes from TIF/Feature branch.

Signed-off-by: John <jonawals@amazon.com>
This commit is contained in:
John
2021-08-10 14:20:46 +01:00
parent 4b68e6c766
commit c2a2d1c5c7
42 changed files with 3534 additions and 1069 deletions
@@ -6,6 +6,8 @@
*
*/
#include <TestImpactFramework/TestImpactUtils.h>
#include <Artifact/Factory/TestImpactTestTargetMetaMapFactory.h>
#include <Artifact/TestImpactArtifactException.h>
@@ -67,7 +69,7 @@ namespace TestImpact
{
// Check to see if this test target has the suite we're looking for
if (const auto suiteName = suite[Keys[SuiteKey]].GetString();
strcmp(GetSuiteTypeName(suiteType).c_str(), suiteName) == 0)
strcmp(SuiteTypeAsString(suiteType).c_str(), suiteName) == 0)
{
testMeta.m_suite = suiteName;
testMeta.m_customArgs = suite[Keys[CommandKey]].GetString();
@@ -156,8 +156,6 @@ namespace TestImpact
{
coveringTestTargetIt->second.erase(source);
}
}
// 2.
@@ -9,7 +9,7 @@
#pragma once
#include <TestImpactFramework/TestImpactChangeList.h>
#include <TestImpactFramework/TestImpactTestSequence.h>
#include <TestImpactFramework/TestImpactPolicy.h>
#include <Artifact/Static/TestImpactProductionTargetDescriptor.h>
#include <Artifact/Static/TestImpactTestTargetDescriptor.h>
@@ -8,7 +8,7 @@
#pragma once
#include <TestImpactFramework/TestImpactTestSequence.h>
#include <TestImpactFramework/TestImpactPolicy.h>
#include <Artifact/Static/TestImpactDependencyGraphData.h>
#include <Dependency/TestImpactChangeDependencyList.h>
@@ -6,7 +6,7 @@
*
*/
#include <TestImpactFramework/TestImpactFileUtils.h>
#include <TestImpactFramework/TestImpactUtils.h>
#include <Artifact/Factory/TestImpactTestEnumerationSuiteFactory.h>
#include <TestEngine/TestImpactTestEngineException.h>
@@ -6,7 +6,7 @@
*
*/
#include <TestImpactFramework/TestImpactFileUtils.h>
#include <TestImpactFramework/TestImpactUtils.h>
#include <Artifact/Factory/TestImpactModuleCoverageFactory.h>
#include <Artifact/Factory/TestImpactTestRunSuiteFactory.h>
@@ -6,7 +6,7 @@
*
*/
#include <TestImpactFramework/TestImpactFileUtils.h>
#include <TestImpactFramework/TestImpactUtils.h>
#include <Artifact/Factory/TestImpactTestRunSuiteFactory.h>
#include <TestEngine/TestImpactTestEngineException.h>
@@ -6,7 +6,7 @@
*
*/
#include <TestImpactFramework/TestImpactFileUtils.h>
#include <TestImpactFramework/TestImpactUtils.h>
#include <Target/TestImpactTestTarget.h>
#include <TestEngine/TestImpactTestEngineException.h>
@@ -1,6 +1,7 @@
/*
* Copyright (c) Contributors to the Open 3D Engine Project. For complete copyright and license terms please see the LICENSE at the root of this distribution.
*
* Copyright (c) Contributors to the Open 3D Engine Project.
* For complete copyright and license terms please see the LICENSE at the root of this distribution.
*
* SPDX-License-Identifier: Apache-2.0 OR MIT
*
*/
@@ -11,7 +12,6 @@ namespace TestImpact
{
namespace Client
{
//! Calculates the final sequence result for a composite of multiple sequences.
TestSequenceResult CalculateMultiTestSequenceResult(const AZStd::vector<TestSequenceResult>& results)
{
// Order of precedence:
@@ -19,14 +19,12 @@ namespace TestImpact
// 2. TestSequenceResult::Timeout
// 3. TestSequenceResult::Success
if (const auto it = AZStd::find(results.begin(), results.end(), TestSequenceResult::Failure);
it != results.end())
if (const auto it = AZStd::find(results.begin(), results.end(), TestSequenceResult::Failure); it != results.end())
{
return TestSequenceResult::Failure;
}
if (const auto it = AZStd::find(results.begin(), results.end(), TestSequenceResult::Timeout);
it != results.end())
if (const auto it = AZStd::find(results.begin(), results.end(), TestSequenceResult::Timeout); it != results.end())
{
return TestSequenceResult::Timeout;
}
@@ -38,20 +36,32 @@ namespace TestImpact
TestSequenceResult result,
AZStd::chrono::high_resolution_clock::time_point startTime,
AZStd::chrono::milliseconds duration,
AZStd::vector<TestRun>&& passingTests,
AZStd::vector<TestRunWithTestFailures>&& failingTests,
AZStd::vector<TestRun>&& executionFailureTests,
AZStd::vector<TestRun>&& timedOutTests,
AZStd::vector<TestRun>&& unexecutedTests)
AZStd::vector<PassingTestRun>&& passingTestRuns,
AZStd::vector<FailingTestRun>&& failingTestRuns,
AZStd::vector<TestRunWithExecutionFailure>&& executionFailureTestRuns,
AZStd::vector<TimedOutTestRun>&& timedOutTestRuns,
AZStd::vector<UnexecutedTestRun>&& unexecutedTestRuns)
: m_startTime(startTime)
, m_result(result)
, m_duration(duration)
, m_passingTests(AZStd::move(passingTests))
, m_failingTests(AZStd::move(failingTests))
, m_executionFailureTests(AZStd::move(executionFailureTests))
, m_timedOutTests(AZStd::move(timedOutTests))
, m_unexecutedTests(AZStd::move(unexecutedTests))
, m_passingTestRuns(AZStd::move(passingTestRuns))
, m_failingTestRuns(AZStd::move(failingTestRuns))
, m_executionFailureTestRuns(AZStd::move(executionFailureTestRuns))
, m_timedOutTestRuns(AZStd::move(timedOutTestRuns))
, m_unexecutedTestRuns(AZStd::move(unexecutedTestRuns))
{
for (const auto& failingTestRun : m_failingTestRuns)
{
m_totalNumPassingTests += failingTestRun.GetTotalNumPassingTests();
m_totalNumFailingTests += failingTestRun.GetTotalNumFailingTests();
m_totalNumDisabledTests += failingTestRun.GetTotalNumDisabledTests();
}
for (const auto& passingTestRun : m_passingTestRuns)
{
m_totalNumPassingTests += passingTestRun.GetTotalNumPassingTests();
m_totalNumDisabledTests += passingTestRun.GetTotalNumDisabledTests();
}
}
TestSequenceResult TestRunReport::GetResult() const
@@ -74,234 +84,238 @@ namespace TestImpact
return m_duration;
}
size_t TestRunReport::GetNumPassingTests() const
size_t TestRunReport::GetTotalNumTestRuns() const
{
return m_passingTests.size();
return
GetNumPassingTestRuns() +
GetNumFailingTestRuns() +
GetNumExecutionFailureTestRuns() +
GetNumTimedOutTestRuns() +
GetNumUnexecutedTestRuns();
}
size_t TestRunReport::GetNumFailingTests() const
size_t TestRunReport::GetNumPassingTestRuns() const
{
return m_failingTests.size();
return m_passingTestRuns.size();
}
size_t TestRunReport::GetNumTimedOutTests() const
size_t TestRunReport::GetNumFailingTestRuns() const
{
return m_timedOutTests.size();
return m_failingTestRuns.size();
}
size_t TestRunReport::GetNumUnexecutedTests() const
size_t TestRunReport::GetNumExecutionFailureTestRuns() const
{
return m_unexecutedTests.size();
return m_executionFailureTestRuns.size();
}
const AZStd::vector<TestRun>& TestRunReport::GetPassingTests() const
size_t TestRunReport::TestRunReport::GetNumTimedOutTestRuns() const
{
return m_passingTests;
return m_timedOutTestRuns.size();
}
const AZStd::vector<TestRunWithTestFailures>& TestRunReport::GetFailingTests() const
size_t TestRunReport::GetNumUnexecutedTestRuns() const
{
return m_failingTests;
return m_unexecutedTestRuns.size();
}
const AZStd::vector<TestRun>& TestRunReport::GetExecutionFailureTests() const
const AZStd::vector<PassingTestRun>& TestRunReport::GetPassingTestRuns() const
{
return m_executionFailureTests;
return m_passingTestRuns;
}
const AZStd::vector<TestRun>& TestRunReport::GetTimedOutTests() const
const AZStd::vector<FailingTestRun>& TestRunReport::GetFailingTestRuns() const
{
return m_timedOutTests;
return m_failingTestRuns;
}
const AZStd::vector<TestRun>& TestRunReport::GetUnexecutedTests() const
const AZStd::vector<TestRunWithExecutionFailure>& TestRunReport::GetExecutionFailureTestRuns() const
{
return m_unexecutedTests;
return m_executionFailureTestRuns;
}
SequenceReport::SequenceReport(SuiteType suiteType, const TestRunSelection& selectedTests, TestRunReport&& selectedTestRunReport)
: m_suite(suiteType)
, m_selectedTests(selectedTests)
, m_selectedTestRunReport(AZStd::move(selectedTestRunReport))
const AZStd::vector<TimedOutTestRun>& TestRunReport::GetTimedOutTestRuns() const
{
return m_timedOutTestRuns;
}
TestSequenceResult SequenceReport::GetResult() const
const AZStd::vector<UnexecutedTestRun>& TestRunReport::GetUnexecutedTestRuns() const
{
return m_selectedTestRunReport.GetResult();
return m_unexecutedTestRuns;
}
AZStd::chrono::high_resolution_clock::time_point SequenceReport::GetStartTime() const
size_t TestRunReport::GetTotalNumPassingTests() const
{
return m_selectedTestRunReport.GetStartTime();
return m_totalNumPassingTests;
}
AZStd::chrono::high_resolution_clock::time_point SequenceReport::GetEndTime() const
size_t TestRunReport::GetTotalNumFailingTests() const
{
return GetStartTime() + GetDuration();
return m_totalNumFailingTests;
}
AZStd::chrono::milliseconds SequenceReport::GetDuration() const
size_t TestRunReport::GetTotalNumDisabledTests() const
{
return m_selectedTestRunReport.GetDuration();
return m_totalNumDisabledTests;
}
TestRunSelection SequenceReport::GetSelectedTests() const
{
return m_selectedTests;
}
TestRunReport SequenceReport::GetSelectedTestRunReport() const
{
return m_selectedTestRunReport;
}
size_t SequenceReport::GetTotalNumPassingTests() const
{
return m_selectedTestRunReport.GetNumPassingTests();
}
size_t SequenceReport::GetTotalNumFailingTests() const
{
return m_selectedTestRunReport.GetNumFailingTests();
}
size_t SequenceReport::GetTotalNumTimedOutTests() const
{
return m_selectedTestRunReport.GetNumTimedOutTests();
}
size_t SequenceReport::GetTotalNumUnexecutedTests() const
{
return m_selectedTestRunReport.GetNumUnexecutedTests();
}
DraftingSequenceReport::DraftingSequenceReport(
RegularSequenceReport::RegularSequenceReport(
size_t maxConcurrency,
const AZStd::optional<AZStd::chrono::milliseconds>& testTargetTimeout,
const AZStd::optional<AZStd::chrono::milliseconds>& globalTimeout,
const SequencePolicyState& policyState,
SuiteType suiteType,
const TestRunSelection& selectedTests,
const AZStd::vector<AZStd::string>& draftedTests,
TestRunReport&& selectedTestRunReport,
TestRunReport&& draftedTestRunReport)
: SequenceReport(suiteType, selectedTests, AZStd::move(selectedTestRunReport))
, m_draftedTests(draftedTests)
, m_draftedTestRunReport(AZStd::move(draftedTestRunReport))
const TestRunSelection& selectedTestRuns,
TestRunReport&& selectedTestRunReport)
: SequenceReportBase(
SequenceReportType::RegularSequence,
maxConcurrency,
testTargetTimeout,
globalTimeout,
policyState,
suiteType,
selectedTestRuns,
AZStd::move(selectedTestRunReport))
{
}
TestSequenceResult DraftingSequenceReport::GetResult() const
SeedSequenceReport::SeedSequenceReport(
size_t maxConcurrency,
const AZStd::optional<AZStd::chrono::milliseconds>& testTargetTimeout,
const AZStd::optional<AZStd::chrono::milliseconds>& globalTimeout,
const SequencePolicyState& policyState,
SuiteType suiteType,
const TestRunSelection& selectedTestRuns,
TestRunReport&& selectedTestRunReport)
: SequenceReportBase(
SequenceReportType::SeedSequence,
maxConcurrency,
testTargetTimeout,
globalTimeout,
policyState,
suiteType,
selectedTestRuns,
AZStd::move(selectedTestRunReport))
{
return CalculateMultiTestSequenceResult({SequenceReport::GetResult(), m_draftedTestRunReport.GetResult()});
}
AZStd::chrono::milliseconds DraftingSequenceReport::GetDuration() const
{
return SequenceReport::GetDuration() + m_draftedTestRunReport.GetDuration();
}
size_t DraftingSequenceReport::GetTotalNumPassingTests() const
{
return SequenceReport::GetTotalNumPassingTests() + m_draftedTestRunReport.GetNumPassingTests();
}
size_t DraftingSequenceReport::GetTotalNumFailingTests() const
{
return SequenceReport::GetTotalNumFailingTests() + m_draftedTestRunReport.GetNumFailingTests();
}
size_t DraftingSequenceReport::GetTotalNumTimedOutTests() const
{
return SequenceReport::GetTotalNumTimedOutTests() + m_draftedTestRunReport.GetNumTimedOutTests();
}
size_t DraftingSequenceReport::GetTotalNumUnexecutedTests() const
{
return SequenceReport::GetTotalNumUnexecutedTests() + m_draftedTestRunReport.GetNumUnexecutedTests();
}
const AZStd::vector<AZStd::string>& DraftingSequenceReport::GetDraftedTests() const
{
return m_draftedTests;
}
TestRunReport DraftingSequenceReport::GetDraftedTestRunReport() const
{
return m_draftedTestRunReport;
}
ImpactAnalysisSequenceReport::ImpactAnalysisSequenceReport(
size_t maxConcurrency,
const AZStd::optional<AZStd::chrono::milliseconds>& testTargetTimeout,
const AZStd::optional<AZStd::chrono::milliseconds>& globalTimeout,
const ImpactAnalysisSequencePolicyState& policyState,
SuiteType suiteType,
const TestRunSelection& selectedTests,
const AZStd::vector<AZStd::string>& discardedTests,
const AZStd::vector<AZStd::string>& draftedTests,
const TestRunSelection& selectedTestRuns,
const AZStd::vector<AZStd::string>& discardedTestRuns,
const AZStd::vector<AZStd::string>& draftedTestRuns,
TestRunReport&& selectedTestRunReport,
TestRunReport&& draftedTestRunReport)
: DraftingSequenceReport(
suiteType,
selectedTests,
draftedTests,
AZStd::move(selectedTestRunReport),
AZStd::move(draftedTestRunReport))
, m_discardedTests(discardedTests)
: DraftingSequenceReportBase(
SequenceReportType::ImpactAnalysisSequence,
maxConcurrency,
testTargetTimeout,
globalTimeout,
policyState,
suiteType,
selectedTestRuns,
draftedTestRuns,
AZStd::move(selectedTestRunReport),
AZStd::move(draftedTestRunReport))
, m_discardedTestRuns(discardedTestRuns)
{
}
const AZStd::vector<AZStd::string>& ImpactAnalysisSequenceReport::GetDiscardedTests() const
const AZStd::vector<AZStd::string>& ImpactAnalysisSequenceReport::GetDiscardedTestRuns() const
{
return m_discardedTests;
return m_discardedTestRuns;
}
SafeImpactAnalysisSequenceReport::SafeImpactAnalysisSequenceReport(
size_t maxConcurrency,
const AZStd::optional<AZStd::chrono::milliseconds>& testTargetTimeout,
const AZStd::optional<AZStd::chrono::milliseconds>& globalTimeout,
const SafeImpactAnalysisSequencePolicyState& policyState,
SuiteType suiteType,
const TestRunSelection& selectedTests,
const TestRunSelection& discardedTests,
const AZStd::vector<AZStd::string>& draftedTests,
const TestRunSelection& selectedTestRuns,
const TestRunSelection& discardedTestRuns,
const AZStd::vector<AZStd::string>& draftedTestRuns,
TestRunReport&& selectedTestRunReport,
TestRunReport&& discardedTestRunReport,
TestRunReport&& draftedTestRunReport)
: DraftingSequenceReport(
suiteType,
selectedTests,
draftedTests,
AZStd::move(selectedTestRunReport),
AZStd::move(draftedTestRunReport))
, m_discardedTests(discardedTests)
: DraftingSequenceReportBase(
SequenceReportType::SafeImpactAnalysisSequence,
maxConcurrency,
testTargetTimeout,
globalTimeout,
policyState,
suiteType,
selectedTestRuns,
draftedTestRuns,
AZStd::move(selectedTestRunReport),
AZStd::move(draftedTestRunReport))
, m_discardedTestRuns(discardedTestRuns)
, m_discardedTestRunReport(AZStd::move(discardedTestRunReport))
{
}
TestSequenceResult SafeImpactAnalysisSequenceReport::GetResult() const
{
return CalculateMultiTestSequenceResult({ DraftingSequenceReport::GetResult(), m_discardedTestRunReport.GetResult() });
return CalculateMultiTestSequenceResult({ DraftingSequenceReportBase::GetResult(), m_discardedTestRunReport.GetResult() });
}
AZStd::chrono::milliseconds SafeImpactAnalysisSequenceReport::GetDuration() const
{
return DraftingSequenceReport::GetDuration() + m_discardedTestRunReport.GetDuration();
return DraftingSequenceReportBase::GetDuration() + m_discardedTestRunReport.GetDuration();
}
size_t SafeImpactAnalysisSequenceReport::GetTotalNumTestRuns() const
{
return DraftingSequenceReportBase::GetTotalNumTestRuns() + m_discardedTestRunReport.GetTotalNumTestRuns();
}
size_t SafeImpactAnalysisSequenceReport::GetTotalNumPassingTests() const
{
return DraftingSequenceReport::GetTotalNumPassingTests() + m_discardedTestRunReport.GetNumPassingTests();
return DraftingSequenceReportBase::GetTotalNumPassingTests() + m_discardedTestRunReport.GetTotalNumPassingTests();
}
size_t SafeImpactAnalysisSequenceReport::GetTotalNumFailingTests() const
{
return DraftingSequenceReport::GetTotalNumFailingTests() + m_discardedTestRunReport.GetNumFailingTests();
return DraftingSequenceReportBase::GetTotalNumFailingTests() + m_discardedTestRunReport.GetTotalNumFailingTests();
}
size_t SafeImpactAnalysisSequenceReport::GetTotalNumTimedOutTests() const
size_t SafeImpactAnalysisSequenceReport::GetTotalNumDisabledTests() const
{
return DraftingSequenceReport::GetTotalNumTimedOutTests() + m_discardedTestRunReport.GetNumTimedOutTests();
return DraftingSequenceReportBase::GetTotalNumDisabledTests() + m_discardedTestRunReport.GetTotalNumDisabledTests();
}
size_t SafeImpactAnalysisSequenceReport::GetTotalNumUnexecutedTests() const
size_t SafeImpactAnalysisSequenceReport::GetTotalNumPassingTestRuns() const
{
return DraftingSequenceReport::GetTotalNumUnexecutedTests() + m_discardedTestRunReport.GetNumUnexecutedTests();
return DraftingSequenceReportBase::GetTotalNumPassingTestRuns() + m_discardedTestRunReport.GetNumPassingTestRuns();
}
size_t SafeImpactAnalysisSequenceReport::GetTotalNumFailingTestRuns() const
{
return DraftingSequenceReportBase::GetTotalNumFailingTestRuns() + m_discardedTestRunReport.GetNumFailingTestRuns();
}
size_t SafeImpactAnalysisSequenceReport::GetTotalNumExecutionFailureTestRuns() const
{
return DraftingSequenceReportBase::GetTotalNumExecutionFailureTestRuns() + m_discardedTestRunReport.GetNumExecutionFailureTestRuns();
}
size_t SafeImpactAnalysisSequenceReport::GetTotalNumTimedOutTestRuns() const
{
return DraftingSequenceReportBase::GetTotalNumTimedOutTestRuns() + m_discardedTestRunReport.GetNumTimedOutTestRuns();
}
size_t SafeImpactAnalysisSequenceReport::GetTotalNumUnexecutedTestRuns() const
{
return DraftingSequenceReportBase::GetTotalNumUnexecutedTestRuns() + m_discardedTestRunReport.GetNumUnexecutedTestRuns();
}
const TestRunSelection SafeImpactAnalysisSequenceReport::GetDiscardedTests() const
const TestRunSelection SafeImpactAnalysisSequenceReport::GetDiscardedTestRuns() const
{
return m_discardedTests;
return m_discardedTestRuns;
}
TestRunReport SafeImpactAnalysisSequenceReport::GetDiscardedTestRunReport() const
@@ -0,0 +1,606 @@
/*
* Copyright (c) Contributors to the Open 3D Engine Project.
* For complete copyright and license terms please see the LICENSE at the root of this distribution.
*
* SPDX-License-Identifier: Apache-2.0 OR MIT
*
*/
#include <TestImpactFramework/TestImpactClientSequenceReportSerializer.h>
#include <TestImpactFramework/TestImpactSequenceReportException.h>
#include <TestImpactFramework/TestImpactUtils.h>
#include <AzCore/JSON/document.h>
#include <AzCore/JSON/prettywriter.h>
#include <AzCore/JSON/rapidjson.h>
#include <AzCore/JSON/stringbuffer.h>
namespace TestImpact
{
namespace
{
namespace SequenceReportFields
{
// Keys for pertinent JSON node and attribute names
constexpr const char* Keys[] =
{
"name",
"command_args",
"start_time",
"end_time",
"duration",
"result",
"num_passing_tests",
"num_failing_tests",
"num_disabled_tests",
"tests",
"num_passing_test_runs",
"num_failing_test_runs",
"num_execution_failure_test_runs",
"num_timed_out_test_runs",
"num_unexecuted_test_runs",
"passing_test_runs",
"failing_test_runs",
"execution_failure_test_runs",
"timed_out_test_runs",
"unexecuted_test_runs",
"total_num_passing_tests",
"total_num_failing_tests",
"total_num_disabled_tests",
"total_num_test_runs",
"num_included_test_runs",
"num_excluded_test_runs",
"included_test_runs",
"excluded_test_runs",
"execution_failure",
"coverage_failure",
"test_failure",
"integrity_failure",
"test_sharding",
"target_output_capture",
"test_prioritization",
"dynamic_dependency_map",
"type",
"test_target_timeout",
"global_timeout",
"max_concurrency",
"policy",
"suite",
"selected_test_runs",
"selected_test_run_report",
"total_num_passing_test_runs",
"total_num_failing_test_runs",
"total_num_execution_failure_test_runs",
"total_num_timed_out_test_runs",
"total_num_unexecuted_test_runs",
"drafted_test_runs",
"drafted_test_run_report",
"discarded_test_runs",
"discarded_test_run_report"
};
enum
{
Name,
CommandArgs,
StartTime,
EndTime,
Duration,
Result,
NumPassingTests,
NumFailingTests,
NumDisabledTests,
Tests,
NumPassingTestRuns,
NumFailingTestRuns,
NumExecutionFailureTestRuns,
NumTimedOutTestRuns,
NumUnexecutedTestRuns,
PassingTestRuns,
FailingTestRuns,
ExecutionFailureTestRuns,
TimedOutTestRuns,
UnexecutedTestRuns,
TotalNumPassingTests,
TotalNumFailingTests,
TotalNumDisabledTests,
TotalNumTestRuns,
NumIncludedTestRuns,
NumExcludedTestRuns,
IncludedTestRuns,
ExcludedTestRuns,
ExecutionFailure,
CoverageFailure,
TestFailure,
IntegrityFailure,
TestSharding,
TargetOutputCapture,
TestPrioritization,
DynamicDependencyMap,
Type,
TestTargetTimeout,
GlobalTimeout,
MaxConcurrency,
Policy,
Suite,
SelectedTestRuns,
SelectedTestRunReport,
TotalNumPassingTestRuns,
TotalNumFailingTestRuns,
TotalNumExecutionFailureTestRuns,
TotalNumTimedOutTestRuns,
TotalNumUnexecutedTestRuns,
DraftedTestRuns,
DraftedTestRunReport,
DiscardedTestRuns,
DiscardedTestRunReport
};
} // namespace SequenceReportFields
AZ::u64 TimePointInMsAsInt64(AZStd::chrono::high_resolution_clock::time_point timePoint)
{
return AZStd::chrono::duration_cast<AZStd::chrono::milliseconds>(timePoint.time_since_epoch()).count();
}
void SerializeTestRunMembers(const Client::TestRunBase& testRun, rapidjson::PrettyWriter<rapidjson::StringBuffer>& writer)
{
// Name
writer.Key(SequenceReportFields::Keys[SequenceReportFields::Name]);
writer.String(testRun.GetTargetName().c_str());
// Command string
writer.Key(SequenceReportFields::Keys[SequenceReportFields::CommandArgs]);
writer.String(testRun.GetCommandString().c_str());
// Start time
writer.Key(SequenceReportFields::Keys[SequenceReportFields::StartTime]);
writer.Int64(TimePointInMsAsInt64(testRun.GetStartTime()));
// End time
writer.Key(SequenceReportFields::Keys[SequenceReportFields::EndTime]);
writer.Int64(TimePointInMsAsInt64(testRun.GetEndTime()));
// Duration
writer.Key(SequenceReportFields::Keys[SequenceReportFields::Duration]);
writer.Uint64(testRun.GetDuration().count());
// Result
writer.Key(SequenceReportFields::Keys[SequenceReportFields::Result]);
writer.String(TestRunResultAsString(testRun.GetResult()).c_str());
}
void SerializeTestRun(const Client::TestRunBase& testRun, rapidjson::PrettyWriter<rapidjson::StringBuffer>& writer)
{
writer.StartObject();
{
SerializeTestRunMembers(testRun, writer);
}
writer.EndObject();
}
void SerializeCompletedTestRun(const Client::CompletedTestRun& testRun, rapidjson::PrettyWriter<rapidjson::StringBuffer>& writer)
{
writer.StartObject();
{
SerializeTestRunMembers(testRun, writer);
// Number of passing test cases
writer.Key(SequenceReportFields::Keys[SequenceReportFields::NumPassingTests]);
writer.Uint64(testRun.GetTotalNumPassingTests());
// Number of failing test cases
writer.Key(SequenceReportFields::Keys[SequenceReportFields::NumFailingTests]);
writer.Uint64(testRun.GetTotalNumFailingTests());
// Number of disabled test cases
writer.Key(SequenceReportFields::Keys[SequenceReportFields::NumDisabledTests]);
writer.Uint64(testRun.GetTotalNumDisabledTests());
// Tests
writer.Key(SequenceReportFields::Keys[SequenceReportFields::Tests]);
writer.StartArray();
for (const auto& test : testRun.GetTests())
{
// Test
writer.StartObject();
// Name
writer.Key(SequenceReportFields::Keys[SequenceReportFields::Name]);
writer.String(test.GetName().c_str());
// Result
writer.Key(SequenceReportFields::Keys[SequenceReportFields::Result]);
writer.String(ClientTestResultAsString(test.GetResult()).c_str());
writer.EndObject(); // Test
}
writer.EndArray(); // Tests
}
writer.EndObject();
}
void SerializeTestRunReport(
const Client::TestRunReport& testRunReport, rapidjson::PrettyWriter<rapidjson::StringBuffer>& writer)
{
writer.StartObject();
{
// Result
writer.Key(SequenceReportFields::Keys[SequenceReportFields::Result]);
writer.String(TestSequenceResultAsString(testRunReport.GetResult()).c_str());
// Start time
writer.Key(SequenceReportFields::Keys[SequenceReportFields::StartTime]);
writer.Int64(TimePointInMsAsInt64(testRunReport.GetStartTime()));
// End time
writer.Key(SequenceReportFields::Keys[SequenceReportFields::EndTime]);
writer.Int64(TimePointInMsAsInt64(testRunReport.GetEndTime()));
// Duration
writer.Key(SequenceReportFields::Keys[SequenceReportFields::Duration]);
writer.Uint64(testRunReport.GetDuration().count());
// Number of passing test runs
writer.Key(SequenceReportFields::Keys[SequenceReportFields::NumPassingTestRuns]);
writer.Uint64(testRunReport.GetNumPassingTestRuns());
// Number of failing test runs
writer.Key(SequenceReportFields::Keys[SequenceReportFields::NumFailingTestRuns]);
writer.Uint64(testRunReport.GetNumFailingTestRuns());
// Number of test runs that failed to execute
writer.Key(SequenceReportFields::Keys[SequenceReportFields::NumExecutionFailureTestRuns]);
writer.Uint64(testRunReport.GetNumExecutionFailureTestRuns());
// Number of timed out test runs
writer.Key(SequenceReportFields::Keys[SequenceReportFields::NumTimedOutTestRuns]);
writer.Uint64(testRunReport.GetNumTimedOutTestRuns());
// Number of unexecuted test runs
writer.Key(SequenceReportFields::Keys[SequenceReportFields::NumUnexecutedTestRuns]);
writer.Uint64(testRunReport.GetNumUnexecutedTestRuns());
// Passing test runs
writer.Key(SequenceReportFields::Keys[SequenceReportFields::PassingTestRuns]);
writer.StartArray();
for (const auto& testRun : testRunReport.GetPassingTestRuns())
{
SerializeCompletedTestRun(testRun, writer);
}
writer.EndArray(); // Passing test runs
// Failing test runs
writer.Key(SequenceReportFields::Keys[SequenceReportFields::FailingTestRuns]);
writer.StartArray();
for (const auto& testRun : testRunReport.GetFailingTestRuns())
{
SerializeCompletedTestRun(testRun, writer);
}
writer.EndArray(); // Failing test runs
// Execution failures
writer.Key(SequenceReportFields::Keys[SequenceReportFields::ExecutionFailureTestRuns]);
writer.StartArray();
for (const auto& testRun : testRunReport.GetExecutionFailureTestRuns())
{
SerializeTestRun(testRun, writer);
}
writer.EndArray(); // Execution failures
// Timed out test runs
writer.Key(SequenceReportFields::Keys[SequenceReportFields::TimedOutTestRuns]);
writer.StartArray();
for (const auto& testRun : testRunReport.GetTimedOutTestRuns())
{
SerializeTestRun(testRun, writer);
}
writer.EndArray(); // Timed out test runs
// Unexecuted test runs
writer.Key(SequenceReportFields::Keys[SequenceReportFields::UnexecutedTestRuns]);
writer.StartArray();
for (const auto& testRun : testRunReport.GetUnexecutedTestRuns())
{
SerializeTestRun(testRun, writer);
}
writer.EndArray(); // Unexecuted test runs
// Number of passing tests
writer.Key(SequenceReportFields::Keys[SequenceReportFields::TotalNumPassingTests]);
writer.Uint64(testRunReport.GetTotalNumPassingTests());
// Number of failing tests
writer.Key(SequenceReportFields::Keys[SequenceReportFields::TotalNumFailingTests]);
writer.Uint64(testRunReport.GetTotalNumFailingTests());
// Number of disabled tests
writer.Key(SequenceReportFields::Keys[SequenceReportFields::TotalNumDisabledTests]);
writer.Uint64(testRunReport.GetTotalNumDisabledTests());
}
writer.EndObject();
}
void SerializeTestSelection(
const Client::TestRunSelection& testSelection, rapidjson::PrettyWriter<rapidjson::StringBuffer>& writer)
{
writer.StartObject();
{
// Total number of test runs
writer.Key(SequenceReportFields::Keys[SequenceReportFields::TotalNumTestRuns]);
writer.Uint64(testSelection.GetTotalNumTests());
// Number of included test runs
writer.Key(SequenceReportFields::Keys[SequenceReportFields::NumIncludedTestRuns]);
writer.Uint64(testSelection.GetNumIncludedTestRuns());
// Number of excluded test runs
writer.Key(SequenceReportFields::Keys[SequenceReportFields::NumExcludedTestRuns]);
writer.Uint64(testSelection.GetNumExcludedTestRuns());
// Included test runs
writer.Key(SequenceReportFields::Keys[SequenceReportFields::IncludedTestRuns]);
writer.StartArray();
for (const auto& testRun : testSelection.GetIncludededTestRuns())
{
writer.String(testRun.c_str());
}
writer.EndArray(); // Included test runs
// Excluded test runs
writer.Key(SequenceReportFields::Keys[SequenceReportFields::ExcludedTestRuns]);
writer.StartArray();
for (const auto& testRun : testSelection.GetExcludedTestRuns())
{
writer.String(testRun.c_str());
}
writer.EndArray(); // Excluded test runs
}
writer.EndObject();
}
void SerializePolicyStateBaseMembers(const PolicyStateBase& policyState, rapidjson::PrettyWriter<rapidjson::StringBuffer>& writer)
{
// Execution failure
writer.Key(SequenceReportFields::Keys[SequenceReportFields::ExecutionFailure]);
writer.String(ExecutionFailurePolicyAsString(policyState.m_executionFailurePolicy).c_str());
// Failed test coverage
writer.Key(SequenceReportFields::Keys[SequenceReportFields::CoverageFailure]);
writer.String(FailedTestCoveragePolicyAsString(policyState.m_failedTestCoveragePolicy).c_str());
// Test failure
writer.Key(SequenceReportFields::Keys[SequenceReportFields::TestFailure]);
writer.String(TestFailurePolicyAsString(policyState.m_testFailurePolicy).c_str());
// Integrity failure
writer.Key(SequenceReportFields::Keys[SequenceReportFields::IntegrityFailure]);
writer.String(IntegrityFailurePolicyAsString(policyState.m_integrityFailurePolicy).c_str());
// Test sharding
writer.Key(SequenceReportFields::Keys[SequenceReportFields::TestSharding]);
writer.String(TestShardingPolicyAsString(policyState.m_testShardingPolicy).c_str());
// Target output capture
writer.Key(SequenceReportFields::Keys[SequenceReportFields::TargetOutputCapture]);
writer.String(TargetOutputCapturePolicyAsString(policyState.m_targetOutputCapture).c_str());
}
void SerializePolicyStateMembers(
const SequencePolicyState& policyState, rapidjson::PrettyWriter<rapidjson::StringBuffer>& writer)
{
SerializePolicyStateBaseMembers(policyState.m_basePolicies, writer);
}
void SerializePolicyStateMembers(
const SafeImpactAnalysisSequencePolicyState& policyState, rapidjson::PrettyWriter<rapidjson::StringBuffer>& writer)
{
SerializePolicyStateBaseMembers(policyState.m_basePolicies, writer);
// Test prioritization
writer.Key(SequenceReportFields::Keys[SequenceReportFields::TestPrioritization]);
writer.String(TestPrioritizationPolicyAsString(policyState.m_testPrioritizationPolicy).c_str());
}
void SerializePolicyStateMembers(
const ImpactAnalysisSequencePolicyState& policyState, rapidjson::PrettyWriter<rapidjson::StringBuffer>& writer)
{
SerializePolicyStateBaseMembers(policyState.m_basePolicies, writer);
// Test prioritization
writer.Key(SequenceReportFields::Keys[SequenceReportFields::TestPrioritization]);
writer.String(TestPrioritizationPolicyAsString(policyState.m_testPrioritizationPolicy).c_str());
// Dynamic dependency map
writer.Key(SequenceReportFields::Keys[SequenceReportFields::DynamicDependencyMap]);
writer.String(DynamicDependencyMapPolicyAsString(policyState.m_dynamicDependencyMap).c_str());
}
template<typename PolicyStateType>
void SerializeSequenceReportBaseMembers(
const Client::SequenceReportBase<PolicyStateType>& sequenceReport, rapidjson::PrettyWriter<rapidjson::StringBuffer>& writer)
{
// Type
writer.Key(SequenceReportFields::Keys[SequenceReportFields::Type]);
writer.String(SequenceReportTypeAsString(sequenceReport.GetType()).c_str());
// Test target timeout
writer.Key(SequenceReportFields::Keys[SequenceReportFields::TestTargetTimeout]);
writer.Uint64(sequenceReport.GetTestTargetTimeout().value_or(AZStd::chrono::milliseconds{0}).count());
// Global timeout
writer.Key(SequenceReportFields::Keys[SequenceReportFields::GlobalTimeout]);
writer.Uint64(sequenceReport.GetGlobalTimeout().value_or(AZStd::chrono::milliseconds{ 0 }).count());
// Maximum concurrency
writer.Key(SequenceReportFields::Keys[SequenceReportFields::MaxConcurrency]);
writer.Uint64(sequenceReport.GetMaxConcurrency());
// Policies
writer.Key(SequenceReportFields::Keys[SequenceReportFields::Policy]);
writer.StartObject();
{
SerializePolicyStateMembers(sequenceReport.GetPolicyState(), writer);
}
writer.EndObject(); // Policies
// Suite
writer.Key(SequenceReportFields::Keys[SequenceReportFields::Suite]);
writer.String(SuiteTypeAsString(sequenceReport.GetSuite()).c_str());
// Selected test runs
writer.Key(SequenceReportFields::Keys[SequenceReportFields::SelectedTestRuns]);
SerializeTestSelection(sequenceReport.GetSelectedTestRuns(), writer);
// Selected test run report
writer.Key(SequenceReportFields::Keys[SequenceReportFields::SelectedTestRunReport]);
SerializeTestRunReport(sequenceReport.GetSelectedTestRunReport(), writer);
// Start time
writer.Key(SequenceReportFields::Keys[SequenceReportFields::StartTime]);
writer.Int64(TimePointInMsAsInt64(sequenceReport.GetStartTime()));
// End time
writer.Key(SequenceReportFields::Keys[SequenceReportFields::EndTime]);
writer.Int64(TimePointInMsAsInt64(sequenceReport.GetEndTime()));
// Duration
writer.Key(SequenceReportFields::Keys[SequenceReportFields::Duration]);
writer.Uint64(sequenceReport.GetDuration().count());
// Result
writer.Key(SequenceReportFields::Keys[SequenceReportFields::Result]);
writer.String(TestSequenceResultAsString(sequenceReport.GetResult()).c_str());
// Total number of test runs
writer.Key(SequenceReportFields::Keys[SequenceReportFields::TotalNumTestRuns]);
writer.Uint64(sequenceReport.GetTotalNumTestRuns());
// Total number of passing test runs
writer.Key(SequenceReportFields::Keys[SequenceReportFields::TotalNumPassingTestRuns]);
writer.Uint64(sequenceReport.GetTotalNumPassingTestRuns());
// Total number of failing test runs
writer.Key(SequenceReportFields::Keys[SequenceReportFields::TotalNumFailingTestRuns]);
writer.Uint64(sequenceReport.GetTotalNumFailingTestRuns());
// Total number of test runs that failed to execute
writer.Key(SequenceReportFields::Keys[SequenceReportFields::TotalNumExecutionFailureTestRuns]);
writer.Uint64(sequenceReport.GetTotalNumExecutionFailureTestRuns());
// Total number of timed out test runs
writer.Key(SequenceReportFields::Keys[SequenceReportFields::TotalNumTimedOutTestRuns]);
writer.Uint64(sequenceReport.GetTotalNumTimedOutTestRuns());
// Total number of unexecuted test runs
writer.Key(SequenceReportFields::Keys[SequenceReportFields::TotalNumUnexecutedTestRuns]);
writer.Uint64(sequenceReport.GetTotalNumUnexecutedTestRuns());
// Total number of passing tests
writer.Key(SequenceReportFields::Keys[SequenceReportFields::TotalNumPassingTests]);
writer.Uint64(sequenceReport.GetTotalNumPassingTests());
// Total number of failing tests
writer.Key(SequenceReportFields::Keys[SequenceReportFields::TotalNumFailingTests]);
writer.Uint64(sequenceReport.GetTotalNumFailingTests());
// Total number of disabled tests
writer.Key(SequenceReportFields::Keys[SequenceReportFields::TotalNumDisabledTests]);
writer.Uint64(sequenceReport.GetTotalNumDisabledTests());
}
template<typename PolicyStateType>
void SerializeDraftingSequenceReportMembers(
const Client::DraftingSequenceReportBase<PolicyStateType>& sequenceReport, rapidjson::PrettyWriter<rapidjson::StringBuffer>& writer)
{
SerializeSequenceReportBaseMembers(sequenceReport, writer);
// Drafted test runs
writer.Key(SequenceReportFields::Keys[SequenceReportFields::DraftedTestRuns]);
writer.StartArray();
for (const auto& testRun : sequenceReport.GetDraftedTestRuns())
{
writer.String(testRun.c_str());
}
writer.EndArray(); // Drafted test runs
// Drafted test run report
writer.Key(SequenceReportFields::Keys[SequenceReportFields::DraftedTestRunReport]);
SerializeTestRunReport(sequenceReport.GetDraftedTestRunReport(), writer);
}
} // namespace
AZStd::string SerializeSequenceReport(const Client::RegularSequenceReport& sequenceReport)
{
rapidjson::StringBuffer stringBuffer;
rapidjson::PrettyWriter<rapidjson::StringBuffer> writer(stringBuffer);
writer.StartObject();
{
SerializeSequenceReportBaseMembers(sequenceReport, writer);
}
writer.EndObject();
return stringBuffer.GetString();
}
AZStd::string SerializeSequenceReport(const Client::SeedSequenceReport& sequenceReport)
{
rapidjson::StringBuffer stringBuffer;
rapidjson::PrettyWriter<rapidjson::StringBuffer> writer(stringBuffer);
writer.StartObject();
{
SerializeSequenceReportBaseMembers(sequenceReport, writer);
}
writer.EndObject();
return stringBuffer.GetString();
}
AZStd::string SerializeSequenceReport(const Client::ImpactAnalysisSequenceReport& sequenceReport)
{
rapidjson::StringBuffer stringBuffer;
rapidjson::PrettyWriter<rapidjson::StringBuffer> writer(stringBuffer);
writer.StartObject();
{
SerializeDraftingSequenceReportMembers(sequenceReport, writer);
// Discarded test runs
writer.Key(SequenceReportFields::Keys[SequenceReportFields::DiscardedTestRuns]);
writer.StartArray();
for (const auto& testRun : sequenceReport.GetDiscardedTestRuns())
{
writer.String(testRun.c_str());
}
writer.EndArray(); // Discarded test runs
}
writer.EndObject();
return stringBuffer.GetString();
}
AZStd::string SerializeSequenceReport(const Client::SafeImpactAnalysisSequenceReport& sequenceReport)
{
rapidjson::StringBuffer stringBuffer;
rapidjson::PrettyWriter<rapidjson::StringBuffer> writer(stringBuffer);
writer.StartObject();
{
SerializeDraftingSequenceReportMembers(sequenceReport, writer);
// Discarded test runs
writer.Key(SequenceReportFields::Keys[SequenceReportFields::DiscardedTestRuns]);
SerializeTestSelection(sequenceReport.GetDiscardedTestRuns(), writer);
// Discarded test run report
writer.Key(SequenceReportFields::Keys[SequenceReportFields::DiscardedTestRunReport]);
SerializeTestRunReport(sequenceReport.GetDiscardedTestRunReport(), writer);
}
writer.EndObject();
return stringBuffer.GetString();
}
} // namespace TestImpact
@@ -8,11 +8,13 @@
#include <TestImpactFramework/TestImpactClientTestRun.h>
#include <AzCore/std/tuple.h>
namespace TestImpact
{
namespace Client
{
TestRun::TestRun(
TestRunBase::TestRunBase(
const AZStd::string& name,
const AZStd::string& commandString,
AZStd::chrono::high_resolution_clock::time_point startTime,
@@ -26,107 +28,145 @@ namespace TestImpact
{
}
const AZStd::string& TestRun::GetTargetName() const
const AZStd::string& TestRunBase::GetTargetName() const
{
return m_targetName;
}
const AZStd::string& TestRun::GetCommandString() const
const AZStd::string& TestRunBase::GetCommandString() const
{
return m_commandString;
}
AZStd::chrono::high_resolution_clock::time_point TestRun::GetStartTime() const
AZStd::chrono::high_resolution_clock::time_point TestRunBase::GetStartTime() const
{
return m_startTime;
}
AZStd::chrono::high_resolution_clock::time_point TestRun::GetEndTime() const
AZStd::chrono::high_resolution_clock::time_point TestRunBase::GetEndTime() const
{
return m_startTime + m_duration;
}
AZStd::chrono::milliseconds TestRun::GetDuration() const
AZStd::chrono::milliseconds TestRunBase::GetDuration() const
{
return m_duration;
}
TestRunResult TestRun::GetResult() const
TestRunResult TestRunBase::GetResult() const
{
return m_result;
}
TestFailure::TestFailure(const AZStd::string& testName, const AZStd::string& errorMessage)
TestRunWithExecutionFailure::TestRunWithExecutionFailure(TestRunBase&& testRun)
: TestRunBase(AZStd::move(testRun))
{
}
TimedOutTestRun::TimedOutTestRun(TestRunBase&& testRun)
: TestRunBase(AZStd::move(testRun))
{
}
UnexecutedTestRun::UnexecutedTestRun(TestRunBase&& testRun)
: TestRunBase(AZStd::move(testRun))
{
}
Test::Test(const AZStd::string& testName, TestResult result)
: m_name(testName)
, m_errorMessage(errorMessage)
, m_result(result)
{
}
const AZStd::string& TestFailure::GetName() const
const AZStd::string& Test::GetName() const
{
return m_name;
}
const AZStd::string& TestFailure::GetErrorMessage() const
TestResult Test::GetResult() const
{
return m_errorMessage;
return m_result;
}
TestCaseFailure::TestCaseFailure(const AZStd::string& testCaseName, AZStd::vector<TestFailure>&& testFailures)
: m_name(testCaseName)
, m_testFailures(AZStd::move(testFailures))
AZStd::tuple<size_t, size_t, size_t> CalculateTestCaseMetrics(const AZStd::vector<Test>& tests)
{
}
size_t totalNumPassingTests = 0;
size_t totalNumFailingTests = 0;
size_t totalNumDisabledTests = 0;
const AZStd::string& TestCaseFailure::GetName() const
{
return m_name;
}
const AZStd::vector<TestFailure>& TestCaseFailure::GetTestFailures() const
{
return m_testFailures;
}
static size_t CalculateNumTestRunFailures(const AZStd::vector<TestCaseFailure>& testFailures)
{
size_t numTestFailures = 0;
for (const auto& testCase : testFailures)
for (const auto& test : tests)
{
numTestFailures += testCase.GetTestFailures().size();
if (test.GetResult() == Client::TestResult::Passed)
{
totalNumPassingTests++;
}
else if (test.GetResult() == Client::TestResult::Failed)
{
totalNumFailingTests++;
}
else
{
totalNumDisabledTests++;
}
}
return numTestFailures;
return { totalNumPassingTests, totalNumFailingTests, totalNumDisabledTests };
}
TestRunWithTestFailures::TestRunWithTestFailures(
CompletedTestRun::CompletedTestRun(
const AZStd::string& name,
const AZStd::string& commandString,
AZStd::chrono::high_resolution_clock::time_point startTime,
AZStd::chrono::milliseconds duration,
TestRunResult result,
AZStd::vector<TestCaseFailure>&& testFailures)
: TestRun(name, commandString, startTime, duration, result)
, m_testCaseFailures(AZStd::move(testFailures))
AZStd::vector<Test>&& tests)
: TestRunBase(name, commandString, startTime, duration, result)
, m_tests(AZStd::move(tests))
{
m_numTestFailures = CalculateNumTestRunFailures(m_testCaseFailures);
AZStd::tie(m_totalNumPassingTests, m_totalNumFailingTests, m_totalNumDisabledTests) = CalculateTestCaseMetrics(m_tests);
}
TestRunWithTestFailures::TestRunWithTestFailures(TestRun&& testRun, AZStd::vector<TestCaseFailure>&& testFailures)
: TestRun(AZStd::move(testRun))
, m_testCaseFailures(AZStd::move(testFailures))
CompletedTestRun::CompletedTestRun(TestRunBase&& testRun, AZStd::vector<Test>&& tests)
: TestRunBase(AZStd::move(testRun))
, m_tests(AZStd::move(tests))
{
m_numTestFailures = CalculateNumTestRunFailures(m_testCaseFailures);
AZStd::tie(m_totalNumPassingTests, m_totalNumFailingTests, m_totalNumDisabledTests) = CalculateTestCaseMetrics(m_tests);
}
size_t TestRunWithTestFailures::GetNumTestFailures() const
size_t CompletedTestRun::GetTotalNumTests() const
{
return m_numTestFailures;
return m_tests.size();
}
const AZStd::vector<TestCaseFailure>& TestRunWithTestFailures::GetTestCaseFailures() const
size_t CompletedTestRun::GetTotalNumPassingTests() const
{
return m_totalNumPassingTests;
}
size_t CompletedTestRun::GetTotalNumFailingTests() const
{
return m_totalNumFailingTests;
}
size_t CompletedTestRun::GetTotalNumDisabledTests() const
{
return m_totalNumDisabledTests;
}
const AZStd::vector<Test>& CompletedTestRun::GetTests() const
{
return m_tests;
}
PassingTestRun::PassingTestRun(TestRunBase&& testRun, AZStd::vector<Test>&& tests)
: CompletedTestRun(AZStd::move(testRun), AZStd::move(tests))
{
}
FailingTestRun::FailingTestRun(TestRunBase&& testRun, AZStd::vector<Test>&& tests)
: CompletedTestRun(AZStd::move(testRun), AZStd::move(tests))
{
return m_testCaseFailures;
}
} // namespace Client
} // namespace TestImpact
@@ -6,7 +6,7 @@
*
*/
#include <TestImpactFramework/TestImpactFileUtils.h>
#include <TestImpactFramework/TestImpactUtils.h>
#include <TestImpactFramework/TestImpactRuntime.h>
#include <TestImpactFramework/TestImpactRuntimeException.h>
@@ -81,7 +81,7 @@ namespace TestImpact
{
if (m_testCompleteCallback.has_value())
{
Client::TestRun testRun(
Client::TestRunBase testRun(
testJob.GetTestTarget()->GetName(),
testJob.GetCommandString(),
testJob.GetStartTime(),
@@ -110,8 +110,140 @@ namespace TestImpact
return result;
}
//! Utility structure for holding the pertinent data for test run reports.
template<typename TestJob>
struct TestRunData
{
TestSequenceResult m_result = TestSequenceResult::Success;
AZStd::vector<TestJob> m_jobs;
AZStd::chrono::high_resolution_clock::time_point m_relativeStartTime;
AZStd::chrono::milliseconds m_duration = AZStd::chrono::milliseconds{ 0 };
};
//! Wrapper for the impact analysis test sequence to handle both the updating and non-updating policies through a common pathway.
//! @tparam TestRunnerFunctor The functor for running the specified tests.
//! @tparam TestJob The test engine job type returned by the functor.
//! @param maxConcurrency The maximum concurrency being used for this sequence.
//! @param policyState The policy state being used for the sequence.
//! @param suiteType The suite type used for this sequence.
//! @param timer The timer to use for the test run timings.
//! @param testRunner The test runner functor to use for each of the test runs.
//! @param includedSelectedTestTargets The subset of test targets that were selected to run and not also fully excluded from running.
//! @param excludedSelectedTestTargets The subset of test targets that were selected to run but were fully excluded running.
//! @param discardedTestTargets The subset of test targets that were discarded from the test selection and will not be run.
//! @param globalTimeout The maximum duration the entire test sequence may run for (infinite if empty).
//! @param testSequenceStartCallback The client function to be called after the test targets have been selected but prior to running the
//! tests.
//! @param testSequenceCompleteCallback The client function to be called after the test sequence has completed.
//! @param testRunCompleteCallback The client function to be called after an individual test run has completed.
//! @param updateCoverage The function to call to update the dynamic dependency map with test coverage (if any).
template<typename TestRunnerFunctor, typename TestJob>
Client::ImpactAnalysisSequenceReport ImpactAnalysisTestSequenceWrapper(
size_t maxConcurrency,
const ImpactAnalysisSequencePolicyState& policyState,
SuiteType suiteType,
const Timer& sequenceTimer,
const TestRunnerFunctor& testRunner,
const AZStd::vector<const TestTarget*>& includedSelectedTestTargets,
const AZStd::vector<const TestTarget*>& excludedSelectedTestTargets,
const AZStd::vector<const TestTarget*>& discardedTestTargets,
const AZStd::vector<const TestTarget*>& draftedTestTargets,
const AZStd::optional<AZStd::chrono::milliseconds>& testTargetTimeout,
const AZStd::optional<AZStd::chrono::milliseconds>& globalTimeout,
AZStd::optional<ImpactAnalysisTestSequenceStartCallback> testSequenceStartCallback,
AZStd::optional<TestSequenceCompleteCallback<Client::ImpactAnalysisSequenceReport>> testSequenceEndCallback,
AZStd::optional<TestRunCompleteCallback> testCompleteCallback,
AZStd::optional<AZStd::function<void(const AZStd::vector<TestJob>& jobs)>> updateCoverage)
{
TestRunData<TestJob> selectedTestRunData, draftedTestRunData;
AZStd::optional<AZStd::chrono::milliseconds> sequenceTimeout = globalTimeout;
// Extract the client facing representation of selected, discarded and drafted test targets
const Client::TestRunSelection selectedTests(
ExtractTestTargetNames(includedSelectedTestTargets), ExtractTestTargetNames(excludedSelectedTestTargets));
const auto discardedTests = ExtractTestTargetNames(discardedTestTargets);
const auto draftedTests = ExtractTestTargetNames(draftedTestTargets);
// Inform the client that the sequence is about to start
if (testSequenceStartCallback.has_value())
{
(*testSequenceStartCallback)(suiteType, selectedTests, discardedTests, draftedTests);
}
// We share the test run complete handler between the selected and drafted test runs as to present them together as one
// continuous test sequence to the client rather than two discrete test runs
const size_t totalNumTestRuns = includedSelectedTestTargets.size() + draftedTestTargets.size();
TestRunCompleteCallbackHandler testRunCompleteHandler(totalNumTestRuns, testCompleteCallback);
const auto gatherTestRunData = [&sequenceTimer, &testRunner, &testRunCompleteHandler, &globalTimeout]
(const AZStd::vector<const TestTarget*>& testsTargets, TestRunData<TestJob>& testRunData)
{
const Timer testRunTimer;
testRunData.m_relativeStartTime = testRunTimer.GetStartTimePointRelative(sequenceTimer);
auto [result, jobs] = testRunner(testsTargets, testRunCompleteHandler, globalTimeout);
testRunData.m_result = result;
testRunData.m_jobs = AZStd::move(jobs);
testRunData.m_duration = testRunTimer.GetElapsedMs();
};
if (!includedSelectedTestTargets.empty())
{
// Run the selected test targets and collect the test run results
gatherTestRunData(includedSelectedTestTargets, selectedTestRunData);
// Carry the remaining global sequence time over to the drafted test run
if (globalTimeout.has_value())
{
const auto elapsed = selectedTestRunData.m_duration;
sequenceTimeout = elapsed < globalTimeout.value() ? globalTimeout.value() - elapsed : AZStd::chrono::milliseconds(0);
}
}
if (!draftedTestTargets.empty())
{
// Run the drafted test targets and collect the test run results
gatherTestRunData(draftedTestTargets, draftedTestRunData);
}
// Generate the sequence report for the client
const auto sequenceReport = Client::ImpactAnalysisSequenceReport(
maxConcurrency,
testTargetTimeout,
globalTimeout,
policyState,
suiteType,
selectedTests,
discardedTests,
draftedTests,
GenerateTestRunReport(
selectedTestRunData.m_result,
selectedTestRunData.m_relativeStartTime,
selectedTestRunData.m_duration,
selectedTestRunData.m_jobs),
GenerateTestRunReport(
draftedTestRunData.m_result,
draftedTestRunData.m_relativeStartTime,
draftedTestRunData.m_duration,
draftedTestRunData.m_jobs));
// Inform the client that the sequence has ended
if (testSequenceEndCallback.has_value())
{
(*testSequenceEndCallback)(sequenceReport);
}
// Update the dynamic dependency map with the latest coverage data (if any)
if (updateCoverage.has_value())
{
(*updateCoverage)(ConcatenateVectors(selectedTestRunData.m_jobs, draftedTestRunData.m_jobs));
}
return sequenceReport;
}
Runtime::Runtime(
RuntimeConfig&& config,
AZStd::optional<RepoPath> dataFile,
SuiteType suiteFilter,
Policy::ExecutionFailure executionFailurePolicy,
Policy::FailedTestCoverage failedTestCoveragePolicy,
@@ -151,27 +283,22 @@ namespace TestImpact
try
{
if (dataFile.has_value())
{
m_sparTiaFile = dataFile.value().String();
}
else
{
m_sparTiaFile = m_config.m_workspace.m_active.m_sparTiaFiles[static_cast<size_t>(m_suiteFilter)].String();
}
// Populate the dynamic dependency map with the existing source coverage data (if any)
m_sparTIAFile = m_config.m_workspace.m_active.m_sparTIAFiles[static_cast<size_t>(m_suiteFilter)].String();
const auto tiaDataRaw = ReadFileContents<Exception>(m_sparTIAFile);
const auto tiaDataRaw = ReadFileContents<Exception>(m_sparTiaFile);
const auto tiaData = DeserializeSourceCoveringTestsList(tiaDataRaw);
if (tiaData.GetNumSources())
{
m_dynamicDependencyMap->ReplaceSourceCoverage(tiaData);
m_hasImpactAnalysisData = true;
// Enumerate new test targets
const auto testTargetsWithNoEnumeration = m_dynamicDependencyMap->GetNotCoveringTests();
if (!testTargetsWithNoEnumeration.empty())
{
m_testEngine->UpdateEnumerationCache(
testTargetsWithNoEnumeration,
Policy::ExecutionFailure::Ignore,
Policy::TestFailure::Continue,
AZStd::nullopt,
AZStd::nullopt,
AZStd::nullopt);
}
}
}
catch (const DependencyException& e)
@@ -186,7 +313,7 @@ namespace TestImpact
AZ_Printf(
LogCallSite,
AZStd::string::format(
"No test impact analysis data found for suite '%s' at %s\n", GetSuiteTypeName(m_suiteFilter).c_str(), m_sparTIAFile.c_str()).c_str());
"No test impact analysis data found for suite '%s' at %s\n", SuiteTypeAsString(m_suiteFilter).c_str(), m_sparTiaFile.c_str()).c_str());
}
}
@@ -230,7 +357,7 @@ namespace TestImpact
}
}
AZStd::pair<AZStd::vector<const TestTarget*>, AZStd::vector<const TestTarget*>> Runtime::SelectCoveringTestTargetsAndUpdateEnumerationCache(
AZStd::pair<AZStd::vector<const TestTarget*>, AZStd::vector<const TestTarget*>> Runtime::SelectCoveringTestTargets(
const ChangeList& changeList,
Policy::TestPrioritization testPrioritizationPolicy)
{
@@ -243,9 +370,6 @@ namespace TestImpact
// Populate a set with the selected test targets so that we can infer the discarded test target not selected for this change list
const AZStd::unordered_set<const TestTarget*> selectedTestTargetSet(selectedTestTargets.begin(), selectedTestTargets.end());
// Update the enumeration caches of mutated targets regardless of the current sharding policy
EnumerateMutatedTestTargets(changeDependencyList);
// The test targets in the main list not in the selected test target set are the test targets not selected for this change list
for (const auto& testTarget : m_dynamicDependencyMap->GetTestTargetList().GetTargets())
{
@@ -287,7 +411,7 @@ namespace TestImpact
void Runtime::ClearDynamicDependencyMapAndRemoveExistingFile()
{
m_dynamicDependencyMap->ClearAllSourceCoverage();
DeleteFile(m_sparTIAFile);
DeleteFile(m_sparTiaFile);
}
SourceCoveringTestsList Runtime::CreateSourceCoveringTestFromTestCoverages(const AZStd::vector<TestEngineInstrumentedRun>& jobs)
@@ -368,9 +492,9 @@ namespace TestImpact
}
m_dynamicDependencyMap->ReplaceSourceCoverage(sourceCoverageTestsList);
const auto sparTIA = m_dynamicDependencyMap->ExportSourceCoverage();
const auto sparTIAData = SerializeSourceCoveringTestsList(sparTIA);
WriteFileContents<RuntimeException>(sparTIAData, m_sparTIAFile);
const auto sparTia = m_dynamicDependencyMap->ExportSourceCoverage();
const auto sparTiaData = SerializeSourceCoveringTestsList(sparTia);
WriteFileContents<RuntimeException>(sparTiaData, m_sparTiaFile);
m_hasImpactAnalysisData = true;
}
catch(const RuntimeException& e)
@@ -386,11 +510,42 @@ namespace TestImpact
}
}
Client::SequenceReport Runtime::RegularTestSequence(
PolicyStateBase Runtime::GeneratePolicyStateBase() const
{
PolicyStateBase policyState;
policyState.m_executionFailurePolicy = m_executionFailurePolicy;
policyState.m_failedTestCoveragePolicy = m_failedTestCoveragePolicy;
policyState.m_integrityFailurePolicy = m_integrationFailurePolicy;
policyState.m_targetOutputCapture = m_targetOutputCapture;
policyState.m_testFailurePolicy = m_testFailurePolicy;
policyState.m_testShardingPolicy = m_testShardingPolicy;
return policyState;
}
SequencePolicyState Runtime::GenerateSequencePolicyState() const
{
return { GeneratePolicyStateBase() };
}
SafeImpactAnalysisSequencePolicyState Runtime::GenerateSafeImpactAnalysisSequencePolicyState(
Policy::TestPrioritization testPrioritizationPolicy) const
{
return { GeneratePolicyStateBase(), testPrioritizationPolicy };
}
ImpactAnalysisSequencePolicyState Runtime::GenerateImpactAnalysisSequencePolicyState(
Policy::TestPrioritization testPrioritizationPolicy, Policy::DynamicDependencyMap dynamicDependencyMapPolicy) const
{
return { GeneratePolicyStateBase(), testPrioritizationPolicy, dynamicDependencyMapPolicy };
}
Client::RegularSequenceReport Runtime::RegularTestSequence(
AZStd::optional<AZStd::chrono::milliseconds> testTargetTimeout,
AZStd::optional<AZStd::chrono::milliseconds> globalTimeout,
AZStd::optional<TestSequenceStartCallback> testSequenceStartCallback,
AZStd::optional<TestSequenceCompleteCallback<Client::SequenceReport>> testSequenceEndCallback,
AZStd::optional<TestSequenceCompleteCallback<Client::RegularSequenceReport>> testSequenceEndCallback,
AZStd::optional<TestRunCompleteCallback> testCompleteCallback)
{
const Timer sequenceTimer;
@@ -434,7 +589,11 @@ namespace TestImpact
const auto testRunDuration = testRunTimer.GetElapsedMs();
// Generate the sequence report for the client
const auto sequenceReport = Client::SequenceReport(
const auto sequenceReport = Client::RegularSequenceReport(
m_maxConcurrency,
testTargetTimeout,
globalTimeout,
GenerateSequencePolicyState(),
m_suiteFilter,
selectedTests,
GenerateTestRunReport(result, testRunTimer.GetStartTimePointRelative(sequenceTimer), testRunDuration, testJobs));
@@ -448,95 +607,6 @@ namespace TestImpact
return sequenceReport;
}
//! Wrapper for the impact analysis test sequence to handle both the updating and non-updating policies through a common pathway.
//! @tparam TestRunnerFunctor The functor for running the specified tests.
//! @tparam TestJob The test engine job type returned by the functor.
//! @param suiteType The suite type used for this sequence.
//! @param timer The timer to use for the test run timings.
//! @param testRunner The test runner functor to use for each of the test runs.
//! @param includedSelectedTestTargets The subset of test targets that were selected to run and not also fully excluded from running.
//! @param excludedSelectedTestTargets The subset of test targets that were selected to run but were fully excluded running.
//! @param discardedTestTargets The subset of test targets that were discarded from the test selection and will not be run.
//! @param globalTimeout The maximum duration the entire test sequence may run for (infinite if empty).
//! @param testSequenceStartCallback The client function to be called after the test targets have been selected but prior to running the tests.
//! @param testSequenceCompleteCallback The client function to be called after the test sequence has completed.
//! @param testRunCompleteCallback The client function to be called after an individual test run has completed.
//! @param updateCoverage The function to call to update the dynamic dependency map with test coverage (if any).
template<typename TestRunnerFunctor, typename TestJob>
Client::ImpactAnalysisSequenceReport ImpactAnalysisTestSequenceWrapper(
SuiteType suiteType,
const Timer& sequenceTimer,
const TestRunnerFunctor& testRunner,
const AZStd::vector<const TestTarget*>& includedSelectedTestTargets,
const AZStd::vector<const TestTarget*>& excludedSelectedTestTargets,
const AZStd::vector<const TestTarget*>& discardedTestTargets,
const AZStd::vector<const TestTarget*>& draftedTestTargets,
const AZStd::optional<AZStd::chrono::milliseconds> globalTimeout,
AZStd::optional<ImpactAnalysisTestSequenceStartCallback> testSequenceStartCallback,
AZStd::optional<TestSequenceCompleteCallback<Client::ImpactAnalysisSequenceReport>> testSequenceEndCallback,
AZStd::optional<TestRunCompleteCallback> testCompleteCallback,
AZStd::optional<AZStd::function<void(const AZStd::vector<TestJob>& jobs)>> updateCoverage)
{
AZStd::optional<AZStd::chrono::milliseconds> sequenceTimeout = globalTimeout;
// Extract the client facing representation of selected, discarded and drafted test targets
const Client::TestRunSelection selectedTests(
ExtractTestTargetNames(includedSelectedTestTargets), ExtractTestTargetNames(excludedSelectedTestTargets));
const auto discardedTests = ExtractTestTargetNames(discardedTestTargets);
const auto draftedTests = ExtractTestTargetNames(draftedTestTargets);
// Inform the client that the sequence is about to start
if (testSequenceStartCallback.has_value())
{
(*testSequenceStartCallback)(suiteType, selectedTests, discardedTests, draftedTests);
}
// We share the test run complete handler between the selected and drafted test runs as to present them together as one
// continuous test sequence to the client rather than two discrete test runs
const size_t totalNumTestRuns = includedSelectedTestTargets.size() + draftedTestTargets.size();
TestRunCompleteCallbackHandler testRunCompleteHandler(totalNumTestRuns, testCompleteCallback);
// Run the selected test targets and collect the test run results
const Timer selectedTestRunTimer;
const auto [selectedResult, selectedTestJobs] = testRunner(includedSelectedTestTargets, testRunCompleteHandler, globalTimeout);
const auto selectedTestRunDuration = selectedTestRunTimer.GetElapsedMs();
// Carry the remaining global sequence time over to the drafted test run
if (globalTimeout.has_value())
{
const auto elapsed = selectedTestRunDuration;
sequenceTimeout = elapsed < globalTimeout.value() ? globalTimeout.value() - elapsed : AZStd::chrono::milliseconds(0);
}
// Run the drafted test targets and collect the test run results
Timer draftedTestRunTimer;
const auto [draftedResult, draftedTestJobs] = testRunner(draftedTestTargets, testRunCompleteHandler, globalTimeout);
const auto draftedTestRunDuration = draftedTestRunTimer.GetElapsedMs();
// Generate the sequence report for the client
const auto sequenceReport = Client::ImpactAnalysisSequenceReport(
suiteType,
selectedTests,
discardedTests,
draftedTests,
GenerateTestRunReport(selectedResult, selectedTestRunTimer.GetStartTimePointRelative(sequenceTimer), selectedTestRunDuration, selectedTestJobs),
GenerateTestRunReport(draftedResult, draftedTestRunTimer.GetStartTimePointRelative(sequenceTimer), draftedTestRunDuration, draftedTestJobs));
// Inform the client that the sequence has ended
if (testSequenceEndCallback.has_value())
{
(*testSequenceEndCallback)(sequenceReport);
}
// Update the dynamic dependency map with the latest coverage data (if any)
if (updateCoverage.has_value())
{
(*updateCoverage)(ConcatenateVectors(selectedTestJobs, draftedTestJobs));
}
return sequenceReport;
}
Client::ImpactAnalysisSequenceReport Runtime::ImpactAnalysisTestSequence(
const ChangeList& changeList,
Policy::TestPrioritization testPrioritizationPolicy,
@@ -550,10 +620,30 @@ namespace TestImpact
const Timer sequenceTimer;
// Draft in the test targets that have no coverage entries in the dynamic dependency map
AZStd::vector<const TestTarget*> draftedTestTargets = m_dynamicDependencyMap->GetNotCoveringTests();
const AZStd::vector<const TestTarget*> draftedTestTargets = m_dynamicDependencyMap->GetNotCoveringTests();
// The test targets that were selected for the change list by the dynamic dependency map and the test targets that were not
auto [selectedTestTargets, discardedTestTargets] = SelectCoveringTestTargetsAndUpdateEnumerationCache(changeList, testPrioritizationPolicy);
const auto selectCoveringTestTargetsAndPruneDraftedFromDiscarded =
[this, &draftedTestTargets, &changeList, testPrioritizationPolicy]()
{
// The test targets that were selected for the change list by the dynamic dependency map and the test targets that were not
const auto [selectedTestTargets, discardedTestTargets] =
SelectCoveringTestTargets(changeList, testPrioritizationPolicy);
const AZStd::unordered_set<const TestTarget*> draftedTestTargetsSet(draftedTestTargets.begin(), draftedTestTargets.end());
AZStd::vector<const TestTarget*> discardedNotDraftedTestTargets;
for (const auto* testTarget : discardedTestTargets)
{
if (!draftedTestTargetsSet.count(testTarget))
{
discardedNotDraftedTestTargets.push_back(testTarget);
}
}
return AZStd::pair{ selectedTestTargets, discardedNotDraftedTestTargets };
};
const auto [selectedTestTargets, discardedTestTargets] = selectCoveringTestTargetsAndPruneDraftedFromDiscarded();
// The subset of selected test targets that are not on the configuration's exclude list and those that are
auto [includedSelectedTestTargets, excludedSelectedTestTargets] = SelectTestTargetsByExcludeList(selectedTestTargets);
@@ -604,6 +694,8 @@ namespace TestImpact
};
return ImpactAnalysisTestSequenceWrapper(
m_maxConcurrency,
GenerateImpactAnalysisSequencePolicyState(testPrioritizationPolicy, dynamicDependencyMapPolicy),
m_suiteFilter,
sequenceTimer,
instrumentedTestRun,
@@ -611,6 +703,7 @@ namespace TestImpact
excludedSelectedTestTargets,
discardedTestTargets,
draftedTestTargets,
testTargetTimeout,
globalTimeout,
testSequenceStartCallback,
testSequenceEndCallback,
@@ -620,6 +713,8 @@ namespace TestImpact
else
{
return ImpactAnalysisTestSequenceWrapper(
m_maxConcurrency,
GenerateImpactAnalysisSequencePolicyState(testPrioritizationPolicy, dynamicDependencyMapPolicy),
m_suiteFilter,
sequenceTimer,
regularTestRun,
@@ -627,6 +722,7 @@ namespace TestImpact
excludedSelectedTestTargets,
discardedTestTargets,
draftedTestTargets,
testTargetTimeout,
globalTimeout,
testSequenceStartCallback,
testSequenceEndCallback,
@@ -645,13 +741,15 @@ namespace TestImpact
AZStd::optional<TestRunCompleteCallback> testCompleteCallback)
{
const Timer sequenceTimer;
auto sequenceTimeout = globalTimeout;
TestRunData<TestEngineInstrumentedRun> selectedTestRunData, draftedTestRunData;
TestRunData<TestEngineRegularRun> discardedTestRunData;
AZStd::optional<AZStd::chrono::milliseconds> sequenceTimeout = globalTimeout;
// Draft in the test targets that have no coverage entries in the dynamic dependency map
AZStd::vector<const TestTarget*> draftedTestTargets = m_dynamicDependencyMap->GetNotCoveringTests();
// The test targets that were selected for the change list by the dynamic dependency map and the test targets that were not
auto [selectedTestTargets, discardedTestTargets] = SelectCoveringTestTargetsAndUpdateEnumerationCache(changeList, testPrioritizationPolicy);
const auto [selectedTestTargets, discardedTestTargets] = SelectCoveringTestTargets(changeList, testPrioritizationPolicy);
// The subset of selected test targets that are not on the configuration's exclude list and those that are
auto [includedSelectedTestTargets, excludedSelectedTestTargets] = SelectTestTargetsByExcludeList(selectedTestTargets);
@@ -675,71 +773,107 @@ namespace TestImpact
// continuous test sequence to the client rather than three discrete test runs
const size_t totalNumTestRuns = includedSelectedTestTargets.size() + draftedTestTargets.size() + includedDiscardedTestTargets.size();
TestRunCompleteCallbackHandler testRunCompleteHandler(totalNumTestRuns, testCompleteCallback);
// Run the selected test targets and collect the test run results
const Timer selectedTestRunTimer;
const auto [selectedResult, selectedTestJobs] = m_testEngine->InstrumentedRun(
includedSelectedTestTargets,
m_testShardingPolicy,
m_executionFailurePolicy,
m_integrationFailurePolicy,
m_testFailurePolicy,
m_targetOutputCapture,
testTargetTimeout,
sequenceTimeout,
AZStd::ref(testRunCompleteHandler));
const auto selectedTestRunDuration = selectedTestRunTimer.GetElapsedMs();
// Carry the remaining global sequence time over to the discarded test run
if (globalTimeout.has_value())
// Functor for running instrumented test targets
const auto instrumentedTestRun =
[this, &testTargetTimeout, &sequenceTimeout, &testRunCompleteHandler](const AZStd::vector<const TestTarget*>& testsTargets)
{
const auto elapsed = selectedTestRunDuration;
sequenceTimeout = elapsed < globalTimeout.value() ? globalTimeout.value() - elapsed : AZStd::chrono::milliseconds(0);
return m_testEngine->InstrumentedRun(
testsTargets,
m_testShardingPolicy,
m_executionFailurePolicy,
m_integrationFailurePolicy,
m_testFailurePolicy,
m_targetOutputCapture,
testTargetTimeout,
sequenceTimeout,
AZStd::ref(testRunCompleteHandler));
};
// Functor for running uninstrumented test targets
const auto regularTestRun =
[this, &testTargetTimeout, &sequenceTimeout, &testRunCompleteHandler](const AZStd::vector<const TestTarget*>& testsTargets)
{
return m_testEngine->RegularRun(
testsTargets,
m_testShardingPolicy,
m_executionFailurePolicy,
m_testFailurePolicy,
m_targetOutputCapture,
testTargetTimeout,
sequenceTimeout,
AZStd::ref(testRunCompleteHandler));
};
// Functor for running instrumented test targets
const auto gatherTestRunData = [&sequenceTimer]
(const AZStd::vector<const TestTarget*>& testsTargets, const auto& testRunner, auto& testRunData)
{
const Timer testRunTimer;
testRunData.m_relativeStartTime = testRunTimer.GetStartTimePointRelative(sequenceTimer);
auto [result, jobs] = testRunner(testsTargets);
testRunData.m_result = result;
testRunData.m_jobs = AZStd::move(jobs);
testRunData.m_duration = testRunTimer.GetElapsedMs();
};
if (!includedSelectedTestTargets.empty())
{
// Run the selected test targets and collect the test run results
gatherTestRunData(includedSelectedTestTargets, instrumentedTestRun, selectedTestRunData);
// Carry the remaining global sequence time over to the discarded test run
if (globalTimeout.has_value())
{
const auto elapsed = selectedTestRunData.m_duration;
sequenceTimeout = elapsed < globalTimeout.value() ? globalTimeout.value() - elapsed : AZStd::chrono::milliseconds(0);
}
}
// Run the discarded test targets and collect the test run results
const Timer discardedTestRunTimer;
const auto [discardedResult, discardedTestJobs] = m_testEngine->RegularRun(
includedDiscardedTestTargets,
m_testShardingPolicy,
m_executionFailurePolicy,
m_testFailurePolicy,
m_targetOutputCapture,
testTargetTimeout,
sequenceTimeout,
AZStd::ref(testRunCompleteHandler));
const auto discardedTestRunDuration = discardedTestRunTimer.GetElapsedMs();
// Carry the remaining global sequence time over to the drafted test run
if (globalTimeout.has_value())
if (!includedDiscardedTestTargets.empty())
{
const auto elapsed = selectedTestRunDuration + discardedTestRunDuration;
sequenceTimeout = elapsed < globalTimeout.value() ? globalTimeout.value() - elapsed : AZStd::chrono::milliseconds(0);
// Run the discarded test targets and collect the test run results
gatherTestRunData(includedDiscardedTestTargets, regularTestRun, discardedTestRunData);
// Carry the remaining global sequence time over to the drafted test run
if (globalTimeout.has_value())
{
const auto elapsed = selectedTestRunData.m_duration + discardedTestRunData.m_duration;
sequenceTimeout = elapsed < globalTimeout.value() ? globalTimeout.value() - elapsed : AZStd::chrono::milliseconds(0);
}
}
// Run the drafted test targets and collect the test run results
const Timer draftedTestRunTimer;
const auto [draftedResult, draftedTestJobs] = m_testEngine->InstrumentedRun(
draftedTestTargets,
m_testShardingPolicy,
m_executionFailurePolicy,
m_integrationFailurePolicy,
m_testFailurePolicy,
m_targetOutputCapture,
testTargetTimeout,
sequenceTimeout,
AZStd::ref(testRunCompleteHandler));
const auto draftedTestRunDuration = draftedTestRunTimer.GetElapsedMs();
if (!draftedTestTargets.empty())
{
// Run the drafted test targets and collect the test run results
gatherTestRunData(draftedTestTargets, instrumentedTestRun, draftedTestRunData);
}
// Generate the sequence report for the client
const auto sequenceReport = Client::SafeImpactAnalysisSequenceReport(
m_maxConcurrency,
testTargetTimeout,
globalTimeout,
GenerateSafeImpactAnalysisSequencePolicyState(testPrioritizationPolicy),
m_suiteFilter,
selectedTests,
discardedTests,
draftedTests,
GenerateTestRunReport(selectedResult, selectedTestRunTimer.GetStartTimePointRelative(sequenceTimer), selectedTestRunDuration, selectedTestJobs),
GenerateTestRunReport(discardedResult, discardedTestRunTimer.GetStartTimePointRelative(sequenceTimer), discardedTestRunDuration, discardedTestJobs),
GenerateTestRunReport(draftedResult, draftedTestRunTimer.GetStartTimePointRelative(sequenceTimer), draftedTestRunDuration, draftedTestJobs));
GenerateTestRunReport(
selectedTestRunData.m_result,
selectedTestRunData.m_relativeStartTime,
selectedTestRunData.m_duration,
selectedTestRunData.m_jobs),
GenerateTestRunReport(
discardedTestRunData.m_result,
discardedTestRunData.m_relativeStartTime,
discardedTestRunData.m_duration,
discardedTestRunData.m_jobs),
GenerateTestRunReport(
draftedTestRunData.m_result,
draftedTestRunData.m_relativeStartTime,
draftedTestRunData.m_duration,
draftedTestRunData.m_jobs));
// Inform the client that the sequence has ended
if (testSequenceEndCallback.has_value())
@@ -747,15 +881,15 @@ namespace TestImpact
(*testSequenceEndCallback)(sequenceReport);
}
UpdateAndSerializeDynamicDependencyMap(ConcatenateVectors(selectedTestJobs, draftedTestJobs));
UpdateAndSerializeDynamicDependencyMap(ConcatenateVectors(selectedTestRunData.m_jobs, draftedTestRunData.m_jobs));
return sequenceReport;
}
Client::SequenceReport Runtime::SeededTestSequence(
Client::SeedSequenceReport Runtime::SeededTestSequence(
AZStd::optional<AZStd::chrono::milliseconds> testTargetTimeout,
AZStd::optional<AZStd::chrono::milliseconds> globalTimeout,
AZStd::optional<TestSequenceStartCallback> testSequenceStartCallback,
AZStd::optional<TestSequenceCompleteCallback<Client::SequenceReport>> testSequenceEndCallback,
AZStd::optional<TestSequenceCompleteCallback<Client::SeedSequenceReport>> testSequenceEndCallback,
AZStd::optional<TestRunCompleteCallback> testCompleteCallback)
{
const Timer sequenceTimer;
@@ -799,7 +933,11 @@ namespace TestImpact
const auto testRunDuration = testRunTimer.GetElapsedMs();
// Generate the sequence report for the client
const auto sequenceReport = Client::SequenceReport(
const auto sequenceReport = Client::SeedSequenceReport(
m_maxConcurrency,
testTargetTimeout,
globalTimeout,
GenerateSequencePolicyState(),
m_suiteFilter,
selectedTests,
GenerateTestRunReport(result, testRunTimer.GetStartTimePointRelative(sequenceTimer), testRunDuration, testJobs));
@@ -6,7 +6,7 @@
*
*/
#include <TestImpactFramework/TestImpactFileUtils.h>
#include <TestImpactFramework/TestImpactUtils.h>
#include <TestImpactFramework/TestImpactRuntimeException.h>
#include <TestImpactRuntimeUtils.h>
@@ -38,34 +38,44 @@ namespace TestImpact
//! Extracts the name information from the specified test targets.
AZStd::vector<AZStd::string> ExtractTestTargetNames(const AZStd::vector<const TestTarget*>& testTargets);
//! Generates a test run failure report from the specified test engine job information.
//! Generates the test suites from the specified test engine job information.
//! @tparam TestJob The test engine job type.
template<typename TestJob>
AZStd::vector<Client::TestCaseFailure> GenerateTestCaseFailures(const TestJob& testJob)
AZStd::vector<Client::Test> GenerateClientTests(const TestJob& testJob)
{
AZStd::vector<Client::TestCaseFailure> testCaseFailures;
AZStd::vector<Client::Test> tests;
if (testJob.GetTestRun().has_value())
{
for (const auto& testSuite : testJob.GetTestRun()->GetTestSuites())
{
AZStd::vector<Client::TestFailure> testFailures;
for (const auto& testCase : testSuite.m_tests)
{
if (testCase.m_result.value_or(TestRunResult::Passed) == TestRunResult::Failed)
auto result = Client::TestResult::NotRun;
if (testCase.m_result.has_value())
{
testFailures.push_back(Client::TestFailure(testCase.m_name, "No error message retrieved"));
if (testCase.m_result.value() == TestRunResult::Passed)
{
result = Client::TestResult::Passed;
}
else if (testCase.m_result.value() == TestRunResult::Failed)
{
result = Client::TestResult::Failed;
}
else
{
throw RuntimeException(AZStd::string::format(
"Unexpected test run result: %u", aznumeric_cast<AZ::u32>(testCase.m_result.value())));
}
}
}
if (!testFailures.empty())
{
testCaseFailures.push_back(Client::TestCaseFailure(testSuite.m_name, AZStd::move(testFailures)));
const auto name = AZStd::string::format("%s.%s", testSuite.m_name.c_str(), testCase.m_name.c_str());
tests.push_back(Client::Test(name, result));
}
}
}
return testCaseFailures;
return tests;
}
template<typename TestJob>
@@ -75,11 +85,11 @@ namespace TestImpact
AZStd::chrono::milliseconds duration,
const AZStd::vector<TestJob>& testJobs)
{
AZStd::vector<Client::TestRun> passingTests;
AZStd::vector<Client::TestRunWithTestFailures> failingTests;
AZStd::vector<Client::TestRun> executionFailureTests;
AZStd::vector<Client::TestRun> timedOutTests;
AZStd::vector<Client::TestRun> unexecutedTests;
AZStd::vector<Client::PassingTestRun> passingTests;
AZStd::vector<Client::FailingTestRun> failingTests;
AZStd::vector<Client::TestRunWithExecutionFailure> executionFailureTests;
AZStd::vector<Client::TimedOutTestRun> timedOutTests;
AZStd::vector<Client::UnexecutedTestRun> unexecutedTests;
for (const auto& testJob : testJobs)
{
@@ -88,7 +98,7 @@ namespace TestImpact
AZStd::chrono::high_resolution_clock::time_point() +
AZStd::chrono::duration_cast<AZStd::chrono::milliseconds>(testJob.GetStartTime() - startTime);
Client::TestRun clientTestRun(
Client::TestRunBase clientTestRun(
testJob.GetTestTarget()->GetName(), testJob.GetCommandString(), relativeStartTime, testJob.GetDuration(),
testJob.GetTestResult());
@@ -96,27 +106,27 @@ namespace TestImpact
{
case Client::TestRunResult::FailedToExecute:
{
executionFailureTests.push_back(clientTestRun);
executionFailureTests.emplace_back(AZStd::move(clientTestRun));
break;
}
case Client::TestRunResult::NotRun:
{
unexecutedTests.push_back(clientTestRun);
unexecutedTests.emplace_back(AZStd::move(clientTestRun));
break;
}
case Client::TestRunResult::Timeout:
{
timedOutTests.push_back(clientTestRun);
timedOutTests.emplace_back(AZStd::move(clientTestRun));
break;
}
case Client::TestRunResult::AllTestsPass:
{
passingTests.push_back(clientTestRun);
passingTests.emplace_back(AZStd::move(clientTestRun), GenerateClientTests(testJob));
break;
}
case Client::TestRunResult::TestFailures:
{
failingTests.emplace_back(AZStd::move(clientTestRun), GenerateTestCaseFailures(testJob));
failingTests.emplace_back(AZStd::move(clientTestRun), GenerateClientTests(testJob));
break;
}
default:
@@ -0,0 +1,244 @@
/*
* Copyright (c) Contributors to the Open 3D Engine Project.
* For complete copyright and license terms please see the LICENSE at the root of this distribution.
*
* SPDX-License-Identifier: Apache-2.0 OR MIT
*
*/
#include <TestImpactFramework/TestImpactException.h>
#include <TestImpactFramework/TestImpactUtils.h>
#include <AzCore/std/functional.h>
namespace TestImpact
{
//! Delete the files that match the pattern from the specified directory.
//! @param path The path to the directory to pattern match the files for deletion.
//! @param pattern The pattern to match files for deletion.
size_t DeleteFiles(const RepoPath& path, const AZStd::string& pattern)
{
size_t numFilesDeleted = 0;
AZ::IO::SystemFile::FindFiles(
AZStd::string::format("%s/%s", path.c_str(), pattern.c_str()).c_str(),
[&path, &numFilesDeleted](const char* file, bool isFile)
{
if (isFile)
{
AZ::IO::SystemFile::Delete(AZStd::string::format("%s/%s", path.c_str(), file).c_str());
numFilesDeleted++;
}
return true;
});
return numFilesDeleted;
}
//! Deletes the specified file.
void DeleteFile(const RepoPath& file)
{
DeleteFiles(file.ParentPath(), file.Filename().Native());
}
//! User-friendly names for the test suite types.
AZStd::string SuiteTypeAsString(SuiteType suiteType)
{
switch (suiteType)
{
case SuiteType::Main:
return "main";
case SuiteType::Periodic:
return "periodic";
case SuiteType::Sandbox:
return "sandbox";
default:
throw(Exception("Unexpected suite type"));
}
}
AZStd::string SequenceReportTypeAsString(Client::SequenceReportType type)
{
switch (type)
{
case Client::SequenceReportType::RegularSequence:
return "regular";
case Client::SequenceReportType::SeedSequence:
return "seed";
case Client::SequenceReportType::ImpactAnalysisSequence:
return "impact_analysis";
case Client::SequenceReportType::SafeImpactAnalysisSequence:
return "safe_impact_analysis";
default:
throw(Exception(AZStd::string::format("Unexpected sequence report type: %u", aznumeric_cast<AZ::u32>(type))));
}
}
AZStd::string TestSequenceResultAsString(TestSequenceResult result)
{
switch (result)
{
case TestSequenceResult::Failure:
return "failure";
case TestSequenceResult::Success:
return "success";
case TestSequenceResult::Timeout:
return "timeout";
default:
throw(Exception(AZStd::string::format("Unexpected test sequence result: %u", aznumeric_cast<AZ::u32>(result))));
}
}
AZStd::string TestRunResultAsString(Client::TestRunResult result)
{
switch (result)
{
case Client::TestRunResult::AllTestsPass:
return "all_tests_pass";
case Client::TestRunResult::FailedToExecute:
return "failed_to_execute";
case Client::TestRunResult::NotRun:
return "not_run";
case Client::TestRunResult::TestFailures:
return "test_failures";
case Client::TestRunResult::Timeout:
return "timeout";
default:
throw(Exception(AZStd::string::format("Unexpected test run result: %u", aznumeric_cast<AZ::u32>(result))));
}
}
AZStd::string ExecutionFailurePolicyAsString(Policy::ExecutionFailure executionFailurePolicy)
{
switch (executionFailurePolicy)
{
case Policy::ExecutionFailure::Abort:
return "abort";
case Policy::ExecutionFailure::Continue:
return "continue";
case Policy::ExecutionFailure::Ignore:
return "ignore";
default:
throw(Exception(
AZStd::string::format("Unexpected execution failure policy: %u", aznumeric_cast<AZ::u32>(executionFailurePolicy))));
}
}
AZStd::string FailedTestCoveragePolicyAsString(Policy::FailedTestCoverage failedTestCoveragePolicy)
{
switch (failedTestCoveragePolicy)
{
case Policy::FailedTestCoverage::Discard:
return "discard";
case Policy::FailedTestCoverage::Keep:
return "keep";
default:
throw(Exception(
AZStd::string::format("Unexpected failed test coverage policy: %u", aznumeric_cast<AZ::u32>(failedTestCoveragePolicy))));
}
}
AZStd::string TestPrioritizationPolicyAsString(Policy::TestPrioritization testPrioritizationPolicy)
{
switch (testPrioritizationPolicy)
{
case Policy::TestPrioritization::DependencyLocality:
return "dependency_locality";
case Policy::TestPrioritization::None:
return "none";
default:
throw(Exception(
AZStd::string::format("Unexpected test prioritization policy: %u", aznumeric_cast<AZ::u32>(testPrioritizationPolicy))));
}
}
AZStd::string TestFailurePolicyAsString(Policy::TestFailure testFailurePolicy)
{
switch (testFailurePolicy)
{
case Policy::TestFailure::Abort:
return "abort";
case Policy::TestFailure::Continue:
return "continue";
default:
throw(
Exception(AZStd::string::format("Unexpected test failure policy: %u", aznumeric_cast<AZ::u32>(testFailurePolicy))));
}
}
AZStd::string IntegrityFailurePolicyAsString(Policy::IntegrityFailure integrityFailurePolicy)
{
switch (integrityFailurePolicy)
{
case Policy::IntegrityFailure::Abort:
return "abort";
case Policy::IntegrityFailure::Continue:
return "continue";
default:
throw(Exception(
AZStd::string::format("Unexpected integration failure policy: %u", aznumeric_cast<AZ::u32>(integrityFailurePolicy))));
}
}
AZStd::string DynamicDependencyMapPolicyAsString(Policy::DynamicDependencyMap dynamicDependencyMapPolicy)
{
switch (dynamicDependencyMapPolicy)
{
case Policy::DynamicDependencyMap::Discard:
return "discard";
case Policy::DynamicDependencyMap::Update:
return "update";
default:
throw(Exception(AZStd::string::format(
"Unexpected dynamic dependency map policy: %u", aznumeric_cast<AZ::u32>(dynamicDependencyMapPolicy))));
}
}
AZStd::string TestShardingPolicyAsString(Policy::TestSharding testShardingPolicy)
{
switch (testShardingPolicy)
{
case Policy::TestSharding::Always:
return "always";
case Policy::TestSharding::Never:
return "never";
default:
throw(Exception(
AZStd::string::format("Unexpected test sharding policy: %u", aznumeric_cast<AZ::u32>(testShardingPolicy))));
}
}
AZStd::string TargetOutputCapturePolicyAsString(Policy::TargetOutputCapture targetOutputCapturePolicy)
{
switch (targetOutputCapturePolicy)
{
case Policy::TargetOutputCapture::File:
return "file";
case Policy::TargetOutputCapture::None:
return "none";
case Policy::TargetOutputCapture::StdOut:
return "stdout";
case Policy::TargetOutputCapture::StdOutAndFile:
return "stdout_file";
default:
throw(Exception(
AZStd::string::format("Unexpected target output capture policy: %u", aznumeric_cast<AZ::u32>(targetOutputCapturePolicy))));
}
}
AZStd::string ClientTestResultAsString(Client::TestResult result)
{
switch (result)
{
case Client::TestResult::Failed:
return "failed";
case Client::TestResult::NotRun:
return "not_run";
case Client::TestResult::Passed:
return "passed";
default:
throw(Exception(AZStd::string::format("Unexpected client test case result: %u", aznumeric_cast<AZ::u32>(result))));
}
}
} // namespace TestImpact