Add changes from TIF/Feature branch.
Signed-off-by: John <jonawals@amazon.com>
This commit is contained in:
+3
-1
@@ -6,6 +6,8 @@
|
||||
*
|
||||
*/
|
||||
|
||||
#include <TestImpactFramework/TestImpactUtils.h>
|
||||
|
||||
#include <Artifact/Factory/TestImpactTestTargetMetaMapFactory.h>
|
||||
#include <Artifact/TestImpactArtifactException.h>
|
||||
|
||||
@@ -67,7 +69,7 @@ namespace TestImpact
|
||||
{
|
||||
// Check to see if this test target has the suite we're looking for
|
||||
if (const auto suiteName = suite[Keys[SuiteKey]].GetString();
|
||||
strcmp(GetSuiteTypeName(suiteType).c_str(), suiteName) == 0)
|
||||
strcmp(SuiteTypeAsString(suiteType).c_str(), suiteName) == 0)
|
||||
{
|
||||
testMeta.m_suite = suiteName;
|
||||
testMeta.m_customArgs = suite[Keys[CommandKey]].GetString();
|
||||
|
||||
-2
@@ -156,8 +156,6 @@ namespace TestImpact
|
||||
{
|
||||
coveringTestTargetIt->second.erase(source);
|
||||
}
|
||||
|
||||
|
||||
}
|
||||
|
||||
// 2.
|
||||
|
||||
+1
-1
@@ -9,7 +9,7 @@
|
||||
#pragma once
|
||||
|
||||
#include <TestImpactFramework/TestImpactChangeList.h>
|
||||
#include <TestImpactFramework/TestImpactTestSequence.h>
|
||||
#include <TestImpactFramework/TestImpactPolicy.h>
|
||||
|
||||
#include <Artifact/Static/TestImpactProductionTargetDescriptor.h>
|
||||
#include <Artifact/Static/TestImpactTestTargetDescriptor.h>
|
||||
|
||||
+1
-1
@@ -8,7 +8,7 @@
|
||||
|
||||
#pragma once
|
||||
|
||||
#include <TestImpactFramework/TestImpactTestSequence.h>
|
||||
#include <TestImpactFramework/TestImpactPolicy.h>
|
||||
|
||||
#include <Artifact/Static/TestImpactDependencyGraphData.h>
|
||||
#include <Dependency/TestImpactChangeDependencyList.h>
|
||||
|
||||
+1
-1
@@ -6,7 +6,7 @@
|
||||
*
|
||||
*/
|
||||
|
||||
#include <TestImpactFramework/TestImpactFileUtils.h>
|
||||
#include <TestImpactFramework/TestImpactUtils.h>
|
||||
|
||||
#include <Artifact/Factory/TestImpactTestEnumerationSuiteFactory.h>
|
||||
#include <TestEngine/TestImpactTestEngineException.h>
|
||||
|
||||
+1
-1
@@ -6,7 +6,7 @@
|
||||
*
|
||||
*/
|
||||
|
||||
#include <TestImpactFramework/TestImpactFileUtils.h>
|
||||
#include <TestImpactFramework/TestImpactUtils.h>
|
||||
|
||||
#include <Artifact/Factory/TestImpactModuleCoverageFactory.h>
|
||||
#include <Artifact/Factory/TestImpactTestRunSuiteFactory.h>
|
||||
|
||||
+1
-1
@@ -6,7 +6,7 @@
|
||||
*
|
||||
*/
|
||||
|
||||
#include <TestImpactFramework/TestImpactFileUtils.h>
|
||||
#include <TestImpactFramework/TestImpactUtils.h>
|
||||
|
||||
#include <Artifact/Factory/TestImpactTestRunSuiteFactory.h>
|
||||
#include <TestEngine/TestImpactTestEngineException.h>
|
||||
|
||||
+1
-1
@@ -6,7 +6,7 @@
|
||||
*
|
||||
*/
|
||||
|
||||
#include <TestImpactFramework/TestImpactFileUtils.h>
|
||||
#include <TestImpactFramework/TestImpactUtils.h>
|
||||
|
||||
#include <Target/TestImpactTestTarget.h>
|
||||
#include <TestEngine/TestImpactTestEngineException.h>
|
||||
|
||||
+169
-155
@@ -1,6 +1,7 @@
|
||||
/*
|
||||
* Copyright (c) Contributors to the Open 3D Engine Project. For complete copyright and license terms please see the LICENSE at the root of this distribution.
|
||||
*
|
||||
* Copyright (c) Contributors to the Open 3D Engine Project.
|
||||
* For complete copyright and license terms please see the LICENSE at the root of this distribution.
|
||||
*
|
||||
* SPDX-License-Identifier: Apache-2.0 OR MIT
|
||||
*
|
||||
*/
|
||||
@@ -11,7 +12,6 @@ namespace TestImpact
|
||||
{
|
||||
namespace Client
|
||||
{
|
||||
//! Calculates the final sequence result for a composite of multiple sequences.
|
||||
TestSequenceResult CalculateMultiTestSequenceResult(const AZStd::vector<TestSequenceResult>& results)
|
||||
{
|
||||
// Order of precedence:
|
||||
@@ -19,14 +19,12 @@ namespace TestImpact
|
||||
// 2. TestSequenceResult::Timeout
|
||||
// 3. TestSequenceResult::Success
|
||||
|
||||
if (const auto it = AZStd::find(results.begin(), results.end(), TestSequenceResult::Failure);
|
||||
it != results.end())
|
||||
if (const auto it = AZStd::find(results.begin(), results.end(), TestSequenceResult::Failure); it != results.end())
|
||||
{
|
||||
return TestSequenceResult::Failure;
|
||||
}
|
||||
|
||||
if (const auto it = AZStd::find(results.begin(), results.end(), TestSequenceResult::Timeout);
|
||||
it != results.end())
|
||||
|
||||
if (const auto it = AZStd::find(results.begin(), results.end(), TestSequenceResult::Timeout); it != results.end())
|
||||
{
|
||||
return TestSequenceResult::Timeout;
|
||||
}
|
||||
@@ -38,20 +36,32 @@ namespace TestImpact
|
||||
TestSequenceResult result,
|
||||
AZStd::chrono::high_resolution_clock::time_point startTime,
|
||||
AZStd::chrono::milliseconds duration,
|
||||
AZStd::vector<TestRun>&& passingTests,
|
||||
AZStd::vector<TestRunWithTestFailures>&& failingTests,
|
||||
AZStd::vector<TestRun>&& executionFailureTests,
|
||||
AZStd::vector<TestRun>&& timedOutTests,
|
||||
AZStd::vector<TestRun>&& unexecutedTests)
|
||||
AZStd::vector<PassingTestRun>&& passingTestRuns,
|
||||
AZStd::vector<FailingTestRun>&& failingTestRuns,
|
||||
AZStd::vector<TestRunWithExecutionFailure>&& executionFailureTestRuns,
|
||||
AZStd::vector<TimedOutTestRun>&& timedOutTestRuns,
|
||||
AZStd::vector<UnexecutedTestRun>&& unexecutedTestRuns)
|
||||
: m_startTime(startTime)
|
||||
, m_result(result)
|
||||
, m_duration(duration)
|
||||
, m_passingTests(AZStd::move(passingTests))
|
||||
, m_failingTests(AZStd::move(failingTests))
|
||||
, m_executionFailureTests(AZStd::move(executionFailureTests))
|
||||
, m_timedOutTests(AZStd::move(timedOutTests))
|
||||
, m_unexecutedTests(AZStd::move(unexecutedTests))
|
||||
, m_passingTestRuns(AZStd::move(passingTestRuns))
|
||||
, m_failingTestRuns(AZStd::move(failingTestRuns))
|
||||
, m_executionFailureTestRuns(AZStd::move(executionFailureTestRuns))
|
||||
, m_timedOutTestRuns(AZStd::move(timedOutTestRuns))
|
||||
, m_unexecutedTestRuns(AZStd::move(unexecutedTestRuns))
|
||||
{
|
||||
for (const auto& failingTestRun : m_failingTestRuns)
|
||||
{
|
||||
m_totalNumPassingTests += failingTestRun.GetTotalNumPassingTests();
|
||||
m_totalNumFailingTests += failingTestRun.GetTotalNumFailingTests();
|
||||
m_totalNumDisabledTests += failingTestRun.GetTotalNumDisabledTests();
|
||||
}
|
||||
|
||||
for (const auto& passingTestRun : m_passingTestRuns)
|
||||
{
|
||||
m_totalNumPassingTests += passingTestRun.GetTotalNumPassingTests();
|
||||
m_totalNumDisabledTests += passingTestRun.GetTotalNumDisabledTests();
|
||||
}
|
||||
}
|
||||
|
||||
TestSequenceResult TestRunReport::GetResult() const
|
||||
@@ -74,234 +84,238 @@ namespace TestImpact
|
||||
return m_duration;
|
||||
}
|
||||
|
||||
size_t TestRunReport::GetNumPassingTests() const
|
||||
size_t TestRunReport::GetTotalNumTestRuns() const
|
||||
{
|
||||
return m_passingTests.size();
|
||||
return
|
||||
GetNumPassingTestRuns() +
|
||||
GetNumFailingTestRuns() +
|
||||
GetNumExecutionFailureTestRuns() +
|
||||
GetNumTimedOutTestRuns() +
|
||||
GetNumUnexecutedTestRuns();
|
||||
}
|
||||
|
||||
size_t TestRunReport::GetNumFailingTests() const
|
||||
size_t TestRunReport::GetNumPassingTestRuns() const
|
||||
{
|
||||
return m_failingTests.size();
|
||||
return m_passingTestRuns.size();
|
||||
}
|
||||
|
||||
size_t TestRunReport::GetNumTimedOutTests() const
|
||||
size_t TestRunReport::GetNumFailingTestRuns() const
|
||||
{
|
||||
return m_timedOutTests.size();
|
||||
return m_failingTestRuns.size();
|
||||
}
|
||||
|
||||
size_t TestRunReport::GetNumUnexecutedTests() const
|
||||
size_t TestRunReport::GetNumExecutionFailureTestRuns() const
|
||||
{
|
||||
return m_unexecutedTests.size();
|
||||
return m_executionFailureTestRuns.size();
|
||||
}
|
||||
|
||||
const AZStd::vector<TestRun>& TestRunReport::GetPassingTests() const
|
||||
size_t TestRunReport::TestRunReport::GetNumTimedOutTestRuns() const
|
||||
{
|
||||
return m_passingTests;
|
||||
return m_timedOutTestRuns.size();
|
||||
}
|
||||
|
||||
const AZStd::vector<TestRunWithTestFailures>& TestRunReport::GetFailingTests() const
|
||||
size_t TestRunReport::GetNumUnexecutedTestRuns() const
|
||||
{
|
||||
return m_failingTests;
|
||||
return m_unexecutedTestRuns.size();
|
||||
}
|
||||
|
||||
const AZStd::vector<TestRun>& TestRunReport::GetExecutionFailureTests() const
|
||||
const AZStd::vector<PassingTestRun>& TestRunReport::GetPassingTestRuns() const
|
||||
{
|
||||
return m_executionFailureTests;
|
||||
return m_passingTestRuns;
|
||||
}
|
||||
|
||||
const AZStd::vector<TestRun>& TestRunReport::GetTimedOutTests() const
|
||||
const AZStd::vector<FailingTestRun>& TestRunReport::GetFailingTestRuns() const
|
||||
{
|
||||
return m_timedOutTests;
|
||||
return m_failingTestRuns;
|
||||
}
|
||||
|
||||
const AZStd::vector<TestRun>& TestRunReport::GetUnexecutedTests() const
|
||||
const AZStd::vector<TestRunWithExecutionFailure>& TestRunReport::GetExecutionFailureTestRuns() const
|
||||
{
|
||||
return m_unexecutedTests;
|
||||
return m_executionFailureTestRuns;
|
||||
}
|
||||
|
||||
SequenceReport::SequenceReport(SuiteType suiteType, const TestRunSelection& selectedTests, TestRunReport&& selectedTestRunReport)
|
||||
: m_suite(suiteType)
|
||||
, m_selectedTests(selectedTests)
|
||||
, m_selectedTestRunReport(AZStd::move(selectedTestRunReport))
|
||||
const AZStd::vector<TimedOutTestRun>& TestRunReport::GetTimedOutTestRuns() const
|
||||
{
|
||||
return m_timedOutTestRuns;
|
||||
}
|
||||
|
||||
TestSequenceResult SequenceReport::GetResult() const
|
||||
const AZStd::vector<UnexecutedTestRun>& TestRunReport::GetUnexecutedTestRuns() const
|
||||
{
|
||||
return m_selectedTestRunReport.GetResult();
|
||||
return m_unexecutedTestRuns;
|
||||
}
|
||||
|
||||
AZStd::chrono::high_resolution_clock::time_point SequenceReport::GetStartTime() const
|
||||
size_t TestRunReport::GetTotalNumPassingTests() const
|
||||
{
|
||||
return m_selectedTestRunReport.GetStartTime();
|
||||
return m_totalNumPassingTests;
|
||||
}
|
||||
|
||||
AZStd::chrono::high_resolution_clock::time_point SequenceReport::GetEndTime() const
|
||||
size_t TestRunReport::GetTotalNumFailingTests() const
|
||||
{
|
||||
return GetStartTime() + GetDuration();
|
||||
return m_totalNumFailingTests;
|
||||
}
|
||||
|
||||
AZStd::chrono::milliseconds SequenceReport::GetDuration() const
|
||||
size_t TestRunReport::GetTotalNumDisabledTests() const
|
||||
{
|
||||
return m_selectedTestRunReport.GetDuration();
|
||||
return m_totalNumDisabledTests;
|
||||
}
|
||||
|
||||
TestRunSelection SequenceReport::GetSelectedTests() const
|
||||
{
|
||||
return m_selectedTests;
|
||||
}
|
||||
|
||||
TestRunReport SequenceReport::GetSelectedTestRunReport() const
|
||||
{
|
||||
return m_selectedTestRunReport;
|
||||
}
|
||||
|
||||
size_t SequenceReport::GetTotalNumPassingTests() const
|
||||
{
|
||||
return m_selectedTestRunReport.GetNumPassingTests();
|
||||
}
|
||||
|
||||
size_t SequenceReport::GetTotalNumFailingTests() const
|
||||
{
|
||||
return m_selectedTestRunReport.GetNumFailingTests();
|
||||
}
|
||||
|
||||
size_t SequenceReport::GetTotalNumTimedOutTests() const
|
||||
{
|
||||
return m_selectedTestRunReport.GetNumTimedOutTests();
|
||||
}
|
||||
|
||||
size_t SequenceReport::GetTotalNumUnexecutedTests() const
|
||||
{
|
||||
return m_selectedTestRunReport.GetNumUnexecutedTests();
|
||||
}
|
||||
|
||||
DraftingSequenceReport::DraftingSequenceReport(
|
||||
RegularSequenceReport::RegularSequenceReport(
|
||||
size_t maxConcurrency,
|
||||
const AZStd::optional<AZStd::chrono::milliseconds>& testTargetTimeout,
|
||||
const AZStd::optional<AZStd::chrono::milliseconds>& globalTimeout,
|
||||
const SequencePolicyState& policyState,
|
||||
SuiteType suiteType,
|
||||
const TestRunSelection& selectedTests,
|
||||
const AZStd::vector<AZStd::string>& draftedTests,
|
||||
TestRunReport&& selectedTestRunReport,
|
||||
TestRunReport&& draftedTestRunReport)
|
||||
: SequenceReport(suiteType, selectedTests, AZStd::move(selectedTestRunReport))
|
||||
, m_draftedTests(draftedTests)
|
||||
, m_draftedTestRunReport(AZStd::move(draftedTestRunReport))
|
||||
const TestRunSelection& selectedTestRuns,
|
||||
TestRunReport&& selectedTestRunReport)
|
||||
: SequenceReportBase(
|
||||
SequenceReportType::RegularSequence,
|
||||
maxConcurrency,
|
||||
testTargetTimeout,
|
||||
globalTimeout,
|
||||
policyState,
|
||||
suiteType,
|
||||
selectedTestRuns,
|
||||
AZStd::move(selectedTestRunReport))
|
||||
{
|
||||
}
|
||||
|
||||
TestSequenceResult DraftingSequenceReport::GetResult() const
|
||||
SeedSequenceReport::SeedSequenceReport(
|
||||
size_t maxConcurrency,
|
||||
const AZStd::optional<AZStd::chrono::milliseconds>& testTargetTimeout,
|
||||
const AZStd::optional<AZStd::chrono::milliseconds>& globalTimeout,
|
||||
const SequencePolicyState& policyState,
|
||||
SuiteType suiteType,
|
||||
const TestRunSelection& selectedTestRuns,
|
||||
TestRunReport&& selectedTestRunReport)
|
||||
: SequenceReportBase(
|
||||
SequenceReportType::SeedSequence,
|
||||
maxConcurrency,
|
||||
testTargetTimeout,
|
||||
globalTimeout,
|
||||
policyState,
|
||||
suiteType,
|
||||
selectedTestRuns,
|
||||
AZStd::move(selectedTestRunReport))
|
||||
{
|
||||
return CalculateMultiTestSequenceResult({SequenceReport::GetResult(), m_draftedTestRunReport.GetResult()});
|
||||
}
|
||||
|
||||
AZStd::chrono::milliseconds DraftingSequenceReport::GetDuration() const
|
||||
{
|
||||
return SequenceReport::GetDuration() + m_draftedTestRunReport.GetDuration();
|
||||
}
|
||||
|
||||
size_t DraftingSequenceReport::GetTotalNumPassingTests() const
|
||||
{
|
||||
return SequenceReport::GetTotalNumPassingTests() + m_draftedTestRunReport.GetNumPassingTests();
|
||||
}
|
||||
|
||||
size_t DraftingSequenceReport::GetTotalNumFailingTests() const
|
||||
{
|
||||
return SequenceReport::GetTotalNumFailingTests() + m_draftedTestRunReport.GetNumFailingTests();
|
||||
}
|
||||
|
||||
size_t DraftingSequenceReport::GetTotalNumTimedOutTests() const
|
||||
{
|
||||
return SequenceReport::GetTotalNumTimedOutTests() + m_draftedTestRunReport.GetNumTimedOutTests();
|
||||
}
|
||||
|
||||
size_t DraftingSequenceReport::GetTotalNumUnexecutedTests() const
|
||||
{
|
||||
return SequenceReport::GetTotalNumUnexecutedTests() + m_draftedTestRunReport.GetNumUnexecutedTests();
|
||||
}
|
||||
|
||||
const AZStd::vector<AZStd::string>& DraftingSequenceReport::GetDraftedTests() const
|
||||
{
|
||||
return m_draftedTests;
|
||||
}
|
||||
|
||||
TestRunReport DraftingSequenceReport::GetDraftedTestRunReport() const
|
||||
{
|
||||
return m_draftedTestRunReport;
|
||||
}
|
||||
|
||||
ImpactAnalysisSequenceReport::ImpactAnalysisSequenceReport(
|
||||
size_t maxConcurrency,
|
||||
const AZStd::optional<AZStd::chrono::milliseconds>& testTargetTimeout,
|
||||
const AZStd::optional<AZStd::chrono::milliseconds>& globalTimeout,
|
||||
const ImpactAnalysisSequencePolicyState& policyState,
|
||||
SuiteType suiteType,
|
||||
const TestRunSelection& selectedTests,
|
||||
const AZStd::vector<AZStd::string>& discardedTests,
|
||||
const AZStd::vector<AZStd::string>& draftedTests,
|
||||
const TestRunSelection& selectedTestRuns,
|
||||
const AZStd::vector<AZStd::string>& discardedTestRuns,
|
||||
const AZStd::vector<AZStd::string>& draftedTestRuns,
|
||||
TestRunReport&& selectedTestRunReport,
|
||||
TestRunReport&& draftedTestRunReport)
|
||||
: DraftingSequenceReport(
|
||||
suiteType,
|
||||
selectedTests,
|
||||
draftedTests,
|
||||
AZStd::move(selectedTestRunReport),
|
||||
AZStd::move(draftedTestRunReport))
|
||||
, m_discardedTests(discardedTests)
|
||||
: DraftingSequenceReportBase(
|
||||
SequenceReportType::ImpactAnalysisSequence,
|
||||
maxConcurrency,
|
||||
testTargetTimeout,
|
||||
globalTimeout,
|
||||
policyState,
|
||||
suiteType,
|
||||
selectedTestRuns,
|
||||
draftedTestRuns,
|
||||
AZStd::move(selectedTestRunReport),
|
||||
AZStd::move(draftedTestRunReport))
|
||||
, m_discardedTestRuns(discardedTestRuns)
|
||||
{
|
||||
}
|
||||
|
||||
const AZStd::vector<AZStd::string>& ImpactAnalysisSequenceReport::GetDiscardedTests() const
|
||||
const AZStd::vector<AZStd::string>& ImpactAnalysisSequenceReport::GetDiscardedTestRuns() const
|
||||
{
|
||||
return m_discardedTests;
|
||||
return m_discardedTestRuns;
|
||||
}
|
||||
|
||||
SafeImpactAnalysisSequenceReport::SafeImpactAnalysisSequenceReport(
|
||||
size_t maxConcurrency,
|
||||
const AZStd::optional<AZStd::chrono::milliseconds>& testTargetTimeout,
|
||||
const AZStd::optional<AZStd::chrono::milliseconds>& globalTimeout,
|
||||
const SafeImpactAnalysisSequencePolicyState& policyState,
|
||||
SuiteType suiteType,
|
||||
const TestRunSelection& selectedTests,
|
||||
const TestRunSelection& discardedTests,
|
||||
const AZStd::vector<AZStd::string>& draftedTests,
|
||||
const TestRunSelection& selectedTestRuns,
|
||||
const TestRunSelection& discardedTestRuns,
|
||||
const AZStd::vector<AZStd::string>& draftedTestRuns,
|
||||
TestRunReport&& selectedTestRunReport,
|
||||
TestRunReport&& discardedTestRunReport,
|
||||
TestRunReport&& draftedTestRunReport)
|
||||
: DraftingSequenceReport(
|
||||
suiteType,
|
||||
selectedTests,
|
||||
draftedTests,
|
||||
AZStd::move(selectedTestRunReport),
|
||||
AZStd::move(draftedTestRunReport))
|
||||
, m_discardedTests(discardedTests)
|
||||
: DraftingSequenceReportBase(
|
||||
SequenceReportType::SafeImpactAnalysisSequence,
|
||||
maxConcurrency,
|
||||
testTargetTimeout,
|
||||
globalTimeout,
|
||||
policyState,
|
||||
suiteType,
|
||||
selectedTestRuns,
|
||||
draftedTestRuns,
|
||||
AZStd::move(selectedTestRunReport),
|
||||
AZStd::move(draftedTestRunReport))
|
||||
, m_discardedTestRuns(discardedTestRuns)
|
||||
, m_discardedTestRunReport(AZStd::move(discardedTestRunReport))
|
||||
{
|
||||
}
|
||||
|
||||
TestSequenceResult SafeImpactAnalysisSequenceReport::GetResult() const
|
||||
{
|
||||
return CalculateMultiTestSequenceResult({ DraftingSequenceReport::GetResult(), m_discardedTestRunReport.GetResult() });
|
||||
return CalculateMultiTestSequenceResult({ DraftingSequenceReportBase::GetResult(), m_discardedTestRunReport.GetResult() });
|
||||
}
|
||||
|
||||
AZStd::chrono::milliseconds SafeImpactAnalysisSequenceReport::GetDuration() const
|
||||
{
|
||||
return DraftingSequenceReport::GetDuration() + m_discardedTestRunReport.GetDuration();
|
||||
return DraftingSequenceReportBase::GetDuration() + m_discardedTestRunReport.GetDuration();
|
||||
}
|
||||
|
||||
size_t SafeImpactAnalysisSequenceReport::GetTotalNumTestRuns() const
|
||||
{
|
||||
return DraftingSequenceReportBase::GetTotalNumTestRuns() + m_discardedTestRunReport.GetTotalNumTestRuns();
|
||||
}
|
||||
|
||||
size_t SafeImpactAnalysisSequenceReport::GetTotalNumPassingTests() const
|
||||
{
|
||||
return DraftingSequenceReport::GetTotalNumPassingTests() + m_discardedTestRunReport.GetNumPassingTests();
|
||||
return DraftingSequenceReportBase::GetTotalNumPassingTests() + m_discardedTestRunReport.GetTotalNumPassingTests();
|
||||
}
|
||||
|
||||
size_t SafeImpactAnalysisSequenceReport::GetTotalNumFailingTests() const
|
||||
{
|
||||
return DraftingSequenceReport::GetTotalNumFailingTests() + m_discardedTestRunReport.GetNumFailingTests();
|
||||
return DraftingSequenceReportBase::GetTotalNumFailingTests() + m_discardedTestRunReport.GetTotalNumFailingTests();
|
||||
}
|
||||
|
||||
size_t SafeImpactAnalysisSequenceReport::GetTotalNumTimedOutTests() const
|
||||
size_t SafeImpactAnalysisSequenceReport::GetTotalNumDisabledTests() const
|
||||
{
|
||||
return DraftingSequenceReport::GetTotalNumTimedOutTests() + m_discardedTestRunReport.GetNumTimedOutTests();
|
||||
return DraftingSequenceReportBase::GetTotalNumDisabledTests() + m_discardedTestRunReport.GetTotalNumDisabledTests();
|
||||
}
|
||||
|
||||
size_t SafeImpactAnalysisSequenceReport::GetTotalNumUnexecutedTests() const
|
||||
size_t SafeImpactAnalysisSequenceReport::GetTotalNumPassingTestRuns() const
|
||||
{
|
||||
return DraftingSequenceReport::GetTotalNumUnexecutedTests() + m_discardedTestRunReport.GetNumUnexecutedTests();
|
||||
return DraftingSequenceReportBase::GetTotalNumPassingTestRuns() + m_discardedTestRunReport.GetNumPassingTestRuns();
|
||||
}
|
||||
|
||||
size_t SafeImpactAnalysisSequenceReport::GetTotalNumFailingTestRuns() const
|
||||
{
|
||||
return DraftingSequenceReportBase::GetTotalNumFailingTestRuns() + m_discardedTestRunReport.GetNumFailingTestRuns();
|
||||
}
|
||||
|
||||
size_t SafeImpactAnalysisSequenceReport::GetTotalNumExecutionFailureTestRuns() const
|
||||
{
|
||||
return DraftingSequenceReportBase::GetTotalNumExecutionFailureTestRuns() + m_discardedTestRunReport.GetNumExecutionFailureTestRuns();
|
||||
}
|
||||
|
||||
size_t SafeImpactAnalysisSequenceReport::GetTotalNumTimedOutTestRuns() const
|
||||
{
|
||||
return DraftingSequenceReportBase::GetTotalNumTimedOutTestRuns() + m_discardedTestRunReport.GetNumTimedOutTestRuns();
|
||||
}
|
||||
|
||||
size_t SafeImpactAnalysisSequenceReport::GetTotalNumUnexecutedTestRuns() const
|
||||
{
|
||||
return DraftingSequenceReportBase::GetTotalNumUnexecutedTestRuns() + m_discardedTestRunReport.GetNumUnexecutedTestRuns();
|
||||
}
|
||||
|
||||
const TestRunSelection SafeImpactAnalysisSequenceReport::GetDiscardedTests() const
|
||||
const TestRunSelection SafeImpactAnalysisSequenceReport::GetDiscardedTestRuns() const
|
||||
{
|
||||
return m_discardedTests;
|
||||
return m_discardedTestRuns;
|
||||
}
|
||||
|
||||
TestRunReport SafeImpactAnalysisSequenceReport::GetDiscardedTestRunReport() const
|
||||
|
||||
+606
@@ -0,0 +1,606 @@
|
||||
/*
|
||||
* Copyright (c) Contributors to the Open 3D Engine Project.
|
||||
* For complete copyright and license terms please see the LICENSE at the root of this distribution.
|
||||
*
|
||||
* SPDX-License-Identifier: Apache-2.0 OR MIT
|
||||
*
|
||||
*/
|
||||
|
||||
#include <TestImpactFramework/TestImpactClientSequenceReportSerializer.h>
|
||||
#include <TestImpactFramework/TestImpactSequenceReportException.h>
|
||||
#include <TestImpactFramework/TestImpactUtils.h>
|
||||
|
||||
#include <AzCore/JSON/document.h>
|
||||
#include <AzCore/JSON/prettywriter.h>
|
||||
#include <AzCore/JSON/rapidjson.h>
|
||||
#include <AzCore/JSON/stringbuffer.h>
|
||||
|
||||
namespace TestImpact
|
||||
{
|
||||
namespace
|
||||
{
|
||||
namespace SequenceReportFields
|
||||
{
|
||||
// Keys for pertinent JSON node and attribute names
|
||||
constexpr const char* Keys[] =
|
||||
{
|
||||
"name",
|
||||
"command_args",
|
||||
"start_time",
|
||||
"end_time",
|
||||
"duration",
|
||||
"result",
|
||||
"num_passing_tests",
|
||||
"num_failing_tests",
|
||||
"num_disabled_tests",
|
||||
"tests",
|
||||
"num_passing_test_runs",
|
||||
"num_failing_test_runs",
|
||||
"num_execution_failure_test_runs",
|
||||
"num_timed_out_test_runs",
|
||||
"num_unexecuted_test_runs",
|
||||
"passing_test_runs",
|
||||
"failing_test_runs",
|
||||
"execution_failure_test_runs",
|
||||
"timed_out_test_runs",
|
||||
"unexecuted_test_runs",
|
||||
"total_num_passing_tests",
|
||||
"total_num_failing_tests",
|
||||
"total_num_disabled_tests",
|
||||
"total_num_test_runs",
|
||||
"num_included_test_runs",
|
||||
"num_excluded_test_runs",
|
||||
"included_test_runs",
|
||||
"excluded_test_runs",
|
||||
"execution_failure",
|
||||
"coverage_failure",
|
||||
"test_failure",
|
||||
"integrity_failure",
|
||||
"test_sharding",
|
||||
"target_output_capture",
|
||||
"test_prioritization",
|
||||
"dynamic_dependency_map",
|
||||
"type",
|
||||
"test_target_timeout",
|
||||
"global_timeout",
|
||||
"max_concurrency",
|
||||
"policy",
|
||||
"suite",
|
||||
"selected_test_runs",
|
||||
"selected_test_run_report",
|
||||
"total_num_passing_test_runs",
|
||||
"total_num_failing_test_runs",
|
||||
"total_num_execution_failure_test_runs",
|
||||
"total_num_timed_out_test_runs",
|
||||
"total_num_unexecuted_test_runs",
|
||||
"drafted_test_runs",
|
||||
"drafted_test_run_report",
|
||||
"discarded_test_runs",
|
||||
"discarded_test_run_report"
|
||||
};
|
||||
|
||||
enum
|
||||
{
|
||||
Name,
|
||||
CommandArgs,
|
||||
StartTime,
|
||||
EndTime,
|
||||
Duration,
|
||||
Result,
|
||||
NumPassingTests,
|
||||
NumFailingTests,
|
||||
NumDisabledTests,
|
||||
Tests,
|
||||
NumPassingTestRuns,
|
||||
NumFailingTestRuns,
|
||||
NumExecutionFailureTestRuns,
|
||||
NumTimedOutTestRuns,
|
||||
NumUnexecutedTestRuns,
|
||||
PassingTestRuns,
|
||||
FailingTestRuns,
|
||||
ExecutionFailureTestRuns,
|
||||
TimedOutTestRuns,
|
||||
UnexecutedTestRuns,
|
||||
TotalNumPassingTests,
|
||||
TotalNumFailingTests,
|
||||
TotalNumDisabledTests,
|
||||
TotalNumTestRuns,
|
||||
NumIncludedTestRuns,
|
||||
NumExcludedTestRuns,
|
||||
IncludedTestRuns,
|
||||
ExcludedTestRuns,
|
||||
ExecutionFailure,
|
||||
CoverageFailure,
|
||||
TestFailure,
|
||||
IntegrityFailure,
|
||||
TestSharding,
|
||||
TargetOutputCapture,
|
||||
TestPrioritization,
|
||||
DynamicDependencyMap,
|
||||
Type,
|
||||
TestTargetTimeout,
|
||||
GlobalTimeout,
|
||||
MaxConcurrency,
|
||||
Policy,
|
||||
Suite,
|
||||
SelectedTestRuns,
|
||||
SelectedTestRunReport,
|
||||
TotalNumPassingTestRuns,
|
||||
TotalNumFailingTestRuns,
|
||||
TotalNumExecutionFailureTestRuns,
|
||||
TotalNumTimedOutTestRuns,
|
||||
TotalNumUnexecutedTestRuns,
|
||||
DraftedTestRuns,
|
||||
DraftedTestRunReport,
|
||||
DiscardedTestRuns,
|
||||
DiscardedTestRunReport
|
||||
};
|
||||
} // namespace SequenceReportFields
|
||||
|
||||
AZ::u64 TimePointInMsAsInt64(AZStd::chrono::high_resolution_clock::time_point timePoint)
|
||||
{
|
||||
return AZStd::chrono::duration_cast<AZStd::chrono::milliseconds>(timePoint.time_since_epoch()).count();
|
||||
}
|
||||
|
||||
void SerializeTestRunMembers(const Client::TestRunBase& testRun, rapidjson::PrettyWriter<rapidjson::StringBuffer>& writer)
|
||||
{
|
||||
// Name
|
||||
writer.Key(SequenceReportFields::Keys[SequenceReportFields::Name]);
|
||||
writer.String(testRun.GetTargetName().c_str());
|
||||
|
||||
// Command string
|
||||
writer.Key(SequenceReportFields::Keys[SequenceReportFields::CommandArgs]);
|
||||
writer.String(testRun.GetCommandString().c_str());
|
||||
|
||||
// Start time
|
||||
writer.Key(SequenceReportFields::Keys[SequenceReportFields::StartTime]);
|
||||
writer.Int64(TimePointInMsAsInt64(testRun.GetStartTime()));
|
||||
|
||||
// End time
|
||||
writer.Key(SequenceReportFields::Keys[SequenceReportFields::EndTime]);
|
||||
writer.Int64(TimePointInMsAsInt64(testRun.GetEndTime()));
|
||||
|
||||
// Duration
|
||||
writer.Key(SequenceReportFields::Keys[SequenceReportFields::Duration]);
|
||||
writer.Uint64(testRun.GetDuration().count());
|
||||
|
||||
// Result
|
||||
writer.Key(SequenceReportFields::Keys[SequenceReportFields::Result]);
|
||||
writer.String(TestRunResultAsString(testRun.GetResult()).c_str());
|
||||
}
|
||||
|
||||
void SerializeTestRun(const Client::TestRunBase& testRun, rapidjson::PrettyWriter<rapidjson::StringBuffer>& writer)
|
||||
{
|
||||
writer.StartObject();
|
||||
{
|
||||
SerializeTestRunMembers(testRun, writer);
|
||||
}
|
||||
writer.EndObject();
|
||||
}
|
||||
|
||||
void SerializeCompletedTestRun(const Client::CompletedTestRun& testRun, rapidjson::PrettyWriter<rapidjson::StringBuffer>& writer)
|
||||
{
|
||||
writer.StartObject();
|
||||
{
|
||||
SerializeTestRunMembers(testRun, writer);
|
||||
|
||||
// Number of passing test cases
|
||||
writer.Key(SequenceReportFields::Keys[SequenceReportFields::NumPassingTests]);
|
||||
writer.Uint64(testRun.GetTotalNumPassingTests());
|
||||
|
||||
// Number of failing test cases
|
||||
writer.Key(SequenceReportFields::Keys[SequenceReportFields::NumFailingTests]);
|
||||
writer.Uint64(testRun.GetTotalNumFailingTests());
|
||||
|
||||
// Number of disabled test cases
|
||||
writer.Key(SequenceReportFields::Keys[SequenceReportFields::NumDisabledTests]);
|
||||
writer.Uint64(testRun.GetTotalNumDisabledTests());
|
||||
|
||||
// Tests
|
||||
writer.Key(SequenceReportFields::Keys[SequenceReportFields::Tests]);
|
||||
writer.StartArray();
|
||||
|
||||
for (const auto& test : testRun.GetTests())
|
||||
{
|
||||
// Test
|
||||
writer.StartObject();
|
||||
|
||||
// Name
|
||||
writer.Key(SequenceReportFields::Keys[SequenceReportFields::Name]);
|
||||
writer.String(test.GetName().c_str());
|
||||
|
||||
// Result
|
||||
writer.Key(SequenceReportFields::Keys[SequenceReportFields::Result]);
|
||||
writer.String(ClientTestResultAsString(test.GetResult()).c_str());
|
||||
|
||||
writer.EndObject(); // Test
|
||||
}
|
||||
|
||||
writer.EndArray(); // Tests
|
||||
}
|
||||
writer.EndObject();
|
||||
}
|
||||
|
||||
void SerializeTestRunReport(
|
||||
const Client::TestRunReport& testRunReport, rapidjson::PrettyWriter<rapidjson::StringBuffer>& writer)
|
||||
{
|
||||
writer.StartObject();
|
||||
{
|
||||
// Result
|
||||
writer.Key(SequenceReportFields::Keys[SequenceReportFields::Result]);
|
||||
writer.String(TestSequenceResultAsString(testRunReport.GetResult()).c_str());
|
||||
|
||||
// Start time
|
||||
writer.Key(SequenceReportFields::Keys[SequenceReportFields::StartTime]);
|
||||
writer.Int64(TimePointInMsAsInt64(testRunReport.GetStartTime()));
|
||||
|
||||
// End time
|
||||
writer.Key(SequenceReportFields::Keys[SequenceReportFields::EndTime]);
|
||||
writer.Int64(TimePointInMsAsInt64(testRunReport.GetEndTime()));
|
||||
|
||||
// Duration
|
||||
writer.Key(SequenceReportFields::Keys[SequenceReportFields::Duration]);
|
||||
writer.Uint64(testRunReport.GetDuration().count());
|
||||
|
||||
// Number of passing test runs
|
||||
writer.Key(SequenceReportFields::Keys[SequenceReportFields::NumPassingTestRuns]);
|
||||
writer.Uint64(testRunReport.GetNumPassingTestRuns());
|
||||
|
||||
// Number of failing test runs
|
||||
writer.Key(SequenceReportFields::Keys[SequenceReportFields::NumFailingTestRuns]);
|
||||
writer.Uint64(testRunReport.GetNumFailingTestRuns());
|
||||
|
||||
// Number of test runs that failed to execute
|
||||
writer.Key(SequenceReportFields::Keys[SequenceReportFields::NumExecutionFailureTestRuns]);
|
||||
writer.Uint64(testRunReport.GetNumExecutionFailureTestRuns());
|
||||
|
||||
// Number of timed out test runs
|
||||
writer.Key(SequenceReportFields::Keys[SequenceReportFields::NumTimedOutTestRuns]);
|
||||
writer.Uint64(testRunReport.GetNumTimedOutTestRuns());
|
||||
|
||||
// Number of unexecuted test runs
|
||||
writer.Key(SequenceReportFields::Keys[SequenceReportFields::NumUnexecutedTestRuns]);
|
||||
writer.Uint64(testRunReport.GetNumUnexecutedTestRuns());
|
||||
|
||||
// Passing test runs
|
||||
writer.Key(SequenceReportFields::Keys[SequenceReportFields::PassingTestRuns]);
|
||||
writer.StartArray();
|
||||
for (const auto& testRun : testRunReport.GetPassingTestRuns())
|
||||
{
|
||||
SerializeCompletedTestRun(testRun, writer);
|
||||
}
|
||||
writer.EndArray(); // Passing test runs
|
||||
|
||||
// Failing test runs
|
||||
writer.Key(SequenceReportFields::Keys[SequenceReportFields::FailingTestRuns]);
|
||||
writer.StartArray();
|
||||
for (const auto& testRun : testRunReport.GetFailingTestRuns())
|
||||
{
|
||||
SerializeCompletedTestRun(testRun, writer);
|
||||
}
|
||||
writer.EndArray(); // Failing test runs
|
||||
|
||||
// Execution failures
|
||||
writer.Key(SequenceReportFields::Keys[SequenceReportFields::ExecutionFailureTestRuns]);
|
||||
writer.StartArray();
|
||||
for (const auto& testRun : testRunReport.GetExecutionFailureTestRuns())
|
||||
{
|
||||
SerializeTestRun(testRun, writer);
|
||||
}
|
||||
writer.EndArray(); // Execution failures
|
||||
|
||||
// Timed out test runs
|
||||
writer.Key(SequenceReportFields::Keys[SequenceReportFields::TimedOutTestRuns]);
|
||||
writer.StartArray();
|
||||
for (const auto& testRun : testRunReport.GetTimedOutTestRuns())
|
||||
{
|
||||
SerializeTestRun(testRun, writer);
|
||||
}
|
||||
writer.EndArray(); // Timed out test runs
|
||||
|
||||
// Unexecuted test runs
|
||||
writer.Key(SequenceReportFields::Keys[SequenceReportFields::UnexecutedTestRuns]);
|
||||
writer.StartArray();
|
||||
for (const auto& testRun : testRunReport.GetUnexecutedTestRuns())
|
||||
{
|
||||
SerializeTestRun(testRun, writer);
|
||||
}
|
||||
writer.EndArray(); // Unexecuted test runs
|
||||
|
||||
// Number of passing tests
|
||||
writer.Key(SequenceReportFields::Keys[SequenceReportFields::TotalNumPassingTests]);
|
||||
writer.Uint64(testRunReport.GetTotalNumPassingTests());
|
||||
|
||||
// Number of failing tests
|
||||
writer.Key(SequenceReportFields::Keys[SequenceReportFields::TotalNumFailingTests]);
|
||||
writer.Uint64(testRunReport.GetTotalNumFailingTests());
|
||||
|
||||
// Number of disabled tests
|
||||
writer.Key(SequenceReportFields::Keys[SequenceReportFields::TotalNumDisabledTests]);
|
||||
writer.Uint64(testRunReport.GetTotalNumDisabledTests());
|
||||
}
|
||||
writer.EndObject();
|
||||
}
|
||||
|
||||
void SerializeTestSelection(
|
||||
const Client::TestRunSelection& testSelection, rapidjson::PrettyWriter<rapidjson::StringBuffer>& writer)
|
||||
{
|
||||
writer.StartObject();
|
||||
{
|
||||
// Total number of test runs
|
||||
writer.Key(SequenceReportFields::Keys[SequenceReportFields::TotalNumTestRuns]);
|
||||
writer.Uint64(testSelection.GetTotalNumTests());
|
||||
|
||||
// Number of included test runs
|
||||
writer.Key(SequenceReportFields::Keys[SequenceReportFields::NumIncludedTestRuns]);
|
||||
writer.Uint64(testSelection.GetNumIncludedTestRuns());
|
||||
|
||||
// Number of excluded test runs
|
||||
writer.Key(SequenceReportFields::Keys[SequenceReportFields::NumExcludedTestRuns]);
|
||||
writer.Uint64(testSelection.GetNumExcludedTestRuns());
|
||||
|
||||
// Included test runs
|
||||
writer.Key(SequenceReportFields::Keys[SequenceReportFields::IncludedTestRuns]);
|
||||
writer.StartArray();
|
||||
for (const auto& testRun : testSelection.GetIncludededTestRuns())
|
||||
{
|
||||
writer.String(testRun.c_str());
|
||||
}
|
||||
writer.EndArray(); // Included test runs
|
||||
|
||||
// Excluded test runs
|
||||
writer.Key(SequenceReportFields::Keys[SequenceReportFields::ExcludedTestRuns]);
|
||||
writer.StartArray();
|
||||
for (const auto& testRun : testSelection.GetExcludedTestRuns())
|
||||
{
|
||||
writer.String(testRun.c_str());
|
||||
}
|
||||
writer.EndArray(); // Excluded test runs
|
||||
}
|
||||
writer.EndObject();
|
||||
}
|
||||
|
||||
void SerializePolicyStateBaseMembers(const PolicyStateBase& policyState, rapidjson::PrettyWriter<rapidjson::StringBuffer>& writer)
|
||||
{
|
||||
// Execution failure
|
||||
writer.Key(SequenceReportFields::Keys[SequenceReportFields::ExecutionFailure]);
|
||||
writer.String(ExecutionFailurePolicyAsString(policyState.m_executionFailurePolicy).c_str());
|
||||
|
||||
// Failed test coverage
|
||||
writer.Key(SequenceReportFields::Keys[SequenceReportFields::CoverageFailure]);
|
||||
writer.String(FailedTestCoveragePolicyAsString(policyState.m_failedTestCoveragePolicy).c_str());
|
||||
|
||||
// Test failure
|
||||
writer.Key(SequenceReportFields::Keys[SequenceReportFields::TestFailure]);
|
||||
writer.String(TestFailurePolicyAsString(policyState.m_testFailurePolicy).c_str());
|
||||
|
||||
// Integrity failure
|
||||
writer.Key(SequenceReportFields::Keys[SequenceReportFields::IntegrityFailure]);
|
||||
writer.String(IntegrityFailurePolicyAsString(policyState.m_integrityFailurePolicy).c_str());
|
||||
|
||||
// Test sharding
|
||||
writer.Key(SequenceReportFields::Keys[SequenceReportFields::TestSharding]);
|
||||
writer.String(TestShardingPolicyAsString(policyState.m_testShardingPolicy).c_str());
|
||||
|
||||
// Target output capture
|
||||
writer.Key(SequenceReportFields::Keys[SequenceReportFields::TargetOutputCapture]);
|
||||
writer.String(TargetOutputCapturePolicyAsString(policyState.m_targetOutputCapture).c_str());
|
||||
}
|
||||
|
||||
void SerializePolicyStateMembers(
|
||||
const SequencePolicyState& policyState, rapidjson::PrettyWriter<rapidjson::StringBuffer>& writer)
|
||||
{
|
||||
SerializePolicyStateBaseMembers(policyState.m_basePolicies, writer);
|
||||
}
|
||||
|
||||
void SerializePolicyStateMembers(
|
||||
const SafeImpactAnalysisSequencePolicyState& policyState, rapidjson::PrettyWriter<rapidjson::StringBuffer>& writer)
|
||||
{
|
||||
SerializePolicyStateBaseMembers(policyState.m_basePolicies, writer);
|
||||
|
||||
// Test prioritization
|
||||
writer.Key(SequenceReportFields::Keys[SequenceReportFields::TestPrioritization]);
|
||||
writer.String(TestPrioritizationPolicyAsString(policyState.m_testPrioritizationPolicy).c_str());
|
||||
}
|
||||
|
||||
void SerializePolicyStateMembers(
|
||||
const ImpactAnalysisSequencePolicyState& policyState, rapidjson::PrettyWriter<rapidjson::StringBuffer>& writer)
|
||||
{
|
||||
SerializePolicyStateBaseMembers(policyState.m_basePolicies, writer);
|
||||
|
||||
// Test prioritization
|
||||
writer.Key(SequenceReportFields::Keys[SequenceReportFields::TestPrioritization]);
|
||||
writer.String(TestPrioritizationPolicyAsString(policyState.m_testPrioritizationPolicy).c_str());
|
||||
|
||||
// Dynamic dependency map
|
||||
writer.Key(SequenceReportFields::Keys[SequenceReportFields::DynamicDependencyMap]);
|
||||
writer.String(DynamicDependencyMapPolicyAsString(policyState.m_dynamicDependencyMap).c_str());
|
||||
}
|
||||
|
||||
template<typename PolicyStateType>
|
||||
void SerializeSequenceReportBaseMembers(
|
||||
const Client::SequenceReportBase<PolicyStateType>& sequenceReport, rapidjson::PrettyWriter<rapidjson::StringBuffer>& writer)
|
||||
{
|
||||
// Type
|
||||
writer.Key(SequenceReportFields::Keys[SequenceReportFields::Type]);
|
||||
writer.String(SequenceReportTypeAsString(sequenceReport.GetType()).c_str());
|
||||
|
||||
// Test target timeout
|
||||
writer.Key(SequenceReportFields::Keys[SequenceReportFields::TestTargetTimeout]);
|
||||
writer.Uint64(sequenceReport.GetTestTargetTimeout().value_or(AZStd::chrono::milliseconds{0}).count());
|
||||
|
||||
// Global timeout
|
||||
writer.Key(SequenceReportFields::Keys[SequenceReportFields::GlobalTimeout]);
|
||||
writer.Uint64(sequenceReport.GetGlobalTimeout().value_or(AZStd::chrono::milliseconds{ 0 }).count());
|
||||
|
||||
// Maximum concurrency
|
||||
writer.Key(SequenceReportFields::Keys[SequenceReportFields::MaxConcurrency]);
|
||||
writer.Uint64(sequenceReport.GetMaxConcurrency());
|
||||
|
||||
// Policies
|
||||
writer.Key(SequenceReportFields::Keys[SequenceReportFields::Policy]);
|
||||
writer.StartObject();
|
||||
{
|
||||
SerializePolicyStateMembers(sequenceReport.GetPolicyState(), writer);
|
||||
}
|
||||
writer.EndObject(); // Policies
|
||||
|
||||
// Suite
|
||||
writer.Key(SequenceReportFields::Keys[SequenceReportFields::Suite]);
|
||||
writer.String(SuiteTypeAsString(sequenceReport.GetSuite()).c_str());
|
||||
|
||||
// Selected test runs
|
||||
writer.Key(SequenceReportFields::Keys[SequenceReportFields::SelectedTestRuns]);
|
||||
SerializeTestSelection(sequenceReport.GetSelectedTestRuns(), writer);
|
||||
|
||||
// Selected test run report
|
||||
writer.Key(SequenceReportFields::Keys[SequenceReportFields::SelectedTestRunReport]);
|
||||
SerializeTestRunReport(sequenceReport.GetSelectedTestRunReport(), writer);
|
||||
|
||||
// Start time
|
||||
writer.Key(SequenceReportFields::Keys[SequenceReportFields::StartTime]);
|
||||
writer.Int64(TimePointInMsAsInt64(sequenceReport.GetStartTime()));
|
||||
|
||||
// End time
|
||||
writer.Key(SequenceReportFields::Keys[SequenceReportFields::EndTime]);
|
||||
writer.Int64(TimePointInMsAsInt64(sequenceReport.GetEndTime()));
|
||||
|
||||
// Duration
|
||||
writer.Key(SequenceReportFields::Keys[SequenceReportFields::Duration]);
|
||||
writer.Uint64(sequenceReport.GetDuration().count());
|
||||
|
||||
// Result
|
||||
writer.Key(SequenceReportFields::Keys[SequenceReportFields::Result]);
|
||||
writer.String(TestSequenceResultAsString(sequenceReport.GetResult()).c_str());
|
||||
|
||||
// Total number of test runs
|
||||
writer.Key(SequenceReportFields::Keys[SequenceReportFields::TotalNumTestRuns]);
|
||||
writer.Uint64(sequenceReport.GetTotalNumTestRuns());
|
||||
|
||||
// Total number of passing test runs
|
||||
writer.Key(SequenceReportFields::Keys[SequenceReportFields::TotalNumPassingTestRuns]);
|
||||
writer.Uint64(sequenceReport.GetTotalNumPassingTestRuns());
|
||||
|
||||
// Total number of failing test runs
|
||||
writer.Key(SequenceReportFields::Keys[SequenceReportFields::TotalNumFailingTestRuns]);
|
||||
writer.Uint64(sequenceReport.GetTotalNumFailingTestRuns());
|
||||
|
||||
// Total number of test runs that failed to execute
|
||||
writer.Key(SequenceReportFields::Keys[SequenceReportFields::TotalNumExecutionFailureTestRuns]);
|
||||
writer.Uint64(sequenceReport.GetTotalNumExecutionFailureTestRuns());
|
||||
|
||||
// Total number of timed out test runs
|
||||
writer.Key(SequenceReportFields::Keys[SequenceReportFields::TotalNumTimedOutTestRuns]);
|
||||
writer.Uint64(sequenceReport.GetTotalNumTimedOutTestRuns());
|
||||
|
||||
// Total number of unexecuted test runs
|
||||
writer.Key(SequenceReportFields::Keys[SequenceReportFields::TotalNumUnexecutedTestRuns]);
|
||||
writer.Uint64(sequenceReport.GetTotalNumUnexecutedTestRuns());
|
||||
|
||||
// Total number of passing tests
|
||||
writer.Key(SequenceReportFields::Keys[SequenceReportFields::TotalNumPassingTests]);
|
||||
writer.Uint64(sequenceReport.GetTotalNumPassingTests());
|
||||
|
||||
// Total number of failing tests
|
||||
writer.Key(SequenceReportFields::Keys[SequenceReportFields::TotalNumFailingTests]);
|
||||
writer.Uint64(sequenceReport.GetTotalNumFailingTests());
|
||||
|
||||
// Total number of disabled tests
|
||||
writer.Key(SequenceReportFields::Keys[SequenceReportFields::TotalNumDisabledTests]);
|
||||
writer.Uint64(sequenceReport.GetTotalNumDisabledTests());
|
||||
}
|
||||
|
||||
template<typename PolicyStateType>
|
||||
void SerializeDraftingSequenceReportMembers(
|
||||
const Client::DraftingSequenceReportBase<PolicyStateType>& sequenceReport, rapidjson::PrettyWriter<rapidjson::StringBuffer>& writer)
|
||||
{
|
||||
SerializeSequenceReportBaseMembers(sequenceReport, writer);
|
||||
|
||||
// Drafted test runs
|
||||
writer.Key(SequenceReportFields::Keys[SequenceReportFields::DraftedTestRuns]);
|
||||
writer.StartArray();
|
||||
for (const auto& testRun : sequenceReport.GetDraftedTestRuns())
|
||||
{
|
||||
writer.String(testRun.c_str());
|
||||
}
|
||||
writer.EndArray(); // Drafted test runs
|
||||
|
||||
// Drafted test run report
|
||||
writer.Key(SequenceReportFields::Keys[SequenceReportFields::DraftedTestRunReport]);
|
||||
SerializeTestRunReport(sequenceReport.GetDraftedTestRunReport(), writer);
|
||||
}
|
||||
} // namespace
|
||||
|
||||
AZStd::string SerializeSequenceReport(const Client::RegularSequenceReport& sequenceReport)
|
||||
{
|
||||
rapidjson::StringBuffer stringBuffer;
|
||||
rapidjson::PrettyWriter<rapidjson::StringBuffer> writer(stringBuffer);
|
||||
|
||||
writer.StartObject();
|
||||
{
|
||||
SerializeSequenceReportBaseMembers(sequenceReport, writer);
|
||||
}
|
||||
writer.EndObject();
|
||||
|
||||
return stringBuffer.GetString();
|
||||
}
|
||||
|
||||
AZStd::string SerializeSequenceReport(const Client::SeedSequenceReport& sequenceReport)
|
||||
{
|
||||
rapidjson::StringBuffer stringBuffer;
|
||||
rapidjson::PrettyWriter<rapidjson::StringBuffer> writer(stringBuffer);
|
||||
|
||||
writer.StartObject();
|
||||
{
|
||||
SerializeSequenceReportBaseMembers(sequenceReport, writer);
|
||||
}
|
||||
writer.EndObject();
|
||||
|
||||
return stringBuffer.GetString();
|
||||
}
|
||||
|
||||
AZStd::string SerializeSequenceReport(const Client::ImpactAnalysisSequenceReport& sequenceReport)
|
||||
{
|
||||
rapidjson::StringBuffer stringBuffer;
|
||||
rapidjson::PrettyWriter<rapidjson::StringBuffer> writer(stringBuffer);
|
||||
|
||||
writer.StartObject();
|
||||
{
|
||||
SerializeDraftingSequenceReportMembers(sequenceReport, writer);
|
||||
|
||||
// Discarded test runs
|
||||
writer.Key(SequenceReportFields::Keys[SequenceReportFields::DiscardedTestRuns]);
|
||||
writer.StartArray();
|
||||
for (const auto& testRun : sequenceReport.GetDiscardedTestRuns())
|
||||
{
|
||||
writer.String(testRun.c_str());
|
||||
}
|
||||
writer.EndArray(); // Discarded test runs
|
||||
}
|
||||
writer.EndObject();
|
||||
|
||||
return stringBuffer.GetString();
|
||||
}
|
||||
|
||||
AZStd::string SerializeSequenceReport(const Client::SafeImpactAnalysisSequenceReport& sequenceReport)
|
||||
{
|
||||
rapidjson::StringBuffer stringBuffer;
|
||||
rapidjson::PrettyWriter<rapidjson::StringBuffer> writer(stringBuffer);
|
||||
|
||||
writer.StartObject();
|
||||
{
|
||||
SerializeDraftingSequenceReportMembers(sequenceReport, writer);
|
||||
|
||||
// Discarded test runs
|
||||
writer.Key(SequenceReportFields::Keys[SequenceReportFields::DiscardedTestRuns]);
|
||||
SerializeTestSelection(sequenceReport.GetDiscardedTestRuns(), writer);
|
||||
|
||||
// Discarded test run report
|
||||
writer.Key(SequenceReportFields::Keys[SequenceReportFields::DiscardedTestRunReport]);
|
||||
SerializeTestRunReport(sequenceReport.GetDiscardedTestRunReport(), writer);
|
||||
}
|
||||
writer.EndObject();
|
||||
|
||||
return stringBuffer.GetString();
|
||||
}
|
||||
} // namespace TestImpact
|
||||
@@ -8,11 +8,13 @@
|
||||
|
||||
#include <TestImpactFramework/TestImpactClientTestRun.h>
|
||||
|
||||
#include <AzCore/std/tuple.h>
|
||||
|
||||
namespace TestImpact
|
||||
{
|
||||
namespace Client
|
||||
{
|
||||
TestRun::TestRun(
|
||||
TestRunBase::TestRunBase(
|
||||
const AZStd::string& name,
|
||||
const AZStd::string& commandString,
|
||||
AZStd::chrono::high_resolution_clock::time_point startTime,
|
||||
@@ -26,107 +28,145 @@ namespace TestImpact
|
||||
{
|
||||
}
|
||||
|
||||
const AZStd::string& TestRun::GetTargetName() const
|
||||
const AZStd::string& TestRunBase::GetTargetName() const
|
||||
{
|
||||
return m_targetName;
|
||||
}
|
||||
|
||||
const AZStd::string& TestRun::GetCommandString() const
|
||||
const AZStd::string& TestRunBase::GetCommandString() const
|
||||
{
|
||||
return m_commandString;
|
||||
}
|
||||
|
||||
AZStd::chrono::high_resolution_clock::time_point TestRun::GetStartTime() const
|
||||
AZStd::chrono::high_resolution_clock::time_point TestRunBase::GetStartTime() const
|
||||
{
|
||||
return m_startTime;
|
||||
}
|
||||
|
||||
AZStd::chrono::high_resolution_clock::time_point TestRun::GetEndTime() const
|
||||
AZStd::chrono::high_resolution_clock::time_point TestRunBase::GetEndTime() const
|
||||
{
|
||||
return m_startTime + m_duration;
|
||||
}
|
||||
|
||||
AZStd::chrono::milliseconds TestRun::GetDuration() const
|
||||
AZStd::chrono::milliseconds TestRunBase::GetDuration() const
|
||||
{
|
||||
return m_duration;
|
||||
}
|
||||
|
||||
TestRunResult TestRun::GetResult() const
|
||||
TestRunResult TestRunBase::GetResult() const
|
||||
{
|
||||
return m_result;
|
||||
}
|
||||
|
||||
TestFailure::TestFailure(const AZStd::string& testName, const AZStd::string& errorMessage)
|
||||
TestRunWithExecutionFailure::TestRunWithExecutionFailure(TestRunBase&& testRun)
|
||||
: TestRunBase(AZStd::move(testRun))
|
||||
{
|
||||
}
|
||||
|
||||
TimedOutTestRun::TimedOutTestRun(TestRunBase&& testRun)
|
||||
: TestRunBase(AZStd::move(testRun))
|
||||
{
|
||||
}
|
||||
|
||||
UnexecutedTestRun::UnexecutedTestRun(TestRunBase&& testRun)
|
||||
: TestRunBase(AZStd::move(testRun))
|
||||
{
|
||||
}
|
||||
|
||||
Test::Test(const AZStd::string& testName, TestResult result)
|
||||
: m_name(testName)
|
||||
, m_errorMessage(errorMessage)
|
||||
, m_result(result)
|
||||
{
|
||||
}
|
||||
|
||||
const AZStd::string& TestFailure::GetName() const
|
||||
const AZStd::string& Test::GetName() const
|
||||
{
|
||||
return m_name;
|
||||
}
|
||||
|
||||
const AZStd::string& TestFailure::GetErrorMessage() const
|
||||
TestResult Test::GetResult() const
|
||||
{
|
||||
return m_errorMessage;
|
||||
return m_result;
|
||||
}
|
||||
|
||||
TestCaseFailure::TestCaseFailure(const AZStd::string& testCaseName, AZStd::vector<TestFailure>&& testFailures)
|
||||
: m_name(testCaseName)
|
||||
, m_testFailures(AZStd::move(testFailures))
|
||||
AZStd::tuple<size_t, size_t, size_t> CalculateTestCaseMetrics(const AZStd::vector<Test>& tests)
|
||||
{
|
||||
}
|
||||
size_t totalNumPassingTests = 0;
|
||||
size_t totalNumFailingTests = 0;
|
||||
size_t totalNumDisabledTests = 0;
|
||||
|
||||
const AZStd::string& TestCaseFailure::GetName() const
|
||||
{
|
||||
return m_name;
|
||||
}
|
||||
|
||||
const AZStd::vector<TestFailure>& TestCaseFailure::GetTestFailures() const
|
||||
{
|
||||
return m_testFailures;
|
||||
}
|
||||
|
||||
static size_t CalculateNumTestRunFailures(const AZStd::vector<TestCaseFailure>& testFailures)
|
||||
{
|
||||
size_t numTestFailures = 0;
|
||||
for (const auto& testCase : testFailures)
|
||||
for (const auto& test : tests)
|
||||
{
|
||||
numTestFailures += testCase.GetTestFailures().size();
|
||||
if (test.GetResult() == Client::TestResult::Passed)
|
||||
{
|
||||
totalNumPassingTests++;
|
||||
}
|
||||
else if (test.GetResult() == Client::TestResult::Failed)
|
||||
{
|
||||
totalNumFailingTests++;
|
||||
}
|
||||
else
|
||||
{
|
||||
totalNumDisabledTests++;
|
||||
}
|
||||
}
|
||||
|
||||
return numTestFailures;
|
||||
return { totalNumPassingTests, totalNumFailingTests, totalNumDisabledTests };
|
||||
}
|
||||
|
||||
TestRunWithTestFailures::TestRunWithTestFailures(
|
||||
CompletedTestRun::CompletedTestRun(
|
||||
const AZStd::string& name,
|
||||
const AZStd::string& commandString,
|
||||
AZStd::chrono::high_resolution_clock::time_point startTime,
|
||||
AZStd::chrono::milliseconds duration,
|
||||
TestRunResult result,
|
||||
AZStd::vector<TestCaseFailure>&& testFailures)
|
||||
: TestRun(name, commandString, startTime, duration, result)
|
||||
, m_testCaseFailures(AZStd::move(testFailures))
|
||||
AZStd::vector<Test>&& tests)
|
||||
: TestRunBase(name, commandString, startTime, duration, result)
|
||||
, m_tests(AZStd::move(tests))
|
||||
{
|
||||
m_numTestFailures = CalculateNumTestRunFailures(m_testCaseFailures);
|
||||
AZStd::tie(m_totalNumPassingTests, m_totalNumFailingTests, m_totalNumDisabledTests) = CalculateTestCaseMetrics(m_tests);
|
||||
}
|
||||
|
||||
TestRunWithTestFailures::TestRunWithTestFailures(TestRun&& testRun, AZStd::vector<TestCaseFailure>&& testFailures)
|
||||
: TestRun(AZStd::move(testRun))
|
||||
, m_testCaseFailures(AZStd::move(testFailures))
|
||||
CompletedTestRun::CompletedTestRun(TestRunBase&& testRun, AZStd::vector<Test>&& tests)
|
||||
: TestRunBase(AZStd::move(testRun))
|
||||
, m_tests(AZStd::move(tests))
|
||||
{
|
||||
m_numTestFailures = CalculateNumTestRunFailures(m_testCaseFailures);
|
||||
AZStd::tie(m_totalNumPassingTests, m_totalNumFailingTests, m_totalNumDisabledTests) = CalculateTestCaseMetrics(m_tests);
|
||||
}
|
||||
|
||||
size_t TestRunWithTestFailures::GetNumTestFailures() const
|
||||
size_t CompletedTestRun::GetTotalNumTests() const
|
||||
{
|
||||
return m_numTestFailures;
|
||||
return m_tests.size();
|
||||
}
|
||||
|
||||
const AZStd::vector<TestCaseFailure>& TestRunWithTestFailures::GetTestCaseFailures() const
|
||||
size_t CompletedTestRun::GetTotalNumPassingTests() const
|
||||
{
|
||||
return m_totalNumPassingTests;
|
||||
}
|
||||
|
||||
size_t CompletedTestRun::GetTotalNumFailingTests() const
|
||||
{
|
||||
return m_totalNumFailingTests;
|
||||
}
|
||||
|
||||
size_t CompletedTestRun::GetTotalNumDisabledTests() const
|
||||
{
|
||||
return m_totalNumDisabledTests;
|
||||
}
|
||||
|
||||
const AZStd::vector<Test>& CompletedTestRun::GetTests() const
|
||||
{
|
||||
return m_tests;
|
||||
}
|
||||
|
||||
PassingTestRun::PassingTestRun(TestRunBase&& testRun, AZStd::vector<Test>&& tests)
|
||||
: CompletedTestRun(AZStd::move(testRun), AZStd::move(tests))
|
||||
{
|
||||
}
|
||||
|
||||
FailingTestRun::FailingTestRun(TestRunBase&& testRun, AZStd::vector<Test>&& tests)
|
||||
: CompletedTestRun(AZStd::move(testRun), AZStd::move(tests))
|
||||
{
|
||||
return m_testCaseFailures;
|
||||
}
|
||||
} // namespace Client
|
||||
} // namespace TestImpact
|
||||
|
||||
@@ -6,7 +6,7 @@
|
||||
*
|
||||
*/
|
||||
|
||||
#include <TestImpactFramework/TestImpactFileUtils.h>
|
||||
#include <TestImpactFramework/TestImpactUtils.h>
|
||||
#include <TestImpactFramework/TestImpactRuntime.h>
|
||||
#include <TestImpactFramework/TestImpactRuntimeException.h>
|
||||
|
||||
@@ -81,7 +81,7 @@ namespace TestImpact
|
||||
{
|
||||
if (m_testCompleteCallback.has_value())
|
||||
{
|
||||
Client::TestRun testRun(
|
||||
Client::TestRunBase testRun(
|
||||
testJob.GetTestTarget()->GetName(),
|
||||
testJob.GetCommandString(),
|
||||
testJob.GetStartTime(),
|
||||
@@ -110,8 +110,140 @@ namespace TestImpact
|
||||
return result;
|
||||
}
|
||||
|
||||
//! Utility structure for holding the pertinent data for test run reports.
|
||||
template<typename TestJob>
|
||||
struct TestRunData
|
||||
{
|
||||
TestSequenceResult m_result = TestSequenceResult::Success;
|
||||
AZStd::vector<TestJob> m_jobs;
|
||||
AZStd::chrono::high_resolution_clock::time_point m_relativeStartTime;
|
||||
AZStd::chrono::milliseconds m_duration = AZStd::chrono::milliseconds{ 0 };
|
||||
};
|
||||
|
||||
//! Wrapper for the impact analysis test sequence to handle both the updating and non-updating policies through a common pathway.
|
||||
//! @tparam TestRunnerFunctor The functor for running the specified tests.
|
||||
//! @tparam TestJob The test engine job type returned by the functor.
|
||||
//! @param maxConcurrency The maximum concurrency being used for this sequence.
|
||||
//! @param policyState The policy state being used for the sequence.
|
||||
//! @param suiteType The suite type used for this sequence.
|
||||
//! @param timer The timer to use for the test run timings.
|
||||
//! @param testRunner The test runner functor to use for each of the test runs.
|
||||
//! @param includedSelectedTestTargets The subset of test targets that were selected to run and not also fully excluded from running.
|
||||
//! @param excludedSelectedTestTargets The subset of test targets that were selected to run but were fully excluded running.
|
||||
//! @param discardedTestTargets The subset of test targets that were discarded from the test selection and will not be run.
|
||||
//! @param globalTimeout The maximum duration the entire test sequence may run for (infinite if empty).
|
||||
//! @param testSequenceStartCallback The client function to be called after the test targets have been selected but prior to running the
|
||||
//! tests.
|
||||
//! @param testSequenceCompleteCallback The client function to be called after the test sequence has completed.
|
||||
//! @param testRunCompleteCallback The client function to be called after an individual test run has completed.
|
||||
//! @param updateCoverage The function to call to update the dynamic dependency map with test coverage (if any).
|
||||
template<typename TestRunnerFunctor, typename TestJob>
|
||||
Client::ImpactAnalysisSequenceReport ImpactAnalysisTestSequenceWrapper(
|
||||
size_t maxConcurrency,
|
||||
const ImpactAnalysisSequencePolicyState& policyState,
|
||||
SuiteType suiteType,
|
||||
const Timer& sequenceTimer,
|
||||
const TestRunnerFunctor& testRunner,
|
||||
const AZStd::vector<const TestTarget*>& includedSelectedTestTargets,
|
||||
const AZStd::vector<const TestTarget*>& excludedSelectedTestTargets,
|
||||
const AZStd::vector<const TestTarget*>& discardedTestTargets,
|
||||
const AZStd::vector<const TestTarget*>& draftedTestTargets,
|
||||
const AZStd::optional<AZStd::chrono::milliseconds>& testTargetTimeout,
|
||||
const AZStd::optional<AZStd::chrono::milliseconds>& globalTimeout,
|
||||
AZStd::optional<ImpactAnalysisTestSequenceStartCallback> testSequenceStartCallback,
|
||||
AZStd::optional<TestSequenceCompleteCallback<Client::ImpactAnalysisSequenceReport>> testSequenceEndCallback,
|
||||
AZStd::optional<TestRunCompleteCallback> testCompleteCallback,
|
||||
AZStd::optional<AZStd::function<void(const AZStd::vector<TestJob>& jobs)>> updateCoverage)
|
||||
{
|
||||
TestRunData<TestJob> selectedTestRunData, draftedTestRunData;
|
||||
AZStd::optional<AZStd::chrono::milliseconds> sequenceTimeout = globalTimeout;
|
||||
|
||||
// Extract the client facing representation of selected, discarded and drafted test targets
|
||||
const Client::TestRunSelection selectedTests(
|
||||
ExtractTestTargetNames(includedSelectedTestTargets), ExtractTestTargetNames(excludedSelectedTestTargets));
|
||||
const auto discardedTests = ExtractTestTargetNames(discardedTestTargets);
|
||||
const auto draftedTests = ExtractTestTargetNames(draftedTestTargets);
|
||||
|
||||
// Inform the client that the sequence is about to start
|
||||
if (testSequenceStartCallback.has_value())
|
||||
{
|
||||
(*testSequenceStartCallback)(suiteType, selectedTests, discardedTests, draftedTests);
|
||||
}
|
||||
|
||||
// We share the test run complete handler between the selected and drafted test runs as to present them together as one
|
||||
// continuous test sequence to the client rather than two discrete test runs
|
||||
const size_t totalNumTestRuns = includedSelectedTestTargets.size() + draftedTestTargets.size();
|
||||
TestRunCompleteCallbackHandler testRunCompleteHandler(totalNumTestRuns, testCompleteCallback);
|
||||
|
||||
const auto gatherTestRunData = [&sequenceTimer, &testRunner, &testRunCompleteHandler, &globalTimeout]
|
||||
(const AZStd::vector<const TestTarget*>& testsTargets, TestRunData<TestJob>& testRunData)
|
||||
{
|
||||
const Timer testRunTimer;
|
||||
testRunData.m_relativeStartTime = testRunTimer.GetStartTimePointRelative(sequenceTimer);
|
||||
auto [result, jobs] = testRunner(testsTargets, testRunCompleteHandler, globalTimeout);
|
||||
testRunData.m_result = result;
|
||||
testRunData.m_jobs = AZStd::move(jobs);
|
||||
testRunData.m_duration = testRunTimer.GetElapsedMs();
|
||||
};
|
||||
|
||||
if (!includedSelectedTestTargets.empty())
|
||||
{
|
||||
// Run the selected test targets and collect the test run results
|
||||
gatherTestRunData(includedSelectedTestTargets, selectedTestRunData);
|
||||
|
||||
// Carry the remaining global sequence time over to the drafted test run
|
||||
if (globalTimeout.has_value())
|
||||
{
|
||||
const auto elapsed = selectedTestRunData.m_duration;
|
||||
sequenceTimeout = elapsed < globalTimeout.value() ? globalTimeout.value() - elapsed : AZStd::chrono::milliseconds(0);
|
||||
}
|
||||
}
|
||||
|
||||
if (!draftedTestTargets.empty())
|
||||
{
|
||||
// Run the drafted test targets and collect the test run results
|
||||
gatherTestRunData(draftedTestTargets, draftedTestRunData);
|
||||
}
|
||||
|
||||
// Generate the sequence report for the client
|
||||
const auto sequenceReport = Client::ImpactAnalysisSequenceReport(
|
||||
maxConcurrency,
|
||||
testTargetTimeout,
|
||||
globalTimeout,
|
||||
policyState,
|
||||
suiteType,
|
||||
selectedTests,
|
||||
discardedTests,
|
||||
draftedTests,
|
||||
GenerateTestRunReport(
|
||||
selectedTestRunData.m_result,
|
||||
selectedTestRunData.m_relativeStartTime,
|
||||
selectedTestRunData.m_duration,
|
||||
selectedTestRunData.m_jobs),
|
||||
GenerateTestRunReport(
|
||||
draftedTestRunData.m_result,
|
||||
draftedTestRunData.m_relativeStartTime,
|
||||
draftedTestRunData.m_duration,
|
||||
draftedTestRunData.m_jobs));
|
||||
|
||||
// Inform the client that the sequence has ended
|
||||
if (testSequenceEndCallback.has_value())
|
||||
{
|
||||
(*testSequenceEndCallback)(sequenceReport);
|
||||
}
|
||||
|
||||
// Update the dynamic dependency map with the latest coverage data (if any)
|
||||
if (updateCoverage.has_value())
|
||||
{
|
||||
(*updateCoverage)(ConcatenateVectors(selectedTestRunData.m_jobs, draftedTestRunData.m_jobs));
|
||||
}
|
||||
|
||||
return sequenceReport;
|
||||
}
|
||||
|
||||
Runtime::Runtime(
|
||||
RuntimeConfig&& config,
|
||||
AZStd::optional<RepoPath> dataFile,
|
||||
SuiteType suiteFilter,
|
||||
Policy::ExecutionFailure executionFailurePolicy,
|
||||
Policy::FailedTestCoverage failedTestCoveragePolicy,
|
||||
@@ -151,27 +283,22 @@ namespace TestImpact
|
||||
|
||||
try
|
||||
{
|
||||
if (dataFile.has_value())
|
||||
{
|
||||
m_sparTiaFile = dataFile.value().String();
|
||||
}
|
||||
else
|
||||
{
|
||||
m_sparTiaFile = m_config.m_workspace.m_active.m_sparTiaFiles[static_cast<size_t>(m_suiteFilter)].String();
|
||||
}
|
||||
|
||||
// Populate the dynamic dependency map with the existing source coverage data (if any)
|
||||
m_sparTIAFile = m_config.m_workspace.m_active.m_sparTIAFiles[static_cast<size_t>(m_suiteFilter)].String();
|
||||
const auto tiaDataRaw = ReadFileContents<Exception>(m_sparTIAFile);
|
||||
const auto tiaDataRaw = ReadFileContents<Exception>(m_sparTiaFile);
|
||||
const auto tiaData = DeserializeSourceCoveringTestsList(tiaDataRaw);
|
||||
if (tiaData.GetNumSources())
|
||||
{
|
||||
m_dynamicDependencyMap->ReplaceSourceCoverage(tiaData);
|
||||
m_hasImpactAnalysisData = true;
|
||||
|
||||
// Enumerate new test targets
|
||||
const auto testTargetsWithNoEnumeration = m_dynamicDependencyMap->GetNotCoveringTests();
|
||||
if (!testTargetsWithNoEnumeration.empty())
|
||||
{
|
||||
m_testEngine->UpdateEnumerationCache(
|
||||
testTargetsWithNoEnumeration,
|
||||
Policy::ExecutionFailure::Ignore,
|
||||
Policy::TestFailure::Continue,
|
||||
AZStd::nullopt,
|
||||
AZStd::nullopt,
|
||||
AZStd::nullopt);
|
||||
}
|
||||
}
|
||||
}
|
||||
catch (const DependencyException& e)
|
||||
@@ -186,7 +313,7 @@ namespace TestImpact
|
||||
AZ_Printf(
|
||||
LogCallSite,
|
||||
AZStd::string::format(
|
||||
"No test impact analysis data found for suite '%s' at %s\n", GetSuiteTypeName(m_suiteFilter).c_str(), m_sparTIAFile.c_str()).c_str());
|
||||
"No test impact analysis data found for suite '%s' at %s\n", SuiteTypeAsString(m_suiteFilter).c_str(), m_sparTiaFile.c_str()).c_str());
|
||||
}
|
||||
}
|
||||
|
||||
@@ -230,7 +357,7 @@ namespace TestImpact
|
||||
}
|
||||
}
|
||||
|
||||
AZStd::pair<AZStd::vector<const TestTarget*>, AZStd::vector<const TestTarget*>> Runtime::SelectCoveringTestTargetsAndUpdateEnumerationCache(
|
||||
AZStd::pair<AZStd::vector<const TestTarget*>, AZStd::vector<const TestTarget*>> Runtime::SelectCoveringTestTargets(
|
||||
const ChangeList& changeList,
|
||||
Policy::TestPrioritization testPrioritizationPolicy)
|
||||
{
|
||||
@@ -243,9 +370,6 @@ namespace TestImpact
|
||||
// Populate a set with the selected test targets so that we can infer the discarded test target not selected for this change list
|
||||
const AZStd::unordered_set<const TestTarget*> selectedTestTargetSet(selectedTestTargets.begin(), selectedTestTargets.end());
|
||||
|
||||
// Update the enumeration caches of mutated targets regardless of the current sharding policy
|
||||
EnumerateMutatedTestTargets(changeDependencyList);
|
||||
|
||||
// The test targets in the main list not in the selected test target set are the test targets not selected for this change list
|
||||
for (const auto& testTarget : m_dynamicDependencyMap->GetTestTargetList().GetTargets())
|
||||
{
|
||||
@@ -287,7 +411,7 @@ namespace TestImpact
|
||||
void Runtime::ClearDynamicDependencyMapAndRemoveExistingFile()
|
||||
{
|
||||
m_dynamicDependencyMap->ClearAllSourceCoverage();
|
||||
DeleteFile(m_sparTIAFile);
|
||||
DeleteFile(m_sparTiaFile);
|
||||
}
|
||||
|
||||
SourceCoveringTestsList Runtime::CreateSourceCoveringTestFromTestCoverages(const AZStd::vector<TestEngineInstrumentedRun>& jobs)
|
||||
@@ -368,9 +492,9 @@ namespace TestImpact
|
||||
}
|
||||
|
||||
m_dynamicDependencyMap->ReplaceSourceCoverage(sourceCoverageTestsList);
|
||||
const auto sparTIA = m_dynamicDependencyMap->ExportSourceCoverage();
|
||||
const auto sparTIAData = SerializeSourceCoveringTestsList(sparTIA);
|
||||
WriteFileContents<RuntimeException>(sparTIAData, m_sparTIAFile);
|
||||
const auto sparTia = m_dynamicDependencyMap->ExportSourceCoverage();
|
||||
const auto sparTiaData = SerializeSourceCoveringTestsList(sparTia);
|
||||
WriteFileContents<RuntimeException>(sparTiaData, m_sparTiaFile);
|
||||
m_hasImpactAnalysisData = true;
|
||||
}
|
||||
catch(const RuntimeException& e)
|
||||
@@ -386,11 +510,42 @@ namespace TestImpact
|
||||
}
|
||||
}
|
||||
|
||||
Client::SequenceReport Runtime::RegularTestSequence(
|
||||
PolicyStateBase Runtime::GeneratePolicyStateBase() const
|
||||
{
|
||||
PolicyStateBase policyState;
|
||||
|
||||
policyState.m_executionFailurePolicy = m_executionFailurePolicy;
|
||||
policyState.m_failedTestCoveragePolicy = m_failedTestCoveragePolicy;
|
||||
policyState.m_integrityFailurePolicy = m_integrationFailurePolicy;
|
||||
policyState.m_targetOutputCapture = m_targetOutputCapture;
|
||||
policyState.m_testFailurePolicy = m_testFailurePolicy;
|
||||
policyState.m_testShardingPolicy = m_testShardingPolicy;
|
||||
|
||||
return policyState;
|
||||
}
|
||||
|
||||
SequencePolicyState Runtime::GenerateSequencePolicyState() const
|
||||
{
|
||||
return { GeneratePolicyStateBase() };
|
||||
}
|
||||
|
||||
SafeImpactAnalysisSequencePolicyState Runtime::GenerateSafeImpactAnalysisSequencePolicyState(
|
||||
Policy::TestPrioritization testPrioritizationPolicy) const
|
||||
{
|
||||
return { GeneratePolicyStateBase(), testPrioritizationPolicy };
|
||||
}
|
||||
|
||||
ImpactAnalysisSequencePolicyState Runtime::GenerateImpactAnalysisSequencePolicyState(
|
||||
Policy::TestPrioritization testPrioritizationPolicy, Policy::DynamicDependencyMap dynamicDependencyMapPolicy) const
|
||||
{
|
||||
return { GeneratePolicyStateBase(), testPrioritizationPolicy, dynamicDependencyMapPolicy };
|
||||
}
|
||||
|
||||
Client::RegularSequenceReport Runtime::RegularTestSequence(
|
||||
AZStd::optional<AZStd::chrono::milliseconds> testTargetTimeout,
|
||||
AZStd::optional<AZStd::chrono::milliseconds> globalTimeout,
|
||||
AZStd::optional<TestSequenceStartCallback> testSequenceStartCallback,
|
||||
AZStd::optional<TestSequenceCompleteCallback<Client::SequenceReport>> testSequenceEndCallback,
|
||||
AZStd::optional<TestSequenceCompleteCallback<Client::RegularSequenceReport>> testSequenceEndCallback,
|
||||
AZStd::optional<TestRunCompleteCallback> testCompleteCallback)
|
||||
{
|
||||
const Timer sequenceTimer;
|
||||
@@ -434,7 +589,11 @@ namespace TestImpact
|
||||
const auto testRunDuration = testRunTimer.GetElapsedMs();
|
||||
|
||||
// Generate the sequence report for the client
|
||||
const auto sequenceReport = Client::SequenceReport(
|
||||
const auto sequenceReport = Client::RegularSequenceReport(
|
||||
m_maxConcurrency,
|
||||
testTargetTimeout,
|
||||
globalTimeout,
|
||||
GenerateSequencePolicyState(),
|
||||
m_suiteFilter,
|
||||
selectedTests,
|
||||
GenerateTestRunReport(result, testRunTimer.GetStartTimePointRelative(sequenceTimer), testRunDuration, testJobs));
|
||||
@@ -448,95 +607,6 @@ namespace TestImpact
|
||||
return sequenceReport;
|
||||
}
|
||||
|
||||
//! Wrapper for the impact analysis test sequence to handle both the updating and non-updating policies through a common pathway.
|
||||
//! @tparam TestRunnerFunctor The functor for running the specified tests.
|
||||
//! @tparam TestJob The test engine job type returned by the functor.
|
||||
//! @param suiteType The suite type used for this sequence.
|
||||
//! @param timer The timer to use for the test run timings.
|
||||
//! @param testRunner The test runner functor to use for each of the test runs.
|
||||
//! @param includedSelectedTestTargets The subset of test targets that were selected to run and not also fully excluded from running.
|
||||
//! @param excludedSelectedTestTargets The subset of test targets that were selected to run but were fully excluded running.
|
||||
//! @param discardedTestTargets The subset of test targets that were discarded from the test selection and will not be run.
|
||||
//! @param globalTimeout The maximum duration the entire test sequence may run for (infinite if empty).
|
||||
//! @param testSequenceStartCallback The client function to be called after the test targets have been selected but prior to running the tests.
|
||||
//! @param testSequenceCompleteCallback The client function to be called after the test sequence has completed.
|
||||
//! @param testRunCompleteCallback The client function to be called after an individual test run has completed.
|
||||
//! @param updateCoverage The function to call to update the dynamic dependency map with test coverage (if any).
|
||||
template<typename TestRunnerFunctor, typename TestJob>
|
||||
Client::ImpactAnalysisSequenceReport ImpactAnalysisTestSequenceWrapper(
|
||||
SuiteType suiteType,
|
||||
const Timer& sequenceTimer,
|
||||
const TestRunnerFunctor& testRunner,
|
||||
const AZStd::vector<const TestTarget*>& includedSelectedTestTargets,
|
||||
const AZStd::vector<const TestTarget*>& excludedSelectedTestTargets,
|
||||
const AZStd::vector<const TestTarget*>& discardedTestTargets,
|
||||
const AZStd::vector<const TestTarget*>& draftedTestTargets,
|
||||
const AZStd::optional<AZStd::chrono::milliseconds> globalTimeout,
|
||||
AZStd::optional<ImpactAnalysisTestSequenceStartCallback> testSequenceStartCallback,
|
||||
AZStd::optional<TestSequenceCompleteCallback<Client::ImpactAnalysisSequenceReport>> testSequenceEndCallback,
|
||||
AZStd::optional<TestRunCompleteCallback> testCompleteCallback,
|
||||
AZStd::optional<AZStd::function<void(const AZStd::vector<TestJob>& jobs)>> updateCoverage)
|
||||
{
|
||||
AZStd::optional<AZStd::chrono::milliseconds> sequenceTimeout = globalTimeout;
|
||||
|
||||
// Extract the client facing representation of selected, discarded and drafted test targets
|
||||
const Client::TestRunSelection selectedTests(
|
||||
ExtractTestTargetNames(includedSelectedTestTargets), ExtractTestTargetNames(excludedSelectedTestTargets));
|
||||
const auto discardedTests = ExtractTestTargetNames(discardedTestTargets);
|
||||
const auto draftedTests = ExtractTestTargetNames(draftedTestTargets);
|
||||
|
||||
// Inform the client that the sequence is about to start
|
||||
if (testSequenceStartCallback.has_value())
|
||||
{
|
||||
(*testSequenceStartCallback)(suiteType, selectedTests, discardedTests, draftedTests);
|
||||
}
|
||||
|
||||
// We share the test run complete handler between the selected and drafted test runs as to present them together as one
|
||||
// continuous test sequence to the client rather than two discrete test runs
|
||||
const size_t totalNumTestRuns = includedSelectedTestTargets.size() + draftedTestTargets.size();
|
||||
TestRunCompleteCallbackHandler testRunCompleteHandler(totalNumTestRuns, testCompleteCallback);
|
||||
|
||||
// Run the selected test targets and collect the test run results
|
||||
const Timer selectedTestRunTimer;
|
||||
const auto [selectedResult, selectedTestJobs] = testRunner(includedSelectedTestTargets, testRunCompleteHandler, globalTimeout);
|
||||
const auto selectedTestRunDuration = selectedTestRunTimer.GetElapsedMs();
|
||||
|
||||
// Carry the remaining global sequence time over to the drafted test run
|
||||
if (globalTimeout.has_value())
|
||||
{
|
||||
const auto elapsed = selectedTestRunDuration;
|
||||
sequenceTimeout = elapsed < globalTimeout.value() ? globalTimeout.value() - elapsed : AZStd::chrono::milliseconds(0);
|
||||
}
|
||||
|
||||
// Run the drafted test targets and collect the test run results
|
||||
Timer draftedTestRunTimer;
|
||||
const auto [draftedResult, draftedTestJobs] = testRunner(draftedTestTargets, testRunCompleteHandler, globalTimeout);
|
||||
const auto draftedTestRunDuration = draftedTestRunTimer.GetElapsedMs();
|
||||
|
||||
// Generate the sequence report for the client
|
||||
const auto sequenceReport = Client::ImpactAnalysisSequenceReport(
|
||||
suiteType,
|
||||
selectedTests,
|
||||
discardedTests,
|
||||
draftedTests,
|
||||
GenerateTestRunReport(selectedResult, selectedTestRunTimer.GetStartTimePointRelative(sequenceTimer), selectedTestRunDuration, selectedTestJobs),
|
||||
GenerateTestRunReport(draftedResult, draftedTestRunTimer.GetStartTimePointRelative(sequenceTimer), draftedTestRunDuration, draftedTestJobs));
|
||||
|
||||
// Inform the client that the sequence has ended
|
||||
if (testSequenceEndCallback.has_value())
|
||||
{
|
||||
(*testSequenceEndCallback)(sequenceReport);
|
||||
}
|
||||
|
||||
// Update the dynamic dependency map with the latest coverage data (if any)
|
||||
if (updateCoverage.has_value())
|
||||
{
|
||||
(*updateCoverage)(ConcatenateVectors(selectedTestJobs, draftedTestJobs));
|
||||
}
|
||||
|
||||
return sequenceReport;
|
||||
}
|
||||
|
||||
Client::ImpactAnalysisSequenceReport Runtime::ImpactAnalysisTestSequence(
|
||||
const ChangeList& changeList,
|
||||
Policy::TestPrioritization testPrioritizationPolicy,
|
||||
@@ -550,10 +620,30 @@ namespace TestImpact
|
||||
const Timer sequenceTimer;
|
||||
|
||||
// Draft in the test targets that have no coverage entries in the dynamic dependency map
|
||||
AZStd::vector<const TestTarget*> draftedTestTargets = m_dynamicDependencyMap->GetNotCoveringTests();
|
||||
const AZStd::vector<const TestTarget*> draftedTestTargets = m_dynamicDependencyMap->GetNotCoveringTests();
|
||||
|
||||
// The test targets that were selected for the change list by the dynamic dependency map and the test targets that were not
|
||||
auto [selectedTestTargets, discardedTestTargets] = SelectCoveringTestTargetsAndUpdateEnumerationCache(changeList, testPrioritizationPolicy);
|
||||
const auto selectCoveringTestTargetsAndPruneDraftedFromDiscarded =
|
||||
[this, &draftedTestTargets, &changeList, testPrioritizationPolicy]()
|
||||
{
|
||||
// The test targets that were selected for the change list by the dynamic dependency map and the test targets that were not
|
||||
const auto [selectedTestTargets, discardedTestTargets] =
|
||||
SelectCoveringTestTargets(changeList, testPrioritizationPolicy);
|
||||
|
||||
const AZStd::unordered_set<const TestTarget*> draftedTestTargetsSet(draftedTestTargets.begin(), draftedTestTargets.end());
|
||||
|
||||
AZStd::vector<const TestTarget*> discardedNotDraftedTestTargets;
|
||||
for (const auto* testTarget : discardedTestTargets)
|
||||
{
|
||||
if (!draftedTestTargetsSet.count(testTarget))
|
||||
{
|
||||
discardedNotDraftedTestTargets.push_back(testTarget);
|
||||
}
|
||||
}
|
||||
|
||||
return AZStd::pair{ selectedTestTargets, discardedNotDraftedTestTargets };
|
||||
};
|
||||
|
||||
const auto [selectedTestTargets, discardedTestTargets] = selectCoveringTestTargetsAndPruneDraftedFromDiscarded();
|
||||
|
||||
// The subset of selected test targets that are not on the configuration's exclude list and those that are
|
||||
auto [includedSelectedTestTargets, excludedSelectedTestTargets] = SelectTestTargetsByExcludeList(selectedTestTargets);
|
||||
@@ -604,6 +694,8 @@ namespace TestImpact
|
||||
};
|
||||
|
||||
return ImpactAnalysisTestSequenceWrapper(
|
||||
m_maxConcurrency,
|
||||
GenerateImpactAnalysisSequencePolicyState(testPrioritizationPolicy, dynamicDependencyMapPolicy),
|
||||
m_suiteFilter,
|
||||
sequenceTimer,
|
||||
instrumentedTestRun,
|
||||
@@ -611,6 +703,7 @@ namespace TestImpact
|
||||
excludedSelectedTestTargets,
|
||||
discardedTestTargets,
|
||||
draftedTestTargets,
|
||||
testTargetTimeout,
|
||||
globalTimeout,
|
||||
testSequenceStartCallback,
|
||||
testSequenceEndCallback,
|
||||
@@ -620,6 +713,8 @@ namespace TestImpact
|
||||
else
|
||||
{
|
||||
return ImpactAnalysisTestSequenceWrapper(
|
||||
m_maxConcurrency,
|
||||
GenerateImpactAnalysisSequencePolicyState(testPrioritizationPolicy, dynamicDependencyMapPolicy),
|
||||
m_suiteFilter,
|
||||
sequenceTimer,
|
||||
regularTestRun,
|
||||
@@ -627,6 +722,7 @@ namespace TestImpact
|
||||
excludedSelectedTestTargets,
|
||||
discardedTestTargets,
|
||||
draftedTestTargets,
|
||||
testTargetTimeout,
|
||||
globalTimeout,
|
||||
testSequenceStartCallback,
|
||||
testSequenceEndCallback,
|
||||
@@ -645,13 +741,15 @@ namespace TestImpact
|
||||
AZStd::optional<TestRunCompleteCallback> testCompleteCallback)
|
||||
{
|
||||
const Timer sequenceTimer;
|
||||
auto sequenceTimeout = globalTimeout;
|
||||
TestRunData<TestEngineInstrumentedRun> selectedTestRunData, draftedTestRunData;
|
||||
TestRunData<TestEngineRegularRun> discardedTestRunData;
|
||||
AZStd::optional<AZStd::chrono::milliseconds> sequenceTimeout = globalTimeout;
|
||||
|
||||
// Draft in the test targets that have no coverage entries in the dynamic dependency map
|
||||
AZStd::vector<const TestTarget*> draftedTestTargets = m_dynamicDependencyMap->GetNotCoveringTests();
|
||||
|
||||
// The test targets that were selected for the change list by the dynamic dependency map and the test targets that were not
|
||||
auto [selectedTestTargets, discardedTestTargets] = SelectCoveringTestTargetsAndUpdateEnumerationCache(changeList, testPrioritizationPolicy);
|
||||
const auto [selectedTestTargets, discardedTestTargets] = SelectCoveringTestTargets(changeList, testPrioritizationPolicy);
|
||||
|
||||
// The subset of selected test targets that are not on the configuration's exclude list and those that are
|
||||
auto [includedSelectedTestTargets, excludedSelectedTestTargets] = SelectTestTargetsByExcludeList(selectedTestTargets);
|
||||
@@ -675,71 +773,107 @@ namespace TestImpact
|
||||
// continuous test sequence to the client rather than three discrete test runs
|
||||
const size_t totalNumTestRuns = includedSelectedTestTargets.size() + draftedTestTargets.size() + includedDiscardedTestTargets.size();
|
||||
TestRunCompleteCallbackHandler testRunCompleteHandler(totalNumTestRuns, testCompleteCallback);
|
||||
|
||||
// Run the selected test targets and collect the test run results
|
||||
const Timer selectedTestRunTimer;
|
||||
const auto [selectedResult, selectedTestJobs] = m_testEngine->InstrumentedRun(
|
||||
includedSelectedTestTargets,
|
||||
m_testShardingPolicy,
|
||||
m_executionFailurePolicy,
|
||||
m_integrationFailurePolicy,
|
||||
m_testFailurePolicy,
|
||||
m_targetOutputCapture,
|
||||
testTargetTimeout,
|
||||
sequenceTimeout,
|
||||
AZStd::ref(testRunCompleteHandler));
|
||||
const auto selectedTestRunDuration = selectedTestRunTimer.GetElapsedMs();
|
||||
|
||||
// Carry the remaining global sequence time over to the discarded test run
|
||||
if (globalTimeout.has_value())
|
||||
|
||||
// Functor for running instrumented test targets
|
||||
const auto instrumentedTestRun =
|
||||
[this, &testTargetTimeout, &sequenceTimeout, &testRunCompleteHandler](const AZStd::vector<const TestTarget*>& testsTargets)
|
||||
{
|
||||
const auto elapsed = selectedTestRunDuration;
|
||||
sequenceTimeout = elapsed < globalTimeout.value() ? globalTimeout.value() - elapsed : AZStd::chrono::milliseconds(0);
|
||||
return m_testEngine->InstrumentedRun(
|
||||
testsTargets,
|
||||
m_testShardingPolicy,
|
||||
m_executionFailurePolicy,
|
||||
m_integrationFailurePolicy,
|
||||
m_testFailurePolicy,
|
||||
m_targetOutputCapture,
|
||||
testTargetTimeout,
|
||||
sequenceTimeout,
|
||||
AZStd::ref(testRunCompleteHandler));
|
||||
};
|
||||
|
||||
// Functor for running uninstrumented test targets
|
||||
const auto regularTestRun =
|
||||
[this, &testTargetTimeout, &sequenceTimeout, &testRunCompleteHandler](const AZStd::vector<const TestTarget*>& testsTargets)
|
||||
{
|
||||
return m_testEngine->RegularRun(
|
||||
testsTargets,
|
||||
m_testShardingPolicy,
|
||||
m_executionFailurePolicy,
|
||||
m_testFailurePolicy,
|
||||
m_targetOutputCapture,
|
||||
testTargetTimeout,
|
||||
sequenceTimeout,
|
||||
AZStd::ref(testRunCompleteHandler));
|
||||
};
|
||||
|
||||
// Functor for running instrumented test targets
|
||||
const auto gatherTestRunData = [&sequenceTimer]
|
||||
(const AZStd::vector<const TestTarget*>& testsTargets, const auto& testRunner, auto& testRunData)
|
||||
{
|
||||
const Timer testRunTimer;
|
||||
testRunData.m_relativeStartTime = testRunTimer.GetStartTimePointRelative(sequenceTimer);
|
||||
auto [result, jobs] = testRunner(testsTargets);
|
||||
testRunData.m_result = result;
|
||||
testRunData.m_jobs = AZStd::move(jobs);
|
||||
testRunData.m_duration = testRunTimer.GetElapsedMs();
|
||||
};
|
||||
|
||||
if (!includedSelectedTestTargets.empty())
|
||||
{
|
||||
// Run the selected test targets and collect the test run results
|
||||
gatherTestRunData(includedSelectedTestTargets, instrumentedTestRun, selectedTestRunData);
|
||||
|
||||
// Carry the remaining global sequence time over to the discarded test run
|
||||
if (globalTimeout.has_value())
|
||||
{
|
||||
const auto elapsed = selectedTestRunData.m_duration;
|
||||
sequenceTimeout = elapsed < globalTimeout.value() ? globalTimeout.value() - elapsed : AZStd::chrono::milliseconds(0);
|
||||
}
|
||||
}
|
||||
|
||||
// Run the discarded test targets and collect the test run results
|
||||
const Timer discardedTestRunTimer;
|
||||
const auto [discardedResult, discardedTestJobs] = m_testEngine->RegularRun(
|
||||
includedDiscardedTestTargets,
|
||||
m_testShardingPolicy,
|
||||
m_executionFailurePolicy,
|
||||
m_testFailurePolicy,
|
||||
m_targetOutputCapture,
|
||||
testTargetTimeout,
|
||||
sequenceTimeout,
|
||||
AZStd::ref(testRunCompleteHandler));
|
||||
const auto discardedTestRunDuration = discardedTestRunTimer.GetElapsedMs();
|
||||
|
||||
// Carry the remaining global sequence time over to the drafted test run
|
||||
if (globalTimeout.has_value())
|
||||
if (!includedDiscardedTestTargets.empty())
|
||||
{
|
||||
const auto elapsed = selectedTestRunDuration + discardedTestRunDuration;
|
||||
sequenceTimeout = elapsed < globalTimeout.value() ? globalTimeout.value() - elapsed : AZStd::chrono::milliseconds(0);
|
||||
// Run the discarded test targets and collect the test run results
|
||||
gatherTestRunData(includedDiscardedTestTargets, regularTestRun, discardedTestRunData);
|
||||
|
||||
// Carry the remaining global sequence time over to the drafted test run
|
||||
if (globalTimeout.has_value())
|
||||
{
|
||||
const auto elapsed = selectedTestRunData.m_duration + discardedTestRunData.m_duration;
|
||||
sequenceTimeout = elapsed < globalTimeout.value() ? globalTimeout.value() - elapsed : AZStd::chrono::milliseconds(0);
|
||||
}
|
||||
}
|
||||
|
||||
// Run the drafted test targets and collect the test run results
|
||||
const Timer draftedTestRunTimer;
|
||||
const auto [draftedResult, draftedTestJobs] = m_testEngine->InstrumentedRun(
|
||||
draftedTestTargets,
|
||||
m_testShardingPolicy,
|
||||
m_executionFailurePolicy,
|
||||
m_integrationFailurePolicy,
|
||||
m_testFailurePolicy,
|
||||
m_targetOutputCapture,
|
||||
testTargetTimeout,
|
||||
sequenceTimeout,
|
||||
AZStd::ref(testRunCompleteHandler));
|
||||
const auto draftedTestRunDuration = draftedTestRunTimer.GetElapsedMs();
|
||||
if (!draftedTestTargets.empty())
|
||||
{
|
||||
// Run the drafted test targets and collect the test run results
|
||||
gatherTestRunData(draftedTestTargets, instrumentedTestRun, draftedTestRunData);
|
||||
}
|
||||
|
||||
// Generate the sequence report for the client
|
||||
const auto sequenceReport = Client::SafeImpactAnalysisSequenceReport(
|
||||
m_maxConcurrency,
|
||||
testTargetTimeout,
|
||||
globalTimeout,
|
||||
GenerateSafeImpactAnalysisSequencePolicyState(testPrioritizationPolicy),
|
||||
m_suiteFilter,
|
||||
selectedTests,
|
||||
discardedTests,
|
||||
draftedTests,
|
||||
GenerateTestRunReport(selectedResult, selectedTestRunTimer.GetStartTimePointRelative(sequenceTimer), selectedTestRunDuration, selectedTestJobs),
|
||||
GenerateTestRunReport(discardedResult, discardedTestRunTimer.GetStartTimePointRelative(sequenceTimer), discardedTestRunDuration, discardedTestJobs),
|
||||
GenerateTestRunReport(draftedResult, draftedTestRunTimer.GetStartTimePointRelative(sequenceTimer), draftedTestRunDuration, draftedTestJobs));
|
||||
GenerateTestRunReport(
|
||||
selectedTestRunData.m_result,
|
||||
selectedTestRunData.m_relativeStartTime,
|
||||
selectedTestRunData.m_duration,
|
||||
selectedTestRunData.m_jobs),
|
||||
GenerateTestRunReport(
|
||||
discardedTestRunData.m_result,
|
||||
discardedTestRunData.m_relativeStartTime,
|
||||
discardedTestRunData.m_duration,
|
||||
discardedTestRunData.m_jobs),
|
||||
GenerateTestRunReport(
|
||||
draftedTestRunData.m_result,
|
||||
draftedTestRunData.m_relativeStartTime,
|
||||
draftedTestRunData.m_duration,
|
||||
draftedTestRunData.m_jobs));
|
||||
|
||||
// Inform the client that the sequence has ended
|
||||
if (testSequenceEndCallback.has_value())
|
||||
@@ -747,15 +881,15 @@ namespace TestImpact
|
||||
(*testSequenceEndCallback)(sequenceReport);
|
||||
}
|
||||
|
||||
UpdateAndSerializeDynamicDependencyMap(ConcatenateVectors(selectedTestJobs, draftedTestJobs));
|
||||
UpdateAndSerializeDynamicDependencyMap(ConcatenateVectors(selectedTestRunData.m_jobs, draftedTestRunData.m_jobs));
|
||||
return sequenceReport;
|
||||
}
|
||||
|
||||
Client::SequenceReport Runtime::SeededTestSequence(
|
||||
Client::SeedSequenceReport Runtime::SeededTestSequence(
|
||||
AZStd::optional<AZStd::chrono::milliseconds> testTargetTimeout,
|
||||
AZStd::optional<AZStd::chrono::milliseconds> globalTimeout,
|
||||
AZStd::optional<TestSequenceStartCallback> testSequenceStartCallback,
|
||||
AZStd::optional<TestSequenceCompleteCallback<Client::SequenceReport>> testSequenceEndCallback,
|
||||
AZStd::optional<TestSequenceCompleteCallback<Client::SeedSequenceReport>> testSequenceEndCallback,
|
||||
AZStd::optional<TestRunCompleteCallback> testCompleteCallback)
|
||||
{
|
||||
const Timer sequenceTimer;
|
||||
@@ -799,7 +933,11 @@ namespace TestImpact
|
||||
const auto testRunDuration = testRunTimer.GetElapsedMs();
|
||||
|
||||
// Generate the sequence report for the client
|
||||
const auto sequenceReport = Client::SequenceReport(
|
||||
const auto sequenceReport = Client::SeedSequenceReport(
|
||||
m_maxConcurrency,
|
||||
testTargetTimeout,
|
||||
globalTimeout,
|
||||
GenerateSequencePolicyState(),
|
||||
m_suiteFilter,
|
||||
selectedTests,
|
||||
GenerateTestRunReport(result, testRunTimer.GetStartTimePointRelative(sequenceTimer), testRunDuration, testJobs));
|
||||
|
||||
@@ -6,7 +6,7 @@
|
||||
*
|
||||
*/
|
||||
|
||||
#include <TestImpactFramework/TestImpactFileUtils.h>
|
||||
#include <TestImpactFramework/TestImpactUtils.h>
|
||||
#include <TestImpactFramework/TestImpactRuntimeException.h>
|
||||
|
||||
#include <TestImpactRuntimeUtils.h>
|
||||
|
||||
@@ -38,34 +38,44 @@ namespace TestImpact
|
||||
//! Extracts the name information from the specified test targets.
|
||||
AZStd::vector<AZStd::string> ExtractTestTargetNames(const AZStd::vector<const TestTarget*>& testTargets);
|
||||
|
||||
//! Generates a test run failure report from the specified test engine job information.
|
||||
//! Generates the test suites from the specified test engine job information.
|
||||
//! @tparam TestJob The test engine job type.
|
||||
template<typename TestJob>
|
||||
AZStd::vector<Client::TestCaseFailure> GenerateTestCaseFailures(const TestJob& testJob)
|
||||
AZStd::vector<Client::Test> GenerateClientTests(const TestJob& testJob)
|
||||
{
|
||||
AZStd::vector<Client::TestCaseFailure> testCaseFailures;
|
||||
AZStd::vector<Client::Test> tests;
|
||||
|
||||
if (testJob.GetTestRun().has_value())
|
||||
{
|
||||
for (const auto& testSuite : testJob.GetTestRun()->GetTestSuites())
|
||||
{
|
||||
AZStd::vector<Client::TestFailure> testFailures;
|
||||
for (const auto& testCase : testSuite.m_tests)
|
||||
{
|
||||
if (testCase.m_result.value_or(TestRunResult::Passed) == TestRunResult::Failed)
|
||||
auto result = Client::TestResult::NotRun;
|
||||
if (testCase.m_result.has_value())
|
||||
{
|
||||
testFailures.push_back(Client::TestFailure(testCase.m_name, "No error message retrieved"));
|
||||
if (testCase.m_result.value() == TestRunResult::Passed)
|
||||
{
|
||||
result = Client::TestResult::Passed;
|
||||
}
|
||||
else if (testCase.m_result.value() == TestRunResult::Failed)
|
||||
{
|
||||
result = Client::TestResult::Failed;
|
||||
}
|
||||
else
|
||||
{
|
||||
throw RuntimeException(AZStd::string::format(
|
||||
"Unexpected test run result: %u", aznumeric_cast<AZ::u32>(testCase.m_result.value())));
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if (!testFailures.empty())
|
||||
{
|
||||
testCaseFailures.push_back(Client::TestCaseFailure(testSuite.m_name, AZStd::move(testFailures)));
|
||||
|
||||
const auto name = AZStd::string::format("%s.%s", testSuite.m_name.c_str(), testCase.m_name.c_str());
|
||||
tests.push_back(Client::Test(name, result));
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return testCaseFailures;
|
||||
return tests;
|
||||
}
|
||||
|
||||
template<typename TestJob>
|
||||
@@ -75,11 +85,11 @@ namespace TestImpact
|
||||
AZStd::chrono::milliseconds duration,
|
||||
const AZStd::vector<TestJob>& testJobs)
|
||||
{
|
||||
AZStd::vector<Client::TestRun> passingTests;
|
||||
AZStd::vector<Client::TestRunWithTestFailures> failingTests;
|
||||
AZStd::vector<Client::TestRun> executionFailureTests;
|
||||
AZStd::vector<Client::TestRun> timedOutTests;
|
||||
AZStd::vector<Client::TestRun> unexecutedTests;
|
||||
AZStd::vector<Client::PassingTestRun> passingTests;
|
||||
AZStd::vector<Client::FailingTestRun> failingTests;
|
||||
AZStd::vector<Client::TestRunWithExecutionFailure> executionFailureTests;
|
||||
AZStd::vector<Client::TimedOutTestRun> timedOutTests;
|
||||
AZStd::vector<Client::UnexecutedTestRun> unexecutedTests;
|
||||
|
||||
for (const auto& testJob : testJobs)
|
||||
{
|
||||
@@ -88,7 +98,7 @@ namespace TestImpact
|
||||
AZStd::chrono::high_resolution_clock::time_point() +
|
||||
AZStd::chrono::duration_cast<AZStd::chrono::milliseconds>(testJob.GetStartTime() - startTime);
|
||||
|
||||
Client::TestRun clientTestRun(
|
||||
Client::TestRunBase clientTestRun(
|
||||
testJob.GetTestTarget()->GetName(), testJob.GetCommandString(), relativeStartTime, testJob.GetDuration(),
|
||||
testJob.GetTestResult());
|
||||
|
||||
@@ -96,27 +106,27 @@ namespace TestImpact
|
||||
{
|
||||
case Client::TestRunResult::FailedToExecute:
|
||||
{
|
||||
executionFailureTests.push_back(clientTestRun);
|
||||
executionFailureTests.emplace_back(AZStd::move(clientTestRun));
|
||||
break;
|
||||
}
|
||||
case Client::TestRunResult::NotRun:
|
||||
{
|
||||
unexecutedTests.push_back(clientTestRun);
|
||||
unexecutedTests.emplace_back(AZStd::move(clientTestRun));
|
||||
break;
|
||||
}
|
||||
case Client::TestRunResult::Timeout:
|
||||
{
|
||||
timedOutTests.push_back(clientTestRun);
|
||||
timedOutTests.emplace_back(AZStd::move(clientTestRun));
|
||||
break;
|
||||
}
|
||||
case Client::TestRunResult::AllTestsPass:
|
||||
{
|
||||
passingTests.push_back(clientTestRun);
|
||||
passingTests.emplace_back(AZStd::move(clientTestRun), GenerateClientTests(testJob));
|
||||
break;
|
||||
}
|
||||
case Client::TestRunResult::TestFailures:
|
||||
{
|
||||
failingTests.emplace_back(AZStd::move(clientTestRun), GenerateTestCaseFailures(testJob));
|
||||
failingTests.emplace_back(AZStd::move(clientTestRun), GenerateClientTests(testJob));
|
||||
break;
|
||||
}
|
||||
default:
|
||||
|
||||
@@ -0,0 +1,244 @@
|
||||
/*
|
||||
* Copyright (c) Contributors to the Open 3D Engine Project.
|
||||
* For complete copyright and license terms please see the LICENSE at the root of this distribution.
|
||||
*
|
||||
* SPDX-License-Identifier: Apache-2.0 OR MIT
|
||||
*
|
||||
*/
|
||||
|
||||
#include <TestImpactFramework/TestImpactException.h>
|
||||
#include <TestImpactFramework/TestImpactUtils.h>
|
||||
|
||||
#include <AzCore/std/functional.h>
|
||||
|
||||
namespace TestImpact
|
||||
{
|
||||
//! Delete the files that match the pattern from the specified directory.
|
||||
//! @param path The path to the directory to pattern match the files for deletion.
|
||||
//! @param pattern The pattern to match files for deletion.
|
||||
size_t DeleteFiles(const RepoPath& path, const AZStd::string& pattern)
|
||||
{
|
||||
size_t numFilesDeleted = 0;
|
||||
|
||||
AZ::IO::SystemFile::FindFiles(
|
||||
AZStd::string::format("%s/%s", path.c_str(), pattern.c_str()).c_str(),
|
||||
[&path, &numFilesDeleted](const char* file, bool isFile)
|
||||
{
|
||||
if (isFile)
|
||||
{
|
||||
AZ::IO::SystemFile::Delete(AZStd::string::format("%s/%s", path.c_str(), file).c_str());
|
||||
numFilesDeleted++;
|
||||
}
|
||||
|
||||
return true;
|
||||
});
|
||||
|
||||
return numFilesDeleted;
|
||||
}
|
||||
|
||||
//! Deletes the specified file.
|
||||
void DeleteFile(const RepoPath& file)
|
||||
{
|
||||
DeleteFiles(file.ParentPath(), file.Filename().Native());
|
||||
}
|
||||
|
||||
//! User-friendly names for the test suite types.
|
||||
AZStd::string SuiteTypeAsString(SuiteType suiteType)
|
||||
{
|
||||
switch (suiteType)
|
||||
{
|
||||
case SuiteType::Main:
|
||||
return "main";
|
||||
case SuiteType::Periodic:
|
||||
return "periodic";
|
||||
case SuiteType::Sandbox:
|
||||
return "sandbox";
|
||||
default:
|
||||
throw(Exception("Unexpected suite type"));
|
||||
}
|
||||
}
|
||||
|
||||
AZStd::string SequenceReportTypeAsString(Client::SequenceReportType type)
|
||||
{
|
||||
switch (type)
|
||||
{
|
||||
case Client::SequenceReportType::RegularSequence:
|
||||
return "regular";
|
||||
case Client::SequenceReportType::SeedSequence:
|
||||
return "seed";
|
||||
case Client::SequenceReportType::ImpactAnalysisSequence:
|
||||
return "impact_analysis";
|
||||
case Client::SequenceReportType::SafeImpactAnalysisSequence:
|
||||
return "safe_impact_analysis";
|
||||
default:
|
||||
throw(Exception(AZStd::string::format("Unexpected sequence report type: %u", aznumeric_cast<AZ::u32>(type))));
|
||||
}
|
||||
}
|
||||
|
||||
AZStd::string TestSequenceResultAsString(TestSequenceResult result)
|
||||
{
|
||||
switch (result)
|
||||
{
|
||||
case TestSequenceResult::Failure:
|
||||
return "failure";
|
||||
case TestSequenceResult::Success:
|
||||
return "success";
|
||||
case TestSequenceResult::Timeout:
|
||||
return "timeout";
|
||||
default:
|
||||
throw(Exception(AZStd::string::format("Unexpected test sequence result: %u", aznumeric_cast<AZ::u32>(result))));
|
||||
}
|
||||
}
|
||||
|
||||
AZStd::string TestRunResultAsString(Client::TestRunResult result)
|
||||
{
|
||||
switch (result)
|
||||
{
|
||||
case Client::TestRunResult::AllTestsPass:
|
||||
return "all_tests_pass";
|
||||
case Client::TestRunResult::FailedToExecute:
|
||||
return "failed_to_execute";
|
||||
case Client::TestRunResult::NotRun:
|
||||
return "not_run";
|
||||
case Client::TestRunResult::TestFailures:
|
||||
return "test_failures";
|
||||
case Client::TestRunResult::Timeout:
|
||||
return "timeout";
|
||||
default:
|
||||
throw(Exception(AZStd::string::format("Unexpected test run result: %u", aznumeric_cast<AZ::u32>(result))));
|
||||
}
|
||||
}
|
||||
|
||||
AZStd::string ExecutionFailurePolicyAsString(Policy::ExecutionFailure executionFailurePolicy)
|
||||
{
|
||||
switch (executionFailurePolicy)
|
||||
{
|
||||
case Policy::ExecutionFailure::Abort:
|
||||
return "abort";
|
||||
case Policy::ExecutionFailure::Continue:
|
||||
return "continue";
|
||||
case Policy::ExecutionFailure::Ignore:
|
||||
return "ignore";
|
||||
default:
|
||||
throw(Exception(
|
||||
AZStd::string::format("Unexpected execution failure policy: %u", aznumeric_cast<AZ::u32>(executionFailurePolicy))));
|
||||
}
|
||||
}
|
||||
|
||||
AZStd::string FailedTestCoveragePolicyAsString(Policy::FailedTestCoverage failedTestCoveragePolicy)
|
||||
{
|
||||
switch (failedTestCoveragePolicy)
|
||||
{
|
||||
case Policy::FailedTestCoverage::Discard:
|
||||
return "discard";
|
||||
case Policy::FailedTestCoverage::Keep:
|
||||
return "keep";
|
||||
default:
|
||||
throw(Exception(
|
||||
AZStd::string::format("Unexpected failed test coverage policy: %u", aznumeric_cast<AZ::u32>(failedTestCoveragePolicy))));
|
||||
}
|
||||
}
|
||||
|
||||
AZStd::string TestPrioritizationPolicyAsString(Policy::TestPrioritization testPrioritizationPolicy)
|
||||
{
|
||||
switch (testPrioritizationPolicy)
|
||||
{
|
||||
case Policy::TestPrioritization::DependencyLocality:
|
||||
return "dependency_locality";
|
||||
case Policy::TestPrioritization::None:
|
||||
return "none";
|
||||
default:
|
||||
throw(Exception(
|
||||
AZStd::string::format("Unexpected test prioritization policy: %u", aznumeric_cast<AZ::u32>(testPrioritizationPolicy))));
|
||||
}
|
||||
}
|
||||
|
||||
AZStd::string TestFailurePolicyAsString(Policy::TestFailure testFailurePolicy)
|
||||
{
|
||||
switch (testFailurePolicy)
|
||||
{
|
||||
case Policy::TestFailure::Abort:
|
||||
return "abort";
|
||||
case Policy::TestFailure::Continue:
|
||||
return "continue";
|
||||
default:
|
||||
throw(
|
||||
Exception(AZStd::string::format("Unexpected test failure policy: %u", aznumeric_cast<AZ::u32>(testFailurePolicy))));
|
||||
}
|
||||
}
|
||||
|
||||
AZStd::string IntegrityFailurePolicyAsString(Policy::IntegrityFailure integrityFailurePolicy)
|
||||
{
|
||||
switch (integrityFailurePolicy)
|
||||
{
|
||||
case Policy::IntegrityFailure::Abort:
|
||||
return "abort";
|
||||
case Policy::IntegrityFailure::Continue:
|
||||
return "continue";
|
||||
default:
|
||||
throw(Exception(
|
||||
AZStd::string::format("Unexpected integration failure policy: %u", aznumeric_cast<AZ::u32>(integrityFailurePolicy))));
|
||||
}
|
||||
}
|
||||
|
||||
AZStd::string DynamicDependencyMapPolicyAsString(Policy::DynamicDependencyMap dynamicDependencyMapPolicy)
|
||||
{
|
||||
switch (dynamicDependencyMapPolicy)
|
||||
{
|
||||
case Policy::DynamicDependencyMap::Discard:
|
||||
return "discard";
|
||||
case Policy::DynamicDependencyMap::Update:
|
||||
return "update";
|
||||
default:
|
||||
throw(Exception(AZStd::string::format(
|
||||
"Unexpected dynamic dependency map policy: %u", aznumeric_cast<AZ::u32>(dynamicDependencyMapPolicy))));
|
||||
}
|
||||
}
|
||||
|
||||
AZStd::string TestShardingPolicyAsString(Policy::TestSharding testShardingPolicy)
|
||||
{
|
||||
switch (testShardingPolicy)
|
||||
{
|
||||
case Policy::TestSharding::Always:
|
||||
return "always";
|
||||
case Policy::TestSharding::Never:
|
||||
return "never";
|
||||
default:
|
||||
throw(Exception(
|
||||
AZStd::string::format("Unexpected test sharding policy: %u", aznumeric_cast<AZ::u32>(testShardingPolicy))));
|
||||
}
|
||||
}
|
||||
|
||||
AZStd::string TargetOutputCapturePolicyAsString(Policy::TargetOutputCapture targetOutputCapturePolicy)
|
||||
{
|
||||
switch (targetOutputCapturePolicy)
|
||||
{
|
||||
case Policy::TargetOutputCapture::File:
|
||||
return "file";
|
||||
case Policy::TargetOutputCapture::None:
|
||||
return "none";
|
||||
case Policy::TargetOutputCapture::StdOut:
|
||||
return "stdout";
|
||||
case Policy::TargetOutputCapture::StdOutAndFile:
|
||||
return "stdout_file";
|
||||
default:
|
||||
throw(Exception(
|
||||
AZStd::string::format("Unexpected target output capture policy: %u", aznumeric_cast<AZ::u32>(targetOutputCapturePolicy))));
|
||||
}
|
||||
}
|
||||
|
||||
AZStd::string ClientTestResultAsString(Client::TestResult result)
|
||||
{
|
||||
switch (result)
|
||||
{
|
||||
case Client::TestResult::Failed:
|
||||
return "failed";
|
||||
case Client::TestResult::NotRun:
|
||||
return "not_run";
|
||||
case Client::TestResult::Passed:
|
||||
return "passed";
|
||||
default:
|
||||
throw(Exception(AZStd::string::format("Unexpected client test case result: %u", aznumeric_cast<AZ::u32>(result))));
|
||||
}
|
||||
}
|
||||
} // namespace TestImpact
|
||||
Reference in New Issue
Block a user