birkeh/o3de
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Add drafting of failed tests

Browse Source
This commit is contained in:
jonawals
2021-06-07 08:52:20 +01:00
parent b00f569090
commit 4ed0c7b1d8
8 changed files with 119 additions and 62 deletions
Download Patch File Download Diff File Expand all files Collapse all files
Code/Tools/TestImpactFramework/Runtime/Code/Source/Dependency/TestImpactDynamicDependencyMap.cpp
+33 -15
View File
@@ -56,7 +56,7 @@ namespace TestImpact
for (const auto& target : m_testTargets.GetTargets())
{
mapBuildTargetSources(&target);
m_testTargetSourceCoverageCount[&target] = 0;
m_testTargetSourceCoverage[&target] = {};
}
}
@@ -144,18 +144,15 @@ namespace TestImpact
sourceCoverage.GetPath().c_str()).c_str());
auto [sourceDependencyIt, inserted] = m_sourceDependencyMap.insert(sourceCoverage.GetPath().String());
auto& [key, sourceDependency] = *sourceDependencyIt;
auto& [source, sourceDependency] = *sourceDependencyIt;
// Knock down the source coverage count for the test targets and clear any existing coverage for the delta
// Remove the source from the test target covering sources map and clear any existing coverage for the delta
for (const auto& testTarget : sourceDependency.m_coveringTestTargets)
{
if (auto coveringTestTargetIt = m_testTargetSourceCoverageCount.find(testTarget);
coveringTestTargetIt != m_testTargetSourceCoverageCount.end())
if (auto coveringTestTargetIt = m_testTargetSourceCoverage.find(testTarget);
coveringTestTargetIt != m_testTargetSourceCoverage.end())
{
if (coveringTestTargetIt->second > 0)
{
coveringTestTargetIt->second--;
}
coveringTestTargetIt->second.erase(source);
}
}
sourceDependency.m_coveringTestTargets.clear();
@@ -169,8 +166,8 @@ namespace TestImpact
// Source to covering test target mapping
sourceDependency.m_coveringTestTargets.insert(testTarget);
// Test target covering sources count
m_testTargetSourceCoverageCount[testTarget]++;
// Add the source to the test target covering sources map
m_testTargetSourceCoverage[testTarget].insert(source);
// Build target to covering test target mapping
for (const auto& parentTarget : sourceDependency.m_parentTargets)
@@ -429,12 +426,33 @@ namespace TestImpact
return ChangeDependencyList(AZStd::move(createDependencies), AZStd::move(updateDependencies), AZStd::move(deleteDependencies));
}
void DynamicDependencyMap::RemoveTestTargetFromSourceCoverage(const TestTarget* testTarget)
{
if (const auto& it = m_testTargetSourceCoverage.find(testTarget);
it != m_testTargetSourceCoverage.end())
{
for (const auto& source : it->second)
{
const auto sourceDependency = m_sourceDependencyMap.find(source);
AZ_TestImpact_Eval(
sourceDependency != m_sourceDependencyMap.end(),
DependencyException,
AZStd::string::format("Test target '%s' has covering source '%s' yet cannot be found in the dependency map",
testTarget->GetName().c_str(), source.c_str()));
sourceDependency->second.m_coveringTestTargets.erase(testTarget);
}
m_testTargetSourceCoverage.erase(testTarget);
}
}
AZStd::vector<const TestTarget*> DynamicDependencyMap::GetCoveringTests() const
{
AZStd::vector<const TestTarget*> covering;
for (const auto& [testTarget, coveringSources] : m_testTargetSourceCoverageCount)
for (const auto& [testTarget, coveringSources] : m_testTargetSourceCoverage)
{
if (coveringSources > 0)
if (!coveringSources.empty())
{
covering.push_back(testTarget);
}
@@ -446,9 +464,9 @@ namespace TestImpact
AZStd::vector<const TestTarget*> DynamicDependencyMap::GetNotCoveringTests() const
{
AZStd::vector<const TestTarget*> notCovering;
for(const auto& [testTarget, coveringSources] : m_testTargetSourceCoverageCount)
for(const auto& [testTarget, coveringSources] : m_testTargetSourceCoverage)
{
if(coveringSources == 0)
if (coveringSources.empty())
{
notCovering.push_back(testTarget);
}
Code/Tools/TestImpactFramework/Runtime/Code/Source/Dependency/TestImpactDynamicDependencyMap.h
+6 -3
View File
@@ -100,6 +100,9 @@ namespace TestImpact
//! @returns The change list as resolved to the appropriate source dependencies.
[[nodiscard]] ChangeDependencyList ApplyAndResoveChangeList(const ChangeList& changeList);
//! Removes the specified test target from all source coverage.
void RemoveTestTargetFromSourceCoverage(const TestTarget* testTarget);
//! Returns the test targets that cover one or more sources in the repository.
AZStd::vector<const TestTarget*> GetCoveringTests() const;
@@ -120,13 +123,13 @@ namespace TestImpact
//! The dependency map of sources to their parent build targets and covering test targets.
AZStd::unordered_map<AZStd::string, DependencyData> m_sourceDependencyMap;
//! Map of all test targets and the sources they cover.
AZStd::unordered_map<const TestTarget*, AZStd::unordered_set<AZStd::string>> m_testTargetSourceCoverage;
//! The map of build targets and their covering test targets.
AZStd::unordered_map<const BuildTarget*, AZStd::unordered_set<const TestTarget*>> m_buildTargetCoverage;
//! Mapping of autogen input sources to their generated output sources.
AZStd::unordered_map<AZStd::string, AZStd::vector<AZStd::string>> m_autogenInputToOutputMap;
//! Number of sources that each test target in the repository covers.
AZStd::unordered_map<const TestTarget*, size_t> m_testTargetSourceCoverageCount;
};
} // namespace TestImpact
Code/Tools/TestImpactFramework/Runtime/Code/Source/TestImpactRuntime.cpp
+39 -12
View File
@@ -70,11 +70,20 @@ namespace TestImpact
};
}
template<typename T>
AZStd::vector<T> ConcatenateVectors(const AZStd::vector<T>& v1, const AZStd::vector<T>& v2)
{
AZStd::vector<T> result;
result.reserve(v1.size() + v2.size());
result.insert(result.end(), v1.begin(), v1.end());
result.insert(result.end(), v2.begin(), v2.end());
return result;
}
Runtime::Runtime(
RuntimeConfig&& config,
SuiteType suiteFilter,
Policy::ExecutionFailure executionFailurePolicy,
Policy::ExecutionFailureDrafting executionFailureDraftingPolicy,
Policy::FailedTestCoverage failedTestCoveragePolicy,
Policy::TestFailure testFailurePolicy,
Policy::IntegrityFailure integrationFailurePolicy,
Policy::TestSharding testShardingPolicy,
@@ -83,7 +92,7 @@ namespace TestImpact
: m_config(AZStd::move(config))
, m_suiteFilter(suiteFilter)
, m_executionFailurePolicy(executionFailurePolicy)
, m_executionFailureDraftingPolicy(executionFailureDraftingPolicy)
, m_failedTestCoveragePolicy(failedTestCoveragePolicy)
, m_testFailurePolicy(testFailurePolicy)
, m_integrationFailurePolicy(integrationFailurePolicy)
, m_testShardingPolicy(testShardingPolicy)
@@ -145,7 +154,7 @@ namespace TestImpact
{
AZ_Printf("TestImpactRuntime",
AZStd::string::format(
"No test impact analysis data found for suite '%s' at %s", GetSuiteTypeName(m_suiteFilter).c_str(), m_sparTIAFile.c_str()).c_str());
"No test impact analysis data found for suite '%s' at %s\n", GetSuiteTypeName(m_suiteFilter).c_str(), m_sparTIAFile.c_str()).c_str());
}
}
@@ -249,14 +258,28 @@ namespace TestImpact
m_dynamicDependencyMap->ClearAllSourceCoverage();
}
void Runtime::UpdateAndSerializeDynamicDependencyMap(const SourceCoveringTestsList& sourceCoverageTestsList)
void Runtime::UpdateAndSerializeDynamicDependencyMap(const AZStd::vector<TestEngineInstrumentedRun>& jobs)
{
const auto sourceCoverageTestsList = CreateSourceCoveringTestFromTestCoverages(jobs, m_config.m_repo.m_root);
if (!sourceCoverageTestsList.GetNumSources())
{
return;
}
m_dynamicDependencyMap->ReplaceSourceCoverage(sourceCoverageTestsList);
if (m_failedTestCoveragePolicy == Policy::FailedTestCoverage::Remove)
{
for (const auto& job : jobs)
{
if (job.GetTestResult() != Client::TestRunResult::AllTestsPass ||
!job.GetTestCoverge().has_value())
{
m_dynamicDependencyMap->RemoveTestTargetFromSourceCoverage(job.GetTestTarget());
}
}
}
const auto sparTIA = m_dynamicDependencyMap->ExportSourceCoverage();
const auto sparTIAData = SerializeSourceCoveringTestsList(sparTIA);
WriteFileContents<RuntimeException>(sparTIAData, m_sparTIAFile);
@@ -323,11 +346,12 @@ namespace TestImpact
AZStd::optional<TestRunCompleteCallback> testCompleteCallback)
{
Timer timer;
AZStd::vector<const TestTarget*> draftedTestTargets;
AZStd::vector<const TestTarget*> draftedTestTargets = m_dynamicDependencyMap->GetNotCoveringTests();
auto [selectedTestTargets, discardedTestTargets] = SelectCoveringTestTargetsAndUpdateEnumerationCache(changeList, testPrioritizationPolicy);
auto [includedSelectedTestTargets, excludedSelectedTestTargets] = SelectTestTargetsByExcludeList(selectedTestTargets);
AZStd::vector<const TestTarget*> testTargetsToRun = ConcatenateVectors(includedSelectedTestTargets, draftedTestTargets);
if (testSequenceStartCallback.has_value())
{
(*testSequenceStartCallback)(
@@ -336,10 +360,11 @@ namespace TestImpact
ExtractTestTargetNames(draftedTestTargets));
}
if (dynamicDependencyMapPolicy == Policy::DynamicDependencyMap::Update)
{
const auto [result, testJobs] = m_testEngine->InstrumentedRun(
includedSelectedTestTargets,
testTargetsToRun,
m_testShardingPolicy,
m_executionFailurePolicy,
Policy::IntegrityFailure::Continue,
@@ -349,7 +374,7 @@ namespace TestImpact
globalTimeout,
TestRunCompleteCallbackHandler(testCompleteCallback));
UpdateAndSerializeDynamicDependencyMap(CreateSourceCoveringTestFromTestCoverages(testJobs, m_config.m_repo.m_root));
UpdateAndSerializeDynamicDependencyMap(testJobs);
if (testSequenceEndCallback.has_value())
{
@@ -361,7 +386,7 @@ namespace TestImpact
else
{
const auto [result, testJobs] = m_testEngine->RegularRun(
includedSelectedTestTargets,
testTargetsToRun,
m_testShardingPolicy,
m_executionFailurePolicy,
m_testFailurePolicy,
@@ -389,12 +414,13 @@ namespace TestImpact
AZStd::optional<TestRunCompleteCallback> testCompleteCallback)
{
Timer timer;
AZStd::vector<const TestTarget*> draftedTestTargets;
AZStd::vector<const TestTarget*> draftedTestTargets = m_dynamicDependencyMap->GetNotCoveringTests();
auto [selectedTestTargets, discardedTestTargets] = SelectCoveringTestTargetsAndUpdateEnumerationCache(changeList, testPrioritizationPolicy);
auto [includedSelectedTestTargets, excludedSelectedTestTargets] = SelectTestTargetsByExcludeList(selectedTestTargets);
auto [includedDiscardedTestTargets, excludedDiscardedTestTargets] = SelectTestTargetsByExcludeList(discardedTestTargets);
AZStd::vector<const TestTarget*> testTargetsToRun = ConcatenateVectors(includedSelectedTestTargets, draftedTestTargets);
if (testSequenceStartCallback.has_value())
{
(*testSequenceStartCallback)(
@@ -403,9 +429,10 @@ namespace TestImpact
ExtractTestTargetNames(draftedTestTargets));
}
// Impact analysis run of the selected test targets
const auto [selectedResult, selectedTestJobs] = m_testEngine->InstrumentedRun(
includedSelectedTestTargets,
testTargetsToRun,
m_testShardingPolicy,
m_executionFailurePolicy,
Policy::IntegrityFailure::Continue,
@@ -433,7 +460,7 @@ namespace TestImpact
globalTimeout,
TestRunCompleteCallbackHandler(testCompleteCallback));
UpdateAndSerializeDynamicDependencyMap(CreateSourceCoveringTestFromTestCoverages(selectedTestJobs, m_config.m_repo.m_root));
UpdateAndSerializeDynamicDependencyMap(selectedTestJobs);
if (testSequenceEndCallback.has_value())
{
@@ -486,7 +513,7 @@ namespace TestImpact
TestRunCompleteCallbackHandler(testCompleteCallback));
ClearDynamicDependencyMapAndRemoveExistingFile();
UpdateAndSerializeDynamicDependencyMap(CreateSourceCoveringTestFromTestCoverages(testJobs, m_config.m_repo.m_root));
UpdateAndSerializeDynamicDependencyMap(testJobs);
if (testSequenceEndCallback.has_value())
{