From 042a025f619a9cc3c3a9c886e4e22d80f04e40f8 Mon Sep 17 00:00:00 2001 From: jonawals Date: Thu, 27 May 2021 14:06:59 +0100 Subject: [PATCH] Private runtime implementation --- .../Source/TestImpactChangeListSerializer.cpp | 96 ++++ .../Source/TestImpactClientFailureReport.cpp | 124 +++++ .../Code/Source/TestImpactClientTestRun.cpp | 40 ++ .../Source/TestImpactClientTestSelection.cpp | 56 ++ .../Code/Source/TestImpactRepoPath.cpp | 104 ++++ .../Runtime/Code/Source/TestImpactRuntime.cpp | 486 ++++++++++++++++++ .../Code/Source/TestImpactRuntimeUtils.cpp | 133 +++++ .../Code/Source/TestImpactRuntimeUtils.h | 134 +++++ 8 files changed, 1173 insertions(+) create mode 100644 Code/Tools/TestImpactFramework/Runtime/Code/Source/TestImpactChangeListSerializer.cpp create mode 100644 Code/Tools/TestImpactFramework/Runtime/Code/Source/TestImpactClientFailureReport.cpp create mode 100644 Code/Tools/TestImpactFramework/Runtime/Code/Source/TestImpactClientTestRun.cpp create mode 100644 Code/Tools/TestImpactFramework/Runtime/Code/Source/TestImpactClientTestSelection.cpp create mode 100644 Code/Tools/TestImpactFramework/Runtime/Code/Source/TestImpactRepoPath.cpp create mode 100644 Code/Tools/TestImpactFramework/Runtime/Code/Source/TestImpactRuntime.cpp create mode 100644 Code/Tools/TestImpactFramework/Runtime/Code/Source/TestImpactRuntimeUtils.cpp create mode 100644 Code/Tools/TestImpactFramework/Runtime/Code/Source/TestImpactRuntimeUtils.h diff --git a/Code/Tools/TestImpactFramework/Runtime/Code/Source/TestImpactChangeListSerializer.cpp b/Code/Tools/TestImpactFramework/Runtime/Code/Source/TestImpactChangeListSerializer.cpp new file mode 100644 index 0000000000..ef157f1848 --- /dev/null +++ b/Code/Tools/TestImpactFramework/Runtime/Code/Source/TestImpactChangeListSerializer.cpp @@ -0,0 +1,96 @@ +/* + * All or portions of this file Copyright (c) Amazon.com, Inc. or its affiliates or + * its licensors. + * + * For complete copyright and license terms please see the LICENSE at the root of this + * distribution (the "License"). All use of this software is governed by the License, + * or, if provided, by the license below or the license accompanying this file. Do not + * remove or modify any license notices. This file is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * + */ + +#include +#include + +#include +#include +#include +#include + +namespace TestImpact +{ + namespace ChangeListFields + { + // Keys for pertinent JSON node and attribute names + constexpr const char* Keys[] = + { + "createdFiles", + "updatedFiles", + "deletedFiles" + }; + + enum + { + CreateKey, + UpdateKey, + DeleteKey + }; + } // namespace + + AZStd::string SerializeChangeList(const ChangeList& changeList) + { + rapidjson::StringBuffer stringBuffer; + rapidjson::PrettyWriter writer(stringBuffer); + + const auto serializeFileList = [&writer](const char* key, const AZStd::vector& fileList) + { + writer.Key(key); + writer.StartArray(); + + for (const auto& file : fileList) + { + writer.String(file.c_str()); + } + + writer.EndArray(); + }; + + writer.StartObject(); + serializeFileList(ChangeListFields::Keys[ChangeListFields::CreateKey], changeList.m_createdFiles); + serializeFileList(ChangeListFields::Keys[ChangeListFields::UpdateKey], changeList.m_updatedFiles); + serializeFileList(ChangeListFields::Keys[ChangeListFields::DeleteKey], changeList.m_deletedFiles); + writer.EndObject(); + + return stringBuffer.GetString(); + } + + ChangeList DeserializeChangeList(const AZStd::string& changeListString) + { + ChangeList changeList; + rapidjson::Document doc; + + if (doc.Parse<0>(changeListString.c_str()).HasParseError()) + { + throw ChangeListException("Could not parse change list data"); + } + + const auto deserializeFileList = [&doc](const char* key) + { + AZStd::vector fileList; + + for (const auto& file : doc[key].GetArray()) + { + fileList.push_back(file.GetString()); + } + + return fileList; + }; + + changeList.m_createdFiles = deserializeFileList(ChangeListFields::Keys[ChangeListFields::CreateKey]); + changeList.m_updatedFiles = deserializeFileList(ChangeListFields::Keys[ChangeListFields::UpdateKey]); + changeList.m_deletedFiles = deserializeFileList(ChangeListFields::Keys[ChangeListFields::DeleteKey]); + + return changeList; + } +} // namespace TestImpact diff --git a/Code/Tools/TestImpactFramework/Runtime/Code/Source/TestImpactClientFailureReport.cpp b/Code/Tools/TestImpactFramework/Runtime/Code/Source/TestImpactClientFailureReport.cpp new file mode 100644 index 0000000000..f39d8ee426 --- /dev/null +++ b/Code/Tools/TestImpactFramework/Runtime/Code/Source/TestImpactClientFailureReport.cpp @@ -0,0 +1,124 @@ +/* + * All or portions of this file Copyright (c) Amazon.com, Inc. or its affiliates or + * its licensors. + * + * For complete copyright and license terms please see the LICENSE at the root of this + * distribution (the "License"). All use of this software is governed by the License, + * or, if provided, by the license below or the license accompanying this file. Do not + * remove or modify any license notices. This file is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * + */ + +#include + +namespace TestImpact +{ + namespace Client + { + TargetFailure::TargetFailure(const AZStd::string& targetName) + : m_targetName(targetName) + { + } + + const AZStd::string& TargetFailure::GetTargetName() const + { + return m_targetName; + } + + ExecutionFailure::ExecutionFailure(const AZStd::string& targetName, const AZStd::string& command) + : TargetFailure(targetName) + , m_commandString(command) + { + } + + const AZStd::string& ExecutionFailure::GetCommandString() const + { + return m_commandString; + } + + TestFailure::TestFailure(const AZStd::string& testName, const AZStd::string& errorMessage) + : m_name(testName) + , m_errorMessage(errorMessage) + { + } + + const AZStd::string& TestFailure::GetName() const + { + return m_name; + } + + const AZStd::string& TestFailure::GetErrorMessage() const + { + return m_errorMessage; + } + + TestCaseFailure::TestCaseFailure(const AZStd::string& testCaseName, AZStd::vector&& testFailures) + : m_name(testCaseName) + , m_testFailures(AZStd::move(testFailures)) + { + } + + const AZStd::string& TestCaseFailure::GetName() const + { + return m_name; + } + + const AZStd::vector& TestCaseFailure::GetTestFailures() const + { + return m_testFailures; + } + + TestRunFailure::TestRunFailure(const AZStd::string& targetName, AZStd::vector&& testFailures) + : TargetFailure(targetName) + , m_testCaseFailures(AZStd::move(testFailures)) + { + for (const auto& testCase : m_testCaseFailures) + { + m_numTestFailures += testCase.GetTestFailures().size(); + } + } + + size_t TestRunFailure::GetNumTestFailures() const + { + return m_numTestFailures; + } + + const AZStd::vector& TestRunFailure::GetTestCaseFailures() const + { + return m_testCaseFailures; + } + + SequenceFailure::SequenceFailure( + AZStd::vector&& executionFailures, + AZStd::vector&& testRunFailures, + AZStd::vector&& timedOutTests, + AZStd::vector&& unexecutionTests) + : m_executionFailures(AZStd::move(executionFailures)) + , m_testRunFailures(testRunFailures) + , m_timedOutTests(AZStd::move(timedOutTests)) + , m_unexecutedTests(AZStd::move(unexecutionTests)) + { + } + + const AZStd::vector& SequenceFailure::GetExecutionFailures() const + { + return m_executionFailures; + } + + const AZStd::vector& SequenceFailure::GetTestRunFailures() const + { + return m_testRunFailures; + } + + const AZStd::vector& SequenceFailure::GetTimedOutTests() const + { + return m_timedOutTests; + } + + const AZStd::vector& SequenceFailure::GetUnexecutedTests() const + { + return m_unexecutedTests; + } + } +} diff --git a/Code/Tools/TestImpactFramework/Runtime/Code/Source/TestImpactClientTestRun.cpp b/Code/Tools/TestImpactFramework/Runtime/Code/Source/TestImpactClientTestRun.cpp new file mode 100644 index 0000000000..6661de5d69 --- /dev/null +++ b/Code/Tools/TestImpactFramework/Runtime/Code/Source/TestImpactClientTestRun.cpp @@ -0,0 +1,40 @@ +/* + * All or portions of this file Copyright (c) Amazon.com, Inc. or its affiliates or + * its licensors. + * + * For complete copyright and license terms please see the LICENSE at the root of this + * distribution (the "License"). All use of this software is governed by the License, + * or, if provided, by the license below or the license accompanying this file. Do not + * remove or modify any license notices. This file is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * + */ + +#include +namespace TestImpact +{ + namespace Client + { + TestRun::TestRun(const AZStd::string& name, TestRunResult result, AZStd::chrono::milliseconds duration) + : m_targetName(name) + , m_result(result) + , m_duration(duration) + { + } + + const AZStd::string& TestRun::GetTargetName() const + { + return m_targetName; + } + + AZStd::chrono::milliseconds TestRun::GetDuration() const + { + return m_duration; + } + + TestRunResult TestRun::GetResult() const + { + return m_result; + } + } +} diff --git a/Code/Tools/TestImpactFramework/Runtime/Code/Source/TestImpactClientTestSelection.cpp b/Code/Tools/TestImpactFramework/Runtime/Code/Source/TestImpactClientTestSelection.cpp new file mode 100644 index 0000000000..f6c88893d7 --- /dev/null +++ b/Code/Tools/TestImpactFramework/Runtime/Code/Source/TestImpactClientTestSelection.cpp @@ -0,0 +1,56 @@ +/* + * All or portions of this file Copyright (c) Amazon.com, Inc. or its affiliates or + * its licensors. + * + * For complete copyright and license terms please see the LICENSE at the root of this + * distribution (the "License"). All use of this software is governed by the License, + * or, if provided, by the license below or the license accompanying this file. Do not + * remove or modify any license notices. This file is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * + */ + +#include + +namespace TestImpact +{ + namespace Client + { + TestRunSelection::TestRunSelection(const AZStd::vector& includedTests, const AZStd::vector& excludedTests) + : m_includedTestRuns(includedTests) + , m_excludedTestRuns(excludedTests) + { + } + + TestRunSelection::TestRunSelection(AZStd::vector&& includedTests, AZStd::vector&& excludedTests) + : m_includedTestRuns(AZStd::move(includedTests)) + , m_excludedTestRuns(AZStd::move(excludedTests)) + { + } + + const AZStd::vector& TestRunSelection::GetIncludededTestRuns() const + { + return m_includedTestRuns; + } + + const AZStd::vector& TestRunSelection::GetExcludedTestRuns() const + { + return m_excludedTestRuns; + } + + size_t TestRunSelection::GetNumIncludedTestRuns() const + { + return m_includedTestRuns.size(); + } + + size_t TestRunSelection::GetNumExcludedTestRuns() const + { + return m_excludedTestRuns.size(); + } + + size_t TestRunSelection::GetTotalNumTests() const + { + return GetNumIncludedTestRuns() + GetNumExcludedTestRuns(); + } + } +} diff --git a/Code/Tools/TestImpactFramework/Runtime/Code/Source/TestImpactRepoPath.cpp b/Code/Tools/TestImpactFramework/Runtime/Code/Source/TestImpactRepoPath.cpp new file mode 100644 index 0000000000..11b927247f --- /dev/null +++ b/Code/Tools/TestImpactFramework/Runtime/Code/Source/TestImpactRepoPath.cpp @@ -0,0 +1,104 @@ +/* + * All or portions of this file Copyright (c) Amazon.com, Inc. or its affiliates or + * its licensors. + * + * For complete copyright and license terms please see the LICENSE at the root of this + * distribution (the "License"). All use of this software is governed by the License, + * or, if provided, by the license below or the license accompanying this file. Do not + * remove or modify any license notices. This file is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * + */ + +#include + +namespace TestImpact +{ + + + RepoPath& RepoPath::operator=(const string_type& other) noexcept + { + m_path = AZ::IO::Path(other).MakePreferred(); + return *this; + } + + RepoPath& RepoPath::operator=(const value_type* other) noexcept + { + m_path = AZ::IO::Path(other).MakePreferred(); + return *this; + } + + RepoPath& RepoPath::operator=(const AZ::IO::Path& other) noexcept + { + m_path = AZ::IO::Path(other).MakePreferred(); + return *this; + } + + inline RepoPath operator/(const RepoPath& lhs, const AZ::IO::PathView& rhs) + { + RepoPath result(lhs); + result.m_path /= RepoPath(rhs).m_path; + return result; + } + + inline RepoPath operator/(const RepoPath& lhs, AZStd::string_view rhs) + { + RepoPath result(lhs); + result.m_path /= RepoPath(rhs).m_path; + return result; + } + + inline RepoPath operator/(const RepoPath& lhs, const RepoPath::value_type* rhs) + { + RepoPath result(lhs); + result.m_path /= RepoPath(rhs).m_path; + return result; + } + + inline RepoPath operator/(const RepoPath& lhs, const RepoPath& rhs) + { + RepoPath result(lhs); + result.m_path /= rhs.m_path; + return result; + } + + inline RepoPath& RepoPath::operator/=(const AZ::IO::PathView& rhs) + { + m_path /= RepoPath(rhs).m_path; + return *this; + } + + + inline RepoPath& RepoPath::operator/=(AZStd::string_view rhs) + { + m_path /= RepoPath(rhs).m_path; + return *this; + } + + inline RepoPath& RepoPath::operator/=(const RepoPath::value_type* rhs) + { + m_path /= RepoPath(rhs).m_path; + return *this; + } + + inline RepoPath& RepoPath::operator/=(const RepoPath& rhs) + { + m_path /= rhs.m_path; + return *this; + } + + inline bool operator==(const RepoPath& lhs, const RepoPath& rhs) noexcept + { + return lhs.m_path.Compare(rhs.m_path) == 0; + } + + inline bool operator!=(const RepoPath& lhs, const RepoPath& rhs) noexcept + { + return lhs.m_path.Compare(rhs.m_path) != 0; + } + + inline bool operator<([[maybe_unused]] const RepoPath& lhs, [[maybe_unused]] const RepoPath& rhs) noexcept + { + return lhs.m_path.String() < rhs.m_path.String(); + } +} // namespace TestImpact diff --git a/Code/Tools/TestImpactFramework/Runtime/Code/Source/TestImpactRuntime.cpp b/Code/Tools/TestImpactFramework/Runtime/Code/Source/TestImpactRuntime.cpp new file mode 100644 index 0000000000..1bc55ab179 --- /dev/null +++ b/Code/Tools/TestImpactFramework/Runtime/Code/Source/TestImpactRuntime.cpp @@ -0,0 +1,486 @@ +/* + * All or portions of this file Copyright (c) Amazon.com, Inc. or its affiliates or + * its licensors. + * + * For complete copyright and license terms please see the LICENSE at the root of this + * distribution (the "License"). All use of this software is governed by the License, + * or, if provided, by the license below or the license accompanying this file. Do not + * remove or modify any license notices. This file is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * + */ + +#include +#include +#include + +#include +#include +#include +#include +#include +#include + +#include + +namespace TestImpact +{ + namespace + { + // Simple helper class for tracking basic timing information + class Timer + { + public: + Timer() + : m_startTime(AZStd::chrono::high_resolution_clock::now()) + { + } + + // Returns the time elapsed (in milliseconds) since the timer was instantiated + AZStd::chrono::milliseconds Elapsed() + { + const auto endTime = AZStd::chrono::high_resolution_clock::now(); + return AZStd::chrono::duration_cast(endTime - m_startTime); + } + + private: + AZStd::chrono::high_resolution_clock::time_point m_startTime; + }; + + // Handler for test run complete events + class TestRunCompleteCallbackHandler + { + public: + TestRunCompleteCallbackHandler(AZStd::optional testCompleteCallback) + : m_testCompleteCallback(testCompleteCallback) + { + } + + void operator()(const TestEngineJob& testJob) + { + if (m_testCompleteCallback.has_value()) + { + (*m_testCompleteCallback) + (Client::TestRun(testJob.GetTestTarget()->GetName(), testJob.GetTestResult(), testJob.GetDuration())); + } + } + + private: + AZStd::optional m_testCompleteCallback; + }; + } + + Runtime::Runtime( + RuntimeConfig&& config, + Policy::ExecutionFailure executionFailurePolicy, + Policy::ExecutionFailureDrafting executionFailureDraftingPolicy, + Policy::TestFailure testFailurePolicy, + Policy::IntegrityFailure integrationFailurePolicy, + Policy::TestSharding testShardingPolicy, + Policy::TargetOutputCapture targetOutputCapture, + AZStd::optional maxConcurrency) + : m_config(AZStd::move(config)) + , m_executionFailurePolicy(executionFailurePolicy) + , m_executionFailureDraftingPolicy(executionFailureDraftingPolicy) + , m_testFailurePolicy(testFailurePolicy) + , m_integrationFailurePolicy(integrationFailurePolicy) + , m_testShardingPolicy(testShardingPolicy) + , m_targetOutputCapture(targetOutputCapture) + , m_maxConcurrency(maxConcurrency.value_or(AZStd::thread::hardware_concurrency())) + { + // Construct the dynamic dependency map from the build target descriptors + m_dynamicDependencyMap = ConstructDynamicDependencyMap(m_config.m_buildTargetDescriptor, m_config.m_testTargetMeta); + + // Construct the test selector and prioritizer from the dependency graph data (NOTE: currently not implemented) + m_testSelectorAndPrioritizer = AZStd::make_unique(m_dynamicDependencyMap.get(), DependencyGraphDataMap{}); + + // Construct the target exclude list from the target configuration data + m_testTargetExcludeList = ConstructTestTargetExcludeList(m_dynamicDependencyMap->GetTestTargetList(), m_config.m_target.m_excludedTestTargets); + + // Construct the test engine with the workspace path and launcher binaries + m_testEngine = AZStd::make_unique( + m_config.m_repo.m_root, + m_config.m_target.m_outputDirectory, + m_config.m_workspace.m_active.m_relativePaths.m_enumerationCacheDirectory, + m_config.m_workspace.m_temp.m_relativePaths.m_artifactDirectory, + m_config.m_testEngine.m_testRunner.m_binary, + m_config.m_testEngine.m_instrumentation.m_binary, + m_maxConcurrency); + + try + { + // Populate the dynamic dependency map with the existing source coverage data (if any) + const auto tiaDataRaw = ReadFileContents(m_config.m_workspace.m_active.m_relativePaths.m_sparTIAFile); + const auto tiaData = DeserializeSourceCoveringTestsList(tiaDataRaw); + if (tiaData.GetNumSources()) + { + m_dynamicDependencyMap->ReplaceSourceCoverage(tiaData); + m_hasImpactAnalysisData = true; + + // Enumerate new test targets + m_testEngine->UpdateEnumerationCache( + m_dynamicDependencyMap->GetNotCoveringTests(), + Policy::ExecutionFailure::Ignore, + Policy::TestFailure::Continue, + AZStd::nullopt, + AZStd::nullopt, + AZStd::nullopt); + } + } + catch (const DependencyException& e) + { + if (integrationFailurePolicy == Policy::IntegrityFailure::Abort) + { + throw RuntimeException(e.what()); + } + } + catch ([[maybe_unused]]const Exception& e) + { + AZ_Printf("No test impact analysis data found at %s", m_config.m_workspace.m_active.m_relativePaths.m_sparTIAFile.c_str()); + } + } + + Runtime::~Runtime() = default; + + void Runtime::EnumerateMutatedTestTargets(const ChangeDependencyList& changeDependencyList) + { + AZStd::vector testTargets; + const auto addMutatedTestTargetsToEnumerationList = [this, &testTargets](const AZStd::vector& sourceDependency) + { + for (const auto& sourceDependency : sourceDependency) + { + for (const auto& parentTarget : sourceDependency.GetParentTargets()) + { + AZStd::visit([&testTargets]([[maybe_unused]] auto&& target) + { + if constexpr (IsTestTarget) + { + testTargets.push_back(target); + } + }, parentTarget.GetTarget()); + } + } + }; + + // Gather all of the test targets that have had any of their sources modified + addMutatedTestTargetsToEnumerationList(changeDependencyList.GetCreateSourceDependencies()); + addMutatedTestTargetsToEnumerationList(changeDependencyList.GetUpdateSourceDependencies()); + addMutatedTestTargetsToEnumerationList(changeDependencyList.GetDeleteSourceDependencies()); + + // Enumerate the mutated test targets to ensure their enumeration caches are up to date + m_testEngine->UpdateEnumerationCache( + testTargets, + Policy::ExecutionFailure::Ignore, + Policy::TestFailure::Continue, + AZStd::nullopt, + AZStd::nullopt, + AZStd::nullopt); + } + + AZStd::pair, AZStd::vector> Runtime::SelectCoveringTestTargetsAndUpdateEnumerationCache( + const ChangeList& changeList, + Policy::TestPrioritization testPrioritizationPolicy) + { + AZStd::vector discardedTestTargets; + + // Select and prioritize the test targets pertinent to this change list + const auto changeDependecyList = m_dynamicDependencyMap->ApplyAndResoveChangeList(changeList); + const auto selectedTestTargets = m_testSelectorAndPrioritizer->SelectTestTargets(changeDependecyList, testPrioritizationPolicy); + + // Populate a set with the selected test targets so that we can infer the discarded test target not selected for this change list + const AZStd::unordered_set selectedTestTargetSet(selectedTestTargets.begin(), selectedTestTargets.end()); + + if (m_testShardingPolicy == Policy::TestSharding::Always) + { + EnumerateMutatedTestTargets(changeDependecyList); + } + + // The test targets in the main list not in the selected test target set are the test targets not selected for this change list + for (const auto& testTarget : m_dynamicDependencyMap->GetTestTargetList().GetTargets()) + { + if (!selectedTestTargetSet.contains(&testTarget)) + { + discardedTestTargets.push_back(&testTarget); + } + } + + return { selectedTestTargets, discardedTestTargets }; + } + + AZStd::pair, AZStd::vector> Runtime::SelectTestTargetsByExcludeList( + AZStd::vector testTargets) const + { + AZStd::vector includedTestTargets; + AZStd::vector excludedTestTargets; + + if (m_testTargetExcludeList.empty()) + { + return { testTargets, {} }; + } + + for (const auto& testTarget : testTargets) + { + if (!m_testTargetExcludeList.contains(testTarget)) + { + includedTestTargets.push_back(testTarget); + } + else + { + excludedTestTargets.push_back(testTarget); + } + } + + return { includedTestTargets, excludedTestTargets }; + } + + void Runtime::ClearDynamicDependencyMapAndRemoveExistingFile() + { + DeleteFile(m_config.m_workspace.m_active.m_relativePaths.m_sparTIAFile); + m_dynamicDependencyMap->ClearAllSourceCoverage(); + } + + void Runtime::UpdateAndSerializeDynamicDependencyMap(const SourceCoveringTestsList& sourceCoverageTestsList) + { + if (!sourceCoverageTestsList.GetNumSources()) + { + return; + } + + m_dynamicDependencyMap->ReplaceSourceCoverage(sourceCoverageTestsList); + const auto sparTIA = m_dynamicDependencyMap->ExportSourceCoverage(); + const auto sparTIAData = SerializeSourceCoveringTestsList(sparTIA); + WriteFileContents(sparTIAData, m_config.m_workspace.m_active.m_relativePaths.m_sparTIAFile); + m_hasImpactAnalysisData = true; + } + + TestSequenceResult Runtime::RegularTestSequence( + const AZStd::unordered_set suitesFilter, + AZStd::optional testTargetTimeout, + AZStd::optional globalTimeout, + AZStd::optional testSequenceStartCallback, + AZStd::optional testSequenceEndCallback, + AZStd::optional testCompleteCallback) + { + Timer timer; + AZStd::vector includedTestTargets; + AZStd::vector excludedTestTargets; + + // Separate the test targets into those that are excluded by either the test filter or exclusion list and those that are not + for (const auto& testTarget : m_dynamicDependencyMap->GetTestTargetList().GetTargets()) + { + if (!m_testTargetExcludeList.contains(&testTarget)) + { + if (suitesFilter.empty()) + { + // Suite filter is empty, all tests that are not on the excluded list are included + includedTestTargets.push_back(&testTarget); + } + else if(suitesFilter.contains(testTarget.GetSuite())) + { + // Test target belonging to a suite in the suite filter are included, provided that are not on the exclude list + includedTestTargets.push_back(&testTarget); + } + else + { + // Test target not belonging to a suite in the suite filter are excluded + excludedTestTargets.push_back(&testTarget); + } + } + else + { + // Test targets on the exclude list are excluded + excludedTestTargets.push_back(&testTarget); + } + } + + // Sequence start callback + if (testSequenceStartCallback.has_value()) + { + (*testSequenceStartCallback)(Client::TestRunSelection(ExtractTestTargetNames(includedTestTargets), ExtractTestTargetNames(excludedTestTargets))); + } + + const auto [result, testJobs] = m_testEngine->RegularRun( + includedTestTargets, + m_testShardingPolicy, + m_executionFailurePolicy, + m_testFailurePolicy, + m_targetOutputCapture, + testTargetTimeout, + globalTimeout, + TestRunCompleteCallbackHandler(testCompleteCallback)); + + if (testSequenceEndCallback.has_value()) + { + (*testSequenceEndCallback)(CreateSequenceFailureReport(testJobs), timer.Elapsed()); + } + + return result; + } + + TestSequenceResult Runtime::ImpactAnalysisTestSequence( + const ChangeList& changeList, + Policy::TestPrioritization testPrioritizationPolicy, + AZStd::optional testTargetTimeout, + AZStd::optional globalTimeout, + AZStd::optional testSequenceStartCallback, + AZStd::optional testSequenceEndCallback, + AZStd::optional testCompleteCallback) + { + Timer timer; + AZStd::vector draftedTestTargets; + + auto [selectedTestTargets, discardedTestTargets] = SelectCoveringTestTargetsAndUpdateEnumerationCache(changeList, testPrioritizationPolicy); + auto [includedSelectedTestTargets, excludedSelectedTestTargets] = SelectTestTargetsByExcludeList(selectedTestTargets); + + if (testSequenceStartCallback.has_value()) + { + (*testSequenceStartCallback)( + Client::TestRunSelection(ExtractTestTargetNames(includedSelectedTestTargets), ExtractTestTargetNames(excludedSelectedTestTargets)), + ExtractTestTargetNames(discardedTestTargets), + ExtractTestTargetNames(draftedTestTargets)); + } + + const auto [result, testJobs] = m_testEngine->InstrumentedRun( + includedSelectedTestTargets, + m_testShardingPolicy, + m_executionFailurePolicy, + Policy::IntegrityFailure::Continue, + m_testFailurePolicy, + m_targetOutputCapture, + testTargetTimeout, + globalTimeout, + TestRunCompleteCallbackHandler(testCompleteCallback)); + + UpdateAndSerializeDynamicDependencyMap(CreateSourceCoveringTestFromTestCoverages(testJobs, m_config.m_repo.m_root)); + + if (testSequenceEndCallback.has_value()) + { + (*testSequenceEndCallback)(CreateSequenceFailureReport(testJobs), timer.Elapsed()); + } + + return result; + } + + AZStd::pair Runtime::SafeImpactAnalysisTestSequence( + const ChangeList& changeList, + const AZStd::unordered_set suitesFilter, + Policy::TestPrioritization testPrioritizationPolicy, + AZStd::optional testTargetTimeout, + AZStd::optional globalTimeout, + AZStd::optional testSequenceStartCallback, + AZStd::optional testSequenceEndCallback, + AZStd::optional testCompleteCallback) + { + Timer timer; + AZStd::vector draftedTestTargets; + + auto [selectedTestTargets, discardedTestTargets] = SelectCoveringTestTargetsAndUpdateEnumerationCache(changeList, testPrioritizationPolicy); + auto [includedSelectedTestTargets, excludedSelectedTestTargets] = SelectTestTargetsByExcludeList(selectedTestTargets); + auto [includedDiscardedTestTargets, excludedDiscardedTestTargets] = SelectTestTargetsByExcludeList(discardedTestTargets); + + if (testSequenceStartCallback.has_value()) + { + (*testSequenceStartCallback)( + Client::TestRunSelection(ExtractTestTargetNames(includedSelectedTestTargets), ExtractTestTargetNames(excludedSelectedTestTargets)), + Client::TestRunSelection(ExtractTestTargetNames(includedDiscardedTestTargets), ExtractTestTargetNames(excludedDiscardedTestTargets)), + ExtractTestTargetNames(draftedTestTargets)); + } + + // Impact analysis run of the selected test targets + const auto [selectedResult, selectedTestJobs] = m_testEngine->InstrumentedRun( + includedSelectedTestTargets, + m_testShardingPolicy, + m_executionFailurePolicy, + Policy::IntegrityFailure::Continue, + m_testFailurePolicy, + m_targetOutputCapture, + testTargetTimeout, + globalTimeout, + TestRunCompleteCallbackHandler(testCompleteCallback)); + + // Carry the remaining global sequence time over to the discarded test run + if (globalTimeout.has_value()) + { + const auto elapsed = timer.Elapsed(); + globalTimeout = elapsed < globalTimeout.value() ? globalTimeout.value() - elapsed : AZStd::chrono::milliseconds(0); + } + + // Regular run of the discarded test targets + const auto [discardedResult, discardedTestJobs] = m_testEngine->RegularRun( + includedDiscardedTestTargets, + m_testShardingPolicy, + m_executionFailurePolicy, + m_testFailurePolicy, + m_targetOutputCapture, + testTargetTimeout, + globalTimeout, + TestRunCompleteCallbackHandler(testCompleteCallback)); + + UpdateAndSerializeDynamicDependencyMap(CreateSourceCoveringTestFromTestCoverages(selectedTestJobs, m_config.m_repo.m_root)); + + if (testSequenceEndCallback.has_value()) + { + (*testSequenceEndCallback)( + CreateSequenceFailureReport(selectedTestJobs), + CreateSequenceFailureReport(discardedTestJobs), + timer.Elapsed()); + } + + return { selectedResult, discardedResult }; + } + + TestSequenceResult Runtime::SeededTestSequence( + AZStd::optional testTargetTimeout, + AZStd::optional globalTimeout, + AZStd::optional testSequenceStartCallback, + AZStd::optional testSequenceEndCallback, + AZStd::optional testCompleteCallback) + { + Timer timer; + AZStd::vector includedTestTargets; + AZStd::vector excludedTestTargets; + + for (const auto& testTarget : m_dynamicDependencyMap->GetTestTargetList().GetTargets()) + { + if (!m_testTargetExcludeList.contains(&testTarget)) + { + includedTestTargets.push_back(&testTarget); + } + else + { + excludedTestTargets.push_back(&testTarget); + } + } + + if (testSequenceStartCallback.has_value()) + { + (*testSequenceStartCallback)(Client::TestRunSelection(ExtractTestTargetNames(includedTestTargets), ExtractTestTargetNames(excludedTestTargets))); + } + + const auto [result, testJobs] = m_testEngine->InstrumentedRun( + includedTestTargets, + m_testShardingPolicy, + m_executionFailurePolicy, + Policy::IntegrityFailure::Continue, + m_testFailurePolicy, + m_targetOutputCapture, + testTargetTimeout, + globalTimeout, + TestRunCompleteCallbackHandler(testCompleteCallback)); + + ClearDynamicDependencyMapAndRemoveExistingFile(); + UpdateAndSerializeDynamicDependencyMap(CreateSourceCoveringTestFromTestCoverages(testJobs, m_config.m_repo.m_root)); + + if (testSequenceEndCallback.has_value()) + { + (*testSequenceEndCallback)(CreateSequenceFailureReport(testJobs), timer.Elapsed()); + } + + return result; + } + + bool Runtime::HasImpactAnalysisData() const + { + return m_hasImpactAnalysisData; + } +} diff --git a/Code/Tools/TestImpactFramework/Runtime/Code/Source/TestImpactRuntimeUtils.cpp b/Code/Tools/TestImpactFramework/Runtime/Code/Source/TestImpactRuntimeUtils.cpp new file mode 100644 index 0000000000..e0f77c6479 --- /dev/null +++ b/Code/Tools/TestImpactFramework/Runtime/Code/Source/TestImpactRuntimeUtils.cpp @@ -0,0 +1,133 @@ +/* + * All or portions of this file Copyright (c) Amazon.com, Inc. or its affiliates or + * its licensors. + * + * For complete copyright and license terms please see the LICENSE at the root of this + * distribution (the "License"). All use of this software is governed by the License, + * or, if provided, by the license below or the license accompanying this file. Do not + * remove or modify any license notices. This file is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * + */ + +#include +#include + +#include +#include +#include +#include + +#include + +namespace TestImpact +{ + TestTargetMetaMap ReadTestTargetMetaMapFile(const RepoPath& testTargetMetaConfigFile) + { + const auto masterTestListData = ReadFileContents(testTargetMetaConfigFile); + return TestTargetMetaMapFactory(masterTestListData); + } + + AZStd::vector ReadBuildTargetDescriptorFiles(const BuildTargetDescriptorConfig& buildTargetDescriptorConfig) + { + AZStd::vector buildTargetDescriptors; + for (const auto& buildTargetDescriptorFile : std::filesystem::directory_iterator(buildTargetDescriptorConfig.m_mappingDirectory.c_str())) + { + const auto buildTargetDescriptorContents = ReadFileContents(buildTargetDescriptorFile.path().string().c_str()); + auto buildTargetDescriptor = TestImpact::BuildTargetDescriptorFactory( + buildTargetDescriptorContents, + buildTargetDescriptorConfig.m_staticInclusionFilters, + buildTargetDescriptorConfig.m_inputInclusionFilters, + buildTargetDescriptorConfig.m_inputOutputPairer); + buildTargetDescriptors.emplace_back(AZStd::move(buildTargetDescriptor)); + } + + return buildTargetDescriptors; + } + + AZStd::unique_ptr ConstructDynamicDependencyMap( + const BuildTargetDescriptorConfig& buildTargetDescriptorConfig, + const TestTargetMetaConfig& testTargetMetaConfig) + { + auto testTargetmetaMap = ReadTestTargetMetaMapFile(testTargetMetaConfig.m_metaFile); + auto buildTargetDescriptors = ReadBuildTargetDescriptorFiles(buildTargetDescriptorConfig); + auto buildTargets = CompileTargetDescriptors(AZStd::move(buildTargetDescriptors), AZStd::move(testTargetmetaMap)); + auto&& [productionTargets, testTargets] = buildTargets; + return AZStd::make_unique(AZStd::move(productionTargets), AZStd::move(testTargets)); + } + + AZStd::unordered_set ConstructTestTargetExcludeList(const TestTargetList& testTargets, const AZStd::vector& excludedTestTargets) + { + AZStd::unordered_set testTargetExcludeList; + for (const auto& testTargetName : excludedTestTargets) + { + if (const auto* testTarget = testTargets.GetTarget(testTargetName); testTarget != nullptr) + { + testTargetExcludeList.insert(testTarget); + } + } + + return testTargetExcludeList; + } + + AZStd::vector ExtractTestTargetNames(const AZStd::vector testTargets) + { + AZStd::vector testNames; + AZStd::transform(testTargets.begin(), testTargets.end(), AZStd::back_inserter(testNames), [](const TestTarget* testTarget) + { + return testTarget->GetName(); + }); + + return testNames; + } + + SourceCoveringTestsList CreateSourceCoveringTestFromTestCoverages(const AZStd::vector& jobs, const RepoPath& root) + { + AZStd::unordered_map> coverage; + for (const auto& job : jobs) + { + if (const auto testResult = job.GetTestResult(); + testResult == Client::TestRunResult::AllTestsPass || testResult == Client::TestRunResult::TestFailures) + { + if (testResult == Client::TestRunResult::AllTestsPass) + { + // Passing tests should have coverage data, otherwise something is very wrong + AZ_TestImpact_Eval( + job.GetTestCoverge().has_value(), + RuntimeException, + AZStd::string::format( + "Test target '%s' completed its test run successfully but produced no coverage data", + job.GetTestTarget()->GetName().c_str())); + } + else if (!job.GetTestCoverge().has_value()) + { + // When a test run completes with failing tests but produces no coverage artifact that's typically a sign of the + // test aborting due to an unhandled exception, in which case ignore it and let it be picked up in the failure report + continue; + } + + for (const auto& source : job.GetTestCoverge().value().GetSourcesCovered()) + { + coverage[source.String()].insert(job.GetTestTarget()->GetName()); + } + } + } + + AZStd::vector sourceCoveringTests; + sourceCoveringTests.reserve(coverage.size()); + for (auto&& [source, testTargets] : coverage) + { + if (const auto sourcePath = RepoPath(source); + sourcePath.IsRelativeTo(root)) + { + sourceCoveringTests.push_back(SourceCoveringTests(RepoPath(sourcePath.LexicallyRelative(root)), AZStd::move(testTargets))); + } + else + { + AZ_Warning("TestImpact", false, "Ignoring source, source it outside of repo: %s", sourcePath.c_str()); + } + } + + return SourceCoveringTestsList(AZStd::move(sourceCoveringTests)); + } +} diff --git a/Code/Tools/TestImpactFramework/Runtime/Code/Source/TestImpactRuntimeUtils.h b/Code/Tools/TestImpactFramework/Runtime/Code/Source/TestImpactRuntimeUtils.h new file mode 100644 index 0000000000..3210ba808b --- /dev/null +++ b/Code/Tools/TestImpactFramework/Runtime/Code/Source/TestImpactRuntimeUtils.h @@ -0,0 +1,134 @@ +/* + * All or portions of this file Copyright (c) Amazon.com, Inc. or its affiliates or + * its licensors. + * + * For complete copyright and license terms please see the LICENSE at the root of this + * distribution (the "License"). All use of this software is governed by the License, + * or, if provided, by the license below or the license accompanying this file. Do not + * remove or modify any license notices. This file is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * + */ + +#pragma once + +#include +#include +#include + +#include +#include +#include +#include +#include +#include +#include + +#include + +namespace TestImpact +{ + //! Construct a dynamic dependency map from the build target descriptors and test target metas. + AZStd::unique_ptr ConstructDynamicDependencyMap( + const BuildTargetDescriptorConfig& buildTargetDescriptorConfig, + const TestTargetMetaConfig& testTargetMetaConfig); + + //! Constructs the resolved test target exclude list from the specified list of targets and unresolved test target exclude list. + AZStd::unordered_set ConstructTestTargetExcludeList( + const TestTargetList& testTargets, + const AZStd::vector& excludedTestTargets); + + //! Extracts the name information from the specified test targets. + AZStd::vector ExtractTestTargetNames(const AZStd::vector testTargets); + + //! Creates the consolidates source covering tests list from the test engine instrumented run jobs. + SourceCoveringTestsList CreateSourceCoveringTestFromTestCoverages( + const AZStd::vector& jobs, + const RepoPath& root); + + //! Generates a test run failure report from the specified test engine job information. + //! @tparam TestJob The test engine job type. + template + Client::TestRunFailure GenerateTestRunFailure(const TestJob& testJob) + { + if (testJob.GetTestRun().has_value()) + { + AZStd::vector testCaseFailures; + for (const auto& testSuite : testJob.GetTestRun()->GetTestSuites()) + { + AZStd::vector testFailures; + for (const auto& testCase : testSuite.m_tests) + { + if(testCase.m_result.value_or(TestRunResult::Passed) == TestRunResult::Failed) + { + testFailures.push_back(Client::TestFailure(testCase.m_name, "No error message retrieved")); + } + } + + if (!testFailures.empty()) + { + testCaseFailures.push_back(Client::TestCaseFailure(testSuite.m_name, AZStd::move(testFailures))); + } + } + + return Client::TestRunFailure(Client::TestRunFailure(testJob.GetTestTarget()->GetName(), AZStd::move(testCaseFailures))); + } + else + { + return Client::TestRunFailure(testJob.GetTestTarget()->GetName(), { }); + } + } + + //! Generates a sequence failure report from the specified list of test engine jobs. + //! @tparam TestJob The test engine job type. + template + Client::SequenceFailure GenerateSequenceFailureReport(const AZStd::vector& testJobs) + { + AZStd::vector executionFailures; + AZStd::vector testRunFailures; + AZStd::vector timedOutTestRuns; + AZStd::vector unexecutedTestRuns; + + for (const auto& testJob : testJobs) + { + switch (testJob.GetTestResult()) + { + case Client::TestRunResult::FailedToExecute: + { + executionFailures.push_back(Client::ExecutionFailure(testJob.GetTestTarget()->GetName(), testJob.GetCommandString())); + break; + } + case Client::TestRunResult::NotRun: + { + unexecutedTestRuns.push_back(testJob.GetTestTarget()->GetName()); + break; + } + case Client::TestRunResult::Timeout: + { + timedOutTestRuns.push_back(testJob.GetTestTarget()->GetName()); + break; + } + case Client::TestRunResult::AllTestsPass: + { + break; + } + case Client::TestRunResult::TestFailures: + { + testRunFailures.push_back(ExtractTestRunFailure(testJob)); + break; + } + default: + { + throw Exception( + AZStd::string::format("Unexpected client test run result: %u", static_cast(testJob.GetTestResult()))); + } + } + } + + return Client::SequenceFailure( + AZStd::move(executionFailures), + AZStd::move(testRunFailures), + AZStd::move(timedOutTestRuns), + AZStd::move(unexecutedTestRuns)); + } +}