From bdbbedda71c9e60c5c3cc200b9c130fd249811fc Mon Sep 17 00:00:00 2001 From: jonawals Date: Mon, 24 May 2021 15:00:21 +0100 Subject: [PATCH] Add test engine --- .../TestImpactTestEnumerationSerializer.cpp | 4 +- .../Enumeration/TestImpactTestEnumerator.cpp | 71 ++-- .../Enumeration/TestImpactTestEnumerator.h | 4 +- .../Run/TestImpactInstrumentedTestRunner.cpp | 29 +- .../Run/TestImpactInstrumentedTestRunner.h | 14 - .../Run/TestImpactTestRunSerializer.cpp | 4 +- .../TestEngine/Run/TestImpactTestRunner.cpp | 11 +- .../TestEngine/TestImpactTestEngine.cpp | 382 ++++++++++++++++++ .../Source/TestEngine/TestImpactTestEngine.h | 126 ++++++ .../TestImpactTestEngineEnumeration.cpp | 27 ++ .../TestImpactTestEngineEnumeration.h | 32 ++ ...tion.h => TestImpactTestEngineException.h} | 4 +- .../TestImpactTestEngineInstrumentedRun.cpp | 52 +++ .../TestImpactTestEngineInstrumentedRun.h | 32 ++ .../TestEngine/TestImpactTestEngineJob.cpp | 40 ++ .../TestEngine/TestImpactTestEngineJob.h | 41 ++ ...n.h => TestImpactTestEngineRegularRun.cpp} | 19 +- .../TestImpactTestEngineRegularRun.h | 34 ++ 18 files changed, 820 insertions(+), 106 deletions(-) create mode 100644 Code/Tools/TestImpactFramework/Runtime/Code/Source/TestEngine/TestImpactTestEngine.cpp create mode 100644 Code/Tools/TestImpactFramework/Runtime/Code/Source/TestEngine/TestImpactTestEngine.h create mode 100644 Code/Tools/TestImpactFramework/Runtime/Code/Source/TestEngine/TestImpactTestEngineEnumeration.cpp create mode 100644 Code/Tools/TestImpactFramework/Runtime/Code/Source/TestEngine/TestImpactTestEngineEnumeration.h rename Code/Tools/TestImpactFramework/Runtime/Code/Source/TestEngine/{Run/TestImpactTestRunException.h => TestImpactTestEngineException.h} (86%) create mode 100644 Code/Tools/TestImpactFramework/Runtime/Code/Source/TestEngine/TestImpactTestEngineInstrumentedRun.cpp create mode 100644 Code/Tools/TestImpactFramework/Runtime/Code/Source/TestEngine/TestImpactTestEngineInstrumentedRun.h create mode 100644 Code/Tools/TestImpactFramework/Runtime/Code/Source/TestEngine/TestImpactTestEngineJob.cpp create mode 100644 Code/Tools/TestImpactFramework/Runtime/Code/Source/TestEngine/TestImpactTestEngineJob.h rename Code/Tools/TestImpactFramework/Runtime/Code/Source/TestEngine/{Enumeration/TestImpactTestEnumerationException.h => TestImpactTestEngineRegularRun.cpp} (61%) create mode 100644 Code/Tools/TestImpactFramework/Runtime/Code/Source/TestEngine/TestImpactTestEngineRegularRun.h diff --git a/Code/Tools/TestImpactFramework/Runtime/Code/Source/TestEngine/Enumeration/TestImpactTestEnumerationSerializer.cpp b/Code/Tools/TestImpactFramework/Runtime/Code/Source/TestEngine/Enumeration/TestImpactTestEnumerationSerializer.cpp index 05a13a0703..d1e7925cc0 100644 --- a/Code/Tools/TestImpactFramework/Runtime/Code/Source/TestEngine/Enumeration/TestImpactTestEnumerationSerializer.cpp +++ b/Code/Tools/TestImpactFramework/Runtime/Code/Source/TestEngine/Enumeration/TestImpactTestEnumerationSerializer.cpp @@ -10,7 +10,7 @@ * */ -#include +#include #include #include @@ -82,7 +82,7 @@ namespace TestImpact if (doc.Parse<0>(testEnumString.c_str()).HasParseError()) { - throw TestEnumerationException("Could not parse enumeration data"); + throw TestEngineException("Could not parse enumeration data"); } for (const auto& suite : doc[TestEnumFields::Keys[TestEnumFields::SuitesKey]].GetArray()) diff --git a/Code/Tools/TestImpactFramework/Runtime/Code/Source/TestEngine/Enumeration/TestImpactTestEnumerator.cpp b/Code/Tools/TestImpactFramework/Runtime/Code/Source/TestEngine/Enumeration/TestImpactTestEnumerator.cpp index 09cea87d41..f7b6d4a264 100644 --- a/Code/Tools/TestImpactFramework/Runtime/Code/Source/TestEngine/Enumeration/TestImpactTestEnumerator.cpp +++ b/Code/Tools/TestImpactFramework/Runtime/Code/Source/TestEngine/Enumeration/TestImpactTestEnumerator.cpp @@ -13,7 +13,7 @@ #include #include -#include +#include #include #include #include @@ -36,7 +36,7 @@ namespace TestImpact AZ::IO::SystemFile::SF_OPEN_CREATE | AZ::IO::SystemFile::SF_OPEN_CREATE_PATH | AZ::IO::SystemFile::SF_OPEN_WRITE_ONLY)) { AZ_TestImpact_Eval( - !IsFlagSet(cacheExceptionPolicy, Bitwise::CacheExceptionPolicy::OnCacheWriteFailure), TestEnumerationException, + !IsFlagSet(cacheExceptionPolicy, Bitwise::CacheExceptionPolicy::OnCacheWriteFailure), TestEngineException, "Couldn't open cache file for writing"); return; } @@ -44,50 +44,16 @@ namespace TestImpact if (cacheFile.Write(cacheBytes.data(), cacheBytes.size()) == 0) { AZ_TestImpact_Eval( - !IsFlagSet(cacheExceptionPolicy, Bitwise::CacheExceptionPolicy::OnCacheWriteFailure), TestEnumerationException, + !IsFlagSet(cacheExceptionPolicy, Bitwise::CacheExceptionPolicy::OnCacheWriteFailure), TestEngineException, "Couldn't write cache file data"); return; } } - - AZStd::optional ReadCacheFile(const RepoPath& path, Bitwise::CacheExceptionPolicy cacheExceptionPolicy) - { - AZ::IO::SystemFile cacheFile; - AZStd::string cacheJSON; - if (!cacheFile.Open(path.c_str(), AZ::IO::SystemFile::SF_OPEN_READ_ONLY)) - { - AZ_TestImpact_Eval( - !IsFlagSet(cacheExceptionPolicy, Bitwise::CacheExceptionPolicy::OnCacheNotExist), TestEnumerationException, - "Couldn't locate cache file"); - return AZStd::nullopt; - } - - const AZ::IO::SystemFile::SizeType length = cacheFile.Length(); - if (length == 0) - { - AZ_TestImpact_Eval( - !IsFlagSet(cacheExceptionPolicy, Bitwise::CacheExceptionPolicy::OnCacheReadFailure), TestEnumerationException, - "Cache file is empty"); - return AZStd::nullopt; - } - - cacheFile.Seek(0, AZ::IO::SystemFile::SF_SEEK_BEGIN); - cacheJSON.resize(length); - if (cacheFile.Read(length, cacheJSON.data()) != length) - { - AZ_TestImpact_Eval( - !IsFlagSet(cacheExceptionPolicy, Bitwise::CacheExceptionPolicy::OnCacheReadFailure), TestEnumerationException, - "Couldn't read cache file"); - return AZStd::nullopt; - } - - return DeserializeTestEnumeration(cacheJSON); - } } // namespace TestEnumeration ParseTestEnumerationFile(const RepoPath& enumerationFile) { - return TestEnumeration(GTest::TestEnumerationSuitesFactory(ReadFileContents(enumerationFile))); + return TestEnumeration(GTest::TestEnumerationSuitesFactory(ReadFileContents(enumerationFile))); } TestEnumerationJobData::TestEnumerationJobData(const RepoPath& enumerationArtifact, AZStd::optional&& cache) @@ -128,7 +94,16 @@ namespace TestImpact if (jobInfo.GetCache().has_value() && jobInfo.GetCache()->m_policy == JobData::CachePolicy::Read) { JobMeta meta; - auto enumeration = ReadCacheFile(jobInfo.GetCache()->m_file, cacheExceptionPolicy); + AZStd::optional enumeration; + + try + { + enumeration = TestEnumeration(DeserializeTestEnumeration(ReadFileContents(jobInfo.GetCache()->m_file))); + } + catch (const TestEngineException& e) + { + AZ_Printf("Enumerate", "Enumeration cache error: %s", e.what()); + } // Even though cached jobs don't get executed we still give the client the opportunity to handle the job state // change in order to make the caching process transparent to the client @@ -164,12 +139,20 @@ namespace TestImpact const auto& [meta, jobInfo] = jobData; if (meta.m_result == JobResult::ExecutedWithSuccess) { - const auto& enumeration = (enumerations[jobId] = ParseTestEnumerationFile(jobInfo->GetEnumerationArtifactPath())); - - // Write out the enumeration to a cache file if we have a cache write policy for this job - if (jobInfo->GetCache().has_value() && jobInfo->GetCache()->m_policy == JobData::CachePolicy::Write) + try { - WriteCacheFile(enumeration.value(), jobInfo->GetCache()->m_file, cacheExceptionPolicy); + const auto& enumeration = (enumerations[jobId] = ParseTestEnumerationFile(jobInfo->GetEnumerationArtifactPath())); + + // Write out the enumeration to a cache file if we have a cache write policy for this job + if (jobInfo->GetCache().has_value() && jobInfo->GetCache()->m_policy == JobData::CachePolicy::Write) + { + WriteCacheFile(enumeration.value(), jobInfo->GetCache()->m_file, cacheExceptionPolicy); + } + } + catch (const Exception& e) + { + AZ_Warning("Enumerate", false, e.what()); + enumerations[jobId] = AZStd::nullopt; } } } diff --git a/Code/Tools/TestImpactFramework/Runtime/Code/Source/TestEngine/Enumeration/TestImpactTestEnumerator.h b/Code/Tools/TestImpactFramework/Runtime/Code/Source/TestEngine/Enumeration/TestImpactTestEnumerator.h index fedaf300f7..c802b92529 100644 --- a/Code/Tools/TestImpactFramework/Runtime/Code/Source/TestEngine/Enumeration/TestImpactTestEnumerator.h +++ b/Code/Tools/TestImpactFramework/Runtime/Code/Source/TestEngine/Enumeration/TestImpactTestEnumerator.h @@ -57,9 +57,7 @@ namespace TestImpact enum class CacheExceptionPolicy { Never = 0, //! Never throw. - OnCacheNotExist = 1, //! Throw when a cache read policy was in place but a cache file for this job doesn't exist. - OnCacheReadFailure = 1 << 1, //! Throw when a cache read policy is in place but the cache file could not be read. - OnCacheWriteFailure = 1 << 2 //! Throw when a cache write policy is in place but the cache file could not be written. + OnCacheWriteFailure = 1 //! Throw when a cache write policy is in place but the cache file could not be written. }; } // namespace Bitwise diff --git a/Code/Tools/TestImpactFramework/Runtime/Code/Source/TestEngine/Run/TestImpactInstrumentedTestRunner.cpp b/Code/Tools/TestImpactFramework/Runtime/Code/Source/TestEngine/Run/TestImpactInstrumentedTestRunner.cpp index c0373e89f5..3b109c3a77 100644 --- a/Code/Tools/TestImpactFramework/Runtime/Code/Source/TestEngine/Run/TestImpactInstrumentedTestRunner.cpp +++ b/Code/Tools/TestImpactFramework/Runtime/Code/Source/TestEngine/Run/TestImpactInstrumentedTestRunner.cpp @@ -14,8 +14,8 @@ #include #include +#include #include -#include #include #include @@ -36,18 +36,10 @@ namespace TestImpact InstrumentedTestRunner::JobPayload ParseTestRunAndCoverageFiles( const RepoPath& runFile, const RepoPath& coverageFile, - AZStd::chrono::milliseconds duration, - InstrumentedTestRunner::CoverageExceptionPolicy coverageExceptionPolicy) + AZStd::chrono::milliseconds duration) { - TestRun run(GTest::TestRunSuitesFactory(ReadFileContents(runFile)), duration); - AZStd::vector moduleCoverages = Cobertura::ModuleCoveragesFactory(ReadFileContents(coverageFile)); - if (moduleCoverages.empty()) - { - AZ_TestImpact_Eval( - !IsFlagSet(coverageExceptionPolicy, Bitwise::CoverageExceptionPolicy::OnEmptyCoverage), TestRunException, - AZStd::string::format("No coverage data generated for '%s'", coverageFile.c_str())); - } - + TestRun run(GTest::TestRunSuitesFactory(ReadFileContents(runFile)), duration); + AZStd::vector moduleCoverages = Cobertura::ModuleCoveragesFactory(ReadFileContents(coverageFile)); TestCoverage coverage(AZStd::move(moduleCoverages)); return {AZStd::move(run), AZStd::move(coverage)}; } @@ -59,13 +51,12 @@ namespace TestImpact AZStd::pair> InstrumentedTestRunner::RunInstrumentedTests( const AZStd::vector& jobInfos, - CoverageExceptionPolicy coverageExceptionPolicy, JobExceptionPolicy jobExceptionPolicy, AZStd::optional runTimeout, AZStd::optional runnerTimeout, AZStd::optional clientCallback) { - const auto payloadGenerator = [this, coverageExceptionPolicy](const JobDataMap& jobDataMap) + const auto payloadGenerator = [this](const JobDataMap& jobDataMap) { PayloadMap runs; for (const auto& [jobId, jobData] : jobDataMap) @@ -78,18 +69,12 @@ namespace TestImpact runs[jobId] = ParseTestRunAndCoverageFiles( jobInfo->GetRunArtifactPath(), jobInfo->GetCoverageArtifactPath(), - meta.m_duration.value(), - coverageExceptionPolicy); + meta.m_duration.value()); } catch (const Exception& e) { - AZ_Warning("RunInstrumentedTests", false, e.what()); + AZ_Printf("RunInstrumentedTests", e.what()); runs[jobId] = AZStd::nullopt; - - if (coverageExceptionPolicy == CoverageExceptionPolicy::OnEmptyCoverage) - { - break; - } } } } diff --git a/Code/Tools/TestImpactFramework/Runtime/Code/Source/TestEngine/Run/TestImpactInstrumentedTestRunner.h b/Code/Tools/TestImpactFramework/Runtime/Code/Source/TestEngine/Run/TestImpactInstrumentedTestRunner.h index 42b718b241..b2d7f84b6d 100644 --- a/Code/Tools/TestImpactFramework/Runtime/Code/Source/TestEngine/Run/TestImpactInstrumentedTestRunner.h +++ b/Code/Tools/TestImpactFramework/Runtime/Code/Source/TestEngine/Run/TestImpactInstrumentedTestRunner.h @@ -33,16 +33,6 @@ namespace TestImpact RepoPath m_coverageArtifact; //!< Path to coverage data. }; - namespace Bitwise - { - //! Exception policy for test coverage artifacts. - enum class CoverageExceptionPolicy - { - Never = 0, //! Never throw. - OnEmptyCoverage = 1 //! Throw when no coverage data was produced. - }; - } // namespace Bitwise - //! Runs a batch of test targets to determine the test coverage and passes/failures. class InstrumentedTestRunner : public TestJobRunner> @@ -50,15 +40,12 @@ namespace TestImpact using JobRunner = TestJobRunner>; public: - using CoverageExceptionPolicy = Bitwise::CoverageExceptionPolicy; - //! Constructs an instrumented test runner with the specified parameters common to all job runs of this runner. //! @param maxConcurrentRuns The maximum number of runs to be in flight at any given time. explicit InstrumentedTestRunner(size_t maxConcurrentRuns); //! Executes the specified instrumented test run jobs according to the specified job exception policies. //! @param jobInfos The test run jobs to execute. - //! @param CoverageExceptionPolicy The coverage exception policy to be used for this run. //! @param jobExceptionPolicy The test run job exception policy to be used for this run (use //! TestJobExceptionPolicy::OnFailedToExecute to throw on test failures). //! @param runTimeout The maximum duration a run may be in-flight for before being forcefully terminated. @@ -67,7 +54,6 @@ namespace TestImpact //! @return The result of the run sequence and the instrumented run jobs with their associated test run and coverage payloads. AZStd::pair> RunInstrumentedTests( const AZStd::vector& jobInfos, - CoverageExceptionPolicy coverageExceptionPolicy, JobExceptionPolicy jobExceptionPolicy, AZStd::optional runTimeout, AZStd::optional runnerTimeout, diff --git a/Code/Tools/TestImpactFramework/Runtime/Code/Source/TestEngine/Run/TestImpactTestRunSerializer.cpp b/Code/Tools/TestImpactFramework/Runtime/Code/Source/TestEngine/Run/TestImpactTestRunSerializer.cpp index 7fe8789d22..15d3cc04b1 100644 --- a/Code/Tools/TestImpactFramework/Runtime/Code/Source/TestEngine/Run/TestImpactTestRunSerializer.cpp +++ b/Code/Tools/TestImpactFramework/Runtime/Code/Source/TestEngine/Run/TestImpactTestRunSerializer.cpp @@ -10,7 +10,7 @@ * */ -#include +#include #include #include @@ -142,7 +142,7 @@ namespace TestImpact if (doc.Parse<0>(testEnumString.c_str()).HasParseError()) { - throw TestRunException("Could not parse enumeration data"); + throw TestEngineException("Could not parse enumeration data"); } // Run duration diff --git a/Code/Tools/TestImpactFramework/Runtime/Code/Source/TestEngine/Run/TestImpactTestRunner.cpp b/Code/Tools/TestImpactFramework/Runtime/Code/Source/TestEngine/Run/TestImpactTestRunner.cpp index 27780a5f95..401084c934 100644 --- a/Code/Tools/TestImpactFramework/Runtime/Code/Source/TestEngine/Run/TestImpactTestRunner.cpp +++ b/Code/Tools/TestImpactFramework/Runtime/Code/Source/TestEngine/Run/TestImpactTestRunner.cpp @@ -13,7 +13,7 @@ #include #include -#include +#include #include #include @@ -21,11 +21,6 @@ namespace TestImpact { - TestRun ParseTestRunFile(const RepoPath& runFile, AZStd::chrono::milliseconds duration) - { - return TestRun(GTest::TestRunSuitesFactory(ReadFileContents(runFile)), duration); - } - TestRunner::TestRunner(size_t maxConcurrentRuns) : JobRunner(maxConcurrentRuns) { @@ -48,11 +43,11 @@ namespace TestImpact { try { - runs[jobId] = ParseTestRunFile(jobInfo->GetRunArtifactPath(), meta.m_duration.value()); + runs[jobId] = TestRun(GTest::TestRunSuitesFactory(ReadFileContents(jobInfo->GetRunArtifactPath())), meta.m_duration.value()); } catch (const Exception& e) { - AZ_Warning("RunTests", false, e.what()); + AZ_Printf("RunTests", e.what()); runs[jobId] = AZStd::nullopt; } } diff --git a/Code/Tools/TestImpactFramework/Runtime/Code/Source/TestEngine/TestImpactTestEngine.cpp b/Code/Tools/TestImpactFramework/Runtime/Code/Source/TestEngine/TestImpactTestEngine.cpp new file mode 100644 index 0000000000..5046b3f8ee --- /dev/null +++ b/Code/Tools/TestImpactFramework/Runtime/Code/Source/TestEngine/TestImpactTestEngine.cpp @@ -0,0 +1,382 @@ +/* + * All or portions of this file Copyright (c) Amazon.com, Inc. or its affiliates or + * its licensors. + * + * For complete copyright and license terms please see the LICENSE at the root of this + * distribution (the "License"). All use of this software is governed by the License, + * or, if provided, by the license below or the license accompanying this file. Do not + * remove or modify any license notices. This file is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * + */ + +#include +#include +#include +#include +#include +#include +#include + +#include + +namespace TestImpact +{ + // Known error codes for test instrumentation, test runner and unit test library + // This could be refactored into a generic solution agnostic of the tool and library specific details + namespace ErrorCodes + { + namespace OpenCppCoverage + { + static constexpr ReturnCode InvalidArgs = -1618178468; + } + + namespace GTest + { + static constexpr ReturnCode Unsuccessful = 1; + } + + namespace AZTestRunner + { + static constexpr ReturnCode InvalidArgs = 101; + static constexpr ReturnCode FailedToFindTargetBinary = 102; + static constexpr ReturnCode SymbolNotFound = 103; + static constexpr ReturnCode ModuleSkipped = 104; + } + } + + namespace + { + // Calculate the sequence result by analysing the state of the test targets that were run. + template + TestSequenceResult CalculateSequenceResult( + ProcessSchedulerResult result, + const AZStd::vector& engineJobs, + Policy::ExecutionFailure executionFailurePolicy) + { + if (result == ProcessSchedulerResult::Timeout) + { + // Test job runner timing out overrules all other possible sequence results + return TestSequenceResult::Timeout; + } + + bool hasExecutionFailures = false; + bool hasTestFailures = false; + for (const auto& engineJob : engineJobs) + { + switch (engineJob.GetTestResult()) + { + case Client::TestRunResult::FailedToExecute: + { + hasExecutionFailures = true; + break; + } + case Client::TestRunResult::Timeout: + case Client::TestRunResult::TestFailures: + { + hasTestFailures = true; + break; + } + default: + { + continue; + } + } + } + + // Execution failure can be considered test passes if a permissive execution failure policy is used, otherwise they are failures + if ((hasExecutionFailures && executionFailurePolicy != Policy::ExecutionFailure::Ignore) || hasTestFailures) + { + return TestSequenceResult::Failure; + } + else + { + return TestSequenceResult::Success; + } + } + + // Deduces the run result for a given test target based on how the process exited and known return values + Client::TestRunResult GetClientTestRunResultForMeta(const JobMeta& meta) + { + // Attempt to determine why a given test target executed successfully but return with an error code + if (meta.m_returnCode.has_value()) + { + switch (meta.m_returnCode.value()) + { + // We will consider test targets that technically execute but their launcher or unit test library return a know error + // code that pertains to incorrect argument usage as test targets that failed to execute + case ErrorCodes::OpenCppCoverage::InvalidArgs: + case ErrorCodes::AZTestRunner::InvalidArgs: + case ErrorCodes::AZTestRunner::FailedToFindTargetBinary: + case ErrorCodes::AZTestRunner::ModuleSkipped: + case ErrorCodes::AZTestRunner::SymbolNotFound: + return Client::TestRunResult::FailedToExecute; + // The trivial case: the test target has failing tests + case ErrorCodes::GTest::Unsuccessful: + return Client::TestRunResult::TestFailures; + default: + break; + } + } + + switch (meta.m_result) + { + // If the test target executed successfully but returned in an unknown abnormal state it's probably because a test caused + // an unhandled exception, segfault or any other of the weird and wonderful ways a badly behaving test can terminate + case JobResult::ExecutedWithFailure: + return Client::TestRunResult::TestFailures; + // The trivial case: all of the tests in the test target passed + case JobResult::ExecutedWithSuccess: + return Client::TestRunResult::AllTestsPass; + // NotExecuted happens when a test is queued for launch but the test runner terminates the sequence (either due to client abort + // or due to the sequence timer expiring) whereas Terminated happens when the aforementioned scenarios happen when the test target + // is in flight + case JobResult::NotExecuted: + case JobResult::Terminated: + return Client::TestRunResult::NotRun; + // The individual timer for the test target expired + case JobResult::Timeout: + return Client::TestRunResult::Timeout; + default: + throw(TestEngineException(AZStd::string::format("Unexpected job result: %u", static_cast(meta.m_result)))); + } + } + + // Map for storing the test engine job data of completed test target runs + template + using TestEngineJobMap = AZStd::unordered_map; + + // Helper trait for identifying the test engine job specialization for a given test job runner + template + struct TestJobRunnerTrait + {}; + + // Helper function for getting the type directly of the test job runner trait + template + using TestEngineJobType = typename TestJobRunnerTrait::TestEngineJobType; + + // Type trait for the test enumerator + template<> + struct TestJobRunnerTrait + { + using TestEngineJobType = TestEngineEnumeration; + }; + + // Type trait for the test runner + template<> + struct TestJobRunnerTrait + { + using TestEngineJobType = TestEngineRegularRun; + }; + + // Type trait for the instrumented test runner + template<> + struct TestJobRunnerTrait + { + using TestEngineJobType = TestEngineInstrumentedRun; + }; + + // Functor for handling test job runner callbacks + template + class TestJobRunnerCallbackHandler + { + using IdType = typename TestJobRunner::JobInfo::IdType; + using JobInfo = typename TestJobRunner::JobInfo; + public: + TestJobRunnerCallbackHandler( + const AZStd::vector& testTargets, + TestEngineJobMap* engineJobs, + AZStd::optional* callback) + : m_testTargets(testTargets) + , m_engineJobs(engineJobs) + , m_callback(callback) + { + } + + void operator()(const typename JobInfo& jobInfo, const TestImpact::JobMeta& meta) + { + const auto id = jobInfo.GetId().m_value; + const auto& args = jobInfo.GetCommand().m_args; + const auto* target = m_testTargets[id]; + const auto result = GetClientTestRunResultForMeta(meta); + + // Place the test engine job associated with this test run into the map along with its client test run result so + // that it can be retrieved when the sequence has ended (and any associated artifacts processed) + const auto& [it, success] = m_engineJobs->emplace(id, TestEngineJob(target, args, meta, result)); + + if (m_callback->has_value()) + { + (*m_callback).value()(it->second); + } + } + + private: + const AZStd::vector& m_testTargets; + TestEngineJobMap* m_engineJobs; + AZStd::optional* m_callback; + }; + + // Helper function to compile the run type specific test engine jobs from their associated jobs and payloads + template + AZStd::vector> CompileTestEngineRuns( + const AZStd::vector& testTargets, + AZStd::vector& runnerjobs, + TestEngineJobMap&& engineJobs) + { + AZStd::vector> engineRuns; + engineRuns.reserve(testTargets.size()); + + for (auto& job : runnerjobs) + { + const auto id = job.GetJobInfo().GetId().m_value; + if (auto it = engineJobs.find(id); + it != engineJobs.end()) + { + // An entry in the test engine job map means that this job was acted upon (an attempt to execute, successful or otherwise) + auto& engineJob = it->second; + TestEngineJobType run(AZStd::move(engineJob), job.ReleasePayload()); + engineRuns.push_back(AZStd::move(run)); + } + else + { + // No entry in the test engine job map means that this job never had the opportunity to be acted upon (the sequence + // was terminated whilst this job was still queued up for execution) + const auto& args = job.GetJobInfo().GetCommand().m_args; + const auto* target = testTargets[id]; + TestEngineJobType run(TestEngineJob(target, args, {}, Client::TestRunResult::NotRun), {}); + engineRuns.push_back(AZStd::move(run)); + } + } + + return engineRuns; + } + + Bitwise::TestJobExceptionPolicy GetTestJobExceptionPolicy( + Policy::ExecutionFailure executionFailurePolicy, Policy::TestFailure testFailurePolicy) + { + auto jobExecutionPolicy = Bitwise::TestJobExceptionPolicy::Never; + if (executionFailurePolicy == Policy::ExecutionFailure::Abort) + { + jobExecutionPolicy |= Bitwise::TestJobExceptionPolicy::OnFailedToExecute; + } + + if (testFailurePolicy == Policy::TestFailure::Abort) + { + jobExecutionPolicy |= Bitwise::TestJobExceptionPolicy::OnExecutedWithFailure; + } + + return jobExecutionPolicy; + } + } + + TestEngine::TestEngine( + const RepoPath& sourceDir, + const RepoPath& targetBinaryDir, + const RepoPath& cacheDir, + const RepoPath& artifactDir, + const RepoPath& testRunnerBinary, + const RepoPath& instrumentBinary, + size_t maxConcurrentRuns) + : m_maxConcurrentRuns(maxConcurrentRuns) + , m_testJobInfoGenerator(AZStd::make_unique( + sourceDir, targetBinaryDir, cacheDir, artifactDir, testRunnerBinary, instrumentBinary)) + , m_testEnumerator(AZStd::make_unique(maxConcurrentRuns)) + , m_instrumentedTestRunner(AZStd::make_unique(maxConcurrentRuns)) + , m_testRunner(AZStd::make_unique(maxConcurrentRuns)) + { + } + + TestEngine::~TestEngine() = default; + + AZStd::pair> TestEngine::UpdateEnumerationCache( + const AZStd::vector& testTargets, + Policy::ExecutionFailure executionFailurePolicy, + AZStd::optional testTargetTimeout, + AZStd::optional globalTimeout, + AZStd::optional callback) + { + TestEngineJobMap engineJobs; + const auto jobInfos = m_testJobInfoGenerator->GenerateTestEnumerationJobInfos(testTargets, TestEnumerator::JobInfo::CachePolicy::Write); + + const auto jobExecutionPolicy = executionFailurePolicy == Policy::ExecutionFailure::Abort + ? (TestEnumerator::JobExceptionPolicy::OnExecutedWithFailure | TestEnumerator::JobExceptionPolicy::OnFailedToExecute) + : TestEnumerator::JobExceptionPolicy::Never; + + auto [result, runnerJobs] = m_testEnumerator->Enumerate( + jobInfos, + TestEnumerator::CacheExceptionPolicy::OnCacheWriteFailure, + jobExecutionPolicy, + testTargetTimeout, + globalTimeout, + TestJobRunnerCallbackHandler(testTargets, &engineJobs, &callback)); + + auto engineRuns = CompileTestEngineRuns(testTargets, runnerJobs, AZStd::move(engineJobs)); + return { CalculateSequenceResult(result, engineRuns, executionFailurePolicy), AZStd::move(engineRuns) }; + } + + AZStd::pair> TestEngine::RegularRun( + const AZStd::vector& testTargets, + [[maybe_unused]]Policy::TestSharding testShardingPolicy, + Policy::ExecutionFailure executionFailurePolicy, + Policy::TestFailure testFailurePolicy, + [[maybe_unused]]Policy::TargetOutputCapture targetOutputCapture, + AZStd::optional testTargetTimeout, + AZStd::optional globalTimeout, + AZStd::optional callback) + { + TestEngineJobMap engineJobs; + const auto jobInfos = m_testJobInfoGenerator->GenerateRegularTestRunJobInfos(testTargets); + + TestJobRunnerCallbackHandler jobCallback(testTargets, &engineJobs, &callback); + auto [result, runnerJobs] = m_testRunner->RunTests( + jobInfos, + GetTestJobExceptionPolicy(executionFailurePolicy, testFailurePolicy), + testTargetTimeout, + globalTimeout, + jobCallback); + + auto engineRuns = CompileTestEngineRuns(testTargets, runnerJobs, AZStd::move(engineJobs)); + return { CalculateSequenceResult(result, engineRuns, executionFailurePolicy), AZStd::move(engineRuns) }; + } + + AZStd::pair> TestEngine::InstrumentedRun( + const AZStd::vector& testTargets, + [[maybe_unused]] Policy::TestSharding testShardingPolicy, + Policy::ExecutionFailure executionFailurePolicy, + Policy::IntegrityFailure integrityFailurePolicy, + Policy::TestFailure testFailurePolicy, + [[maybe_unused]]Policy::TargetOutputCapture targetOutputCapture, + AZStd::optional testTargetTimeout, + AZStd::optional globalTimeout, + AZStd::optional callback) + { + TestEngineJobMap engineJobs; + const auto jobInfos = m_testJobInfoGenerator->GenerateInstrumentedTestRunJobInfos(testTargets, CoverageLevel::Source); + + auto [result, runnerJobs] = m_instrumentedTestRunner->RunInstrumentedTests( + jobInfos, + GetTestJobExceptionPolicy(executionFailurePolicy, testFailurePolicy), + testTargetTimeout, + globalTimeout, + TestJobRunnerCallbackHandler(testTargets, &engineJobs, &callback)); + + auto engineRuns = CompileTestEngineRuns(testTargets, runnerJobs, AZStd::move(engineJobs)); + + // Now that we know the true result of successful jobs that return non-zero we can deduce if we have any integrity failures + // where a test target ran and completed its tests without incident yet failed to produce coverage data + if (integrityFailurePolicy == Policy::IntegrityFailure::Abort) + { + for (const auto& engineRun : engineRuns) + { + if (const auto testResult = engineRun.GetTestResult(); + testResult == Client::TestRunResult::AllTestsPass || testResult == Client::TestRunResult::TestFailures) + { + AZ_TestImpact_Eval(engineRun.GetTestCoverge().has_value(), TestEngineException, AZStd::string::format( + "Test target %s completed its test run but failed to produce coverage data", engineRun.GetTestTarget()->GetName().c_str())); + } + } + } + + return { CalculateSequenceResult(result, engineRuns, executionFailurePolicy), AZStd::move(engineRuns) }; + } +} // namespace TestImpact diff --git a/Code/Tools/TestImpactFramework/Runtime/Code/Source/TestEngine/TestImpactTestEngine.h b/Code/Tools/TestImpactFramework/Runtime/Code/Source/TestEngine/TestImpactTestEngine.h new file mode 100644 index 0000000000..f6fb6cec11 --- /dev/null +++ b/Code/Tools/TestImpactFramework/Runtime/Code/Source/TestEngine/TestImpactTestEngine.h @@ -0,0 +1,126 @@ +/* + * All or portions of this file Copyright (c) Amazon.com, Inc. or its affiliates or + * its licensors. + * + * For complete copyright and license terms please see the LICENSE at the root of this + * distribution (the "License"). All use of this software is governed by the License, + * or, if provided, by the license below or the license accompanying this file. Do not + * remove or modify any license notices. This file is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * + */ + +#pragma once + +#include +#include +#include + +#include +#include +#include + +#include +#include + +namespace TestImpact +{ + class TestTarget; + class TestJobInfoGenerator; + class TestEnumerator; + class InstrumentedTestRunner; + class TestRunner; + + //! Callback for when a given test engine job completes. + using TestEngineJobCompleteCallback = AZStd::function; + + //! Provides the front end for performing test enumerations and test runs. + class TestEngine + { + public: + //! Configures the test engine with the necessary path information for launching test targets and managing the artifacts they produce. + //! @param sourceDir Root path where source files are found (including subfolders). + //! @param targetBinaryDir Path to where the test target binaries are found. + //! @param cacheDir Path to the persistent folder where test target enumerations are cached. + //! @param artifactDir Path to the transient directory where test artifacts are produced. + //! @param testRunnerBinary Path to the binary responsible for launching test targets that have the TestRunner launch method. + //! @param instrumentBinary Path to the binary responsible for launching test targets with test coverage instrumentation. + //! @param maxConcurrentRuns The maximum number of concurrent test targets that can be in flight at any given moment. + TestEngine( + const RepoPath& sourceDir, + const RepoPath& targetBinaryDir, + const RepoPath& cacheDir, + const RepoPath& artifactDir, + const RepoPath& testRunnerBinary, + const RepoPath& instrumentBinary, + size_t maxConcurrentRuns); + + ~TestEngine(); + + //! Updates the cached enumerations for the specified test targets. + //! @note Whilst test runs will make use of this cache for test target sharding it is the responsibility of the client to + //! ensure any stale caches are up to date by calling this function. No attempt to maintain internal consistency will be made + //! by the test engine itself. + //! @param testTargets The test targets to enumerate. + //! @param executionFailurePolicy The policy for how enumeration execution failures should be handled. + //! @param testTargetTimeout The maximum duration a test target may be in-flight for before being forcefully terminated (infinite if empty). + //! @param globalTimeout The maximum duration the enumeration sequence may run before being forcefully terminated (infinite if empty). + //! @param callback The client callback function to handle completed test target enumerations. + //! @ returns The sequence result and the enumerations for the target that were enumerated. + AZStd::pair < TestSequenceResult, AZStd::vector> UpdateEnumerationCache( + const AZStd::vector& testTargets, + Policy::ExecutionFailure executionFailurePolicy, + AZStd::optional testTargetTimeout, + AZStd::optional globalTimeout, + AZStd::optional callback); + + //! Performs a test run without any instrumentation and, for each test target, returns the test run results and metrics about the run. + //! @param testTargets The test targets to run. + //! @param testShardingPolicy Test sharding policy to use for test targets in this run. + //! @param executionFailurePolicy Policy for how test execution failures should be handled. + //! @param testFailurePolicy Policy for how test targets with failing tests should be handled. + //! @param targetOutputCapture Policy for how test target standard output should be captured and handled. + //! @param testTargetTimeout The maximum duration a test target may be in-flight for before being forcefully terminated (infinite if empty). + //! @param globalTimeout The maximum duration the enumeration sequence may run before being forcefully terminated (infinite if empty). + //! @param callback The client callback function to handle completed test target runs. + //! @ returns The sequence result and the test run results for the test targets that were run. + [[nodiscard]]AZStd::pair> RegularRun( + const AZStd::vector& testTargets, + Policy::TestSharding testShardingPolicy, + Policy::ExecutionFailure executionFailurePolicy, + Policy::TestFailure testFailurePolicy, + Policy::TargetOutputCapture targetOutputCapture, + AZStd::optional testTargetTimeout, + AZStd::optional globalTimeout, + AZStd::optional callback); + + //! Performs a test run with instrumentation and, for each test target, returns the test run results, coverage data and metrics about the run. + //! @param testTargets The test targets to run. + //! @param testShardingPolicy Test sharding policy to use for test targets in this run. + //! @param executionFailurePolicy Policy for how test execution failures should be handled. + //! @param integrityFailurePolicy Policy for how integrty failures of the test impact data and source tree model should be handled. + //! @param testFailurePolicy Policy for how test targets with failing tests should be handled. + //! @param targetOutputCapture Policy for how test target standard output should be captured and handled. + //! @param testTargetTimeout The maximum duration a test target may be in-flight for before being forcefully terminated (infinite if empty). + //! @param globalTimeout The maximum duration the enumeration sequence may run before being forcefully terminated (infinite if empty). + //! @param callback The client callback function to handle completed test target runs. + //! @ returns The sequence result and the test run results and test coverages for the test targets that were run. + [[nodiscard]]AZStd::pair> InstrumentedRun( + const AZStd::vector& testTargets, + Policy::TestSharding testShardingPolicy, + Policy::ExecutionFailure executionFailurePolicy, + Policy::IntegrityFailure integrityFailurePolicy, + Policy::TestFailure testFailurePolicy, + Policy::TargetOutputCapture targetOutputCapture, + AZStd::optional testTargetTimeout, + AZStd::optional globalTimeout, + AZStd::optional callback); + + private: + size_t m_maxConcurrentRuns = 0; + AZStd::unique_ptr m_testJobInfoGenerator; + AZStd::unique_ptr m_testEnumerator; + AZStd::unique_ptr m_instrumentedTestRunner; + AZStd::unique_ptr m_testRunner; + }; +} // namespace TestImpact diff --git a/Code/Tools/TestImpactFramework/Runtime/Code/Source/TestEngine/TestImpactTestEngineEnumeration.cpp b/Code/Tools/TestImpactFramework/Runtime/Code/Source/TestEngine/TestImpactTestEngineEnumeration.cpp new file mode 100644 index 0000000000..3c114d1b33 --- /dev/null +++ b/Code/Tools/TestImpactFramework/Runtime/Code/Source/TestEngine/TestImpactTestEngineEnumeration.cpp @@ -0,0 +1,27 @@ +/* + * All or portions of this file Copyright (c) Amazon.com, Inc. or its affiliates or + * its licensors. + * + * For complete copyright and license terms please see the LICENSE at the root of this + * distribution (the "License"). All use of this software is governed by the License, + * or, if provided, by the license below or the license accompanying this file. Do not + * remove or modify any license notices. This file is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * + */ + +#include + +namespace TestImpact +{ + TestEngineEnumeration::TestEngineEnumeration(TestEngineJob&& job, AZStd::optional&& enumeration) + : TestEngineJob(AZStd::move(job)) + , m_enumeration(AZStd::move(enumeration)) + { + } + + const AZStd::optional& TestEngineEnumeration::GetTestEnumeration() const + { + return m_enumeration; + } +} // namespace TestImpact diff --git a/Code/Tools/TestImpactFramework/Runtime/Code/Source/TestEngine/TestImpactTestEngineEnumeration.h b/Code/Tools/TestImpactFramework/Runtime/Code/Source/TestEngine/TestImpactTestEngineEnumeration.h new file mode 100644 index 0000000000..28473a08bc --- /dev/null +++ b/Code/Tools/TestImpactFramework/Runtime/Code/Source/TestEngine/TestImpactTestEngineEnumeration.h @@ -0,0 +1,32 @@ +/* + * All or portions of this file Copyright (c) Amazon.com, Inc. or its affiliates or + * its licensors. + * + * For complete copyright and license terms please see the LICENSE at the root of this + * distribution (the "License"). All use of this software is governed by the License, + * or, if provided, by the license below or the license accompanying this file. Do not + * remove or modify any license notices. This file is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * + */ + +#pragma once + +#include +#include + +namespace TestImpact +{ + //! Represents the generated test enumeration data for a test engine enumeration. + class TestEngineEnumeration + : public TestEngineJob + { + public: + TestEngineEnumeration(TestEngineJob&& job, AZStd::optional&& enumeration); + + //! Returns the test enumeration payload for this job (if any). + const AZStd::optional& GetTestEnumeration() const; + private: + AZStd::optional m_enumeration; + }; +} // namespace TestImpact diff --git a/Code/Tools/TestImpactFramework/Runtime/Code/Source/TestEngine/Run/TestImpactTestRunException.h b/Code/Tools/TestImpactFramework/Runtime/Code/Source/TestEngine/TestImpactTestEngineException.h similarity index 86% rename from Code/Tools/TestImpactFramework/Runtime/Code/Source/TestEngine/Run/TestImpactTestRunException.h rename to Code/Tools/TestImpactFramework/Runtime/Code/Source/TestEngine/TestImpactTestEngineException.h index ec13a17f57..cf10b68b24 100644 --- a/Code/Tools/TestImpactFramework/Runtime/Code/Source/TestEngine/Run/TestImpactTestRunException.h +++ b/Code/Tools/TestImpactFramework/Runtime/Code/Source/TestEngine/TestImpactTestEngineException.h @@ -16,8 +16,8 @@ namespace TestImpact { - //! Exception for test runs and test run related operations. - class TestRunException : public Exception + //! Exception for test engine runs and related operations. + class TestEngineException : public Exception { public: using Exception::Exception; diff --git a/Code/Tools/TestImpactFramework/Runtime/Code/Source/TestEngine/TestImpactTestEngineInstrumentedRun.cpp b/Code/Tools/TestImpactFramework/Runtime/Code/Source/TestEngine/TestImpactTestEngineInstrumentedRun.cpp new file mode 100644 index 0000000000..b19ec723d1 --- /dev/null +++ b/Code/Tools/TestImpactFramework/Runtime/Code/Source/TestEngine/TestImpactTestEngineInstrumentedRun.cpp @@ -0,0 +1,52 @@ +/* + * All or portions of this file Copyright (c) Amazon.com, Inc. or its affiliates or + * its licensors. + * + * For complete copyright and license terms please see the LICENSE at the root of this + * distribution (the "License"). All use of this software is governed by the License, + * or, if provided, by the license below or the license accompanying this file. Do not + * remove or modify any license notices. This file is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * + */ + +#pragma once + +#include + +namespace TestImpact +{ + namespace + { + AZStd::optional ReleaseTestRun(AZStd::optional>&& testRunAndCoverage) + { + if (testRunAndCoverage.has_value()) + { + return AZStd::move(testRunAndCoverage.value().first); + } + + return AZStd::nullopt; + } + + AZStd::optional ReleaseTestCoverage(AZStd::optional>&& testRunAndCoverage) + { + if (testRunAndCoverage.has_value()) + { + return AZStd::move(testRunAndCoverage.value().second); + } + + return AZStd::nullopt; + } + } + + TestEngineInstrumentedRun::TestEngineInstrumentedRun(TestEngineJob&& testJob, AZStd::optional>&& testRunAndCoverage) + : TestEngineRegularRun(AZStd::move(testJob), ReleaseTestRun(AZStd::move(testRunAndCoverage))) + , m_testCoverage(ReleaseTestCoverage(AZStd::move(testRunAndCoverage))) + { + } + + const AZStd::optional& TestEngineInstrumentedRun::GetTestCoverge() const + { + return m_testCoverage; + } +} // namespace TestImpact diff --git a/Code/Tools/TestImpactFramework/Runtime/Code/Source/TestEngine/TestImpactTestEngineInstrumentedRun.h b/Code/Tools/TestImpactFramework/Runtime/Code/Source/TestEngine/TestImpactTestEngineInstrumentedRun.h new file mode 100644 index 0000000000..b1edd943a4 --- /dev/null +++ b/Code/Tools/TestImpactFramework/Runtime/Code/Source/TestEngine/TestImpactTestEngineInstrumentedRun.h @@ -0,0 +1,32 @@ +/* + * All or portions of this file Copyright (c) Amazon.com, Inc. or its affiliates or + * its licensors. + * + * For complete copyright and license terms please see the LICENSE at the root of this + * distribution (the "License"). All use of this software is governed by the License, + * or, if provided, by the license below or the license accompanying this file. Do not + * remove or modify any license notices. This file is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * + */ + +#pragma once + +#include +#include + +namespace TestImpact +{ + //! Represents the generated test run and coverage data for an instrumented regular test engine run. + class TestEngineInstrumentedRun + : public TestEngineRegularRun + { + public: + TestEngineInstrumentedRun(TestEngineJob&& testJob, AZStd::optional>&& testRunAndCoverage); + + //! Returns the test coverage payload for this job (if any). + const AZStd::optional& GetTestCoverge() const; + private: + AZStd::optional m_testCoverage; + }; +} // namespace TestImpact diff --git a/Code/Tools/TestImpactFramework/Runtime/Code/Source/TestEngine/TestImpactTestEngineJob.cpp b/Code/Tools/TestImpactFramework/Runtime/Code/Source/TestEngine/TestImpactTestEngineJob.cpp new file mode 100644 index 0000000000..26aa71adcd --- /dev/null +++ b/Code/Tools/TestImpactFramework/Runtime/Code/Source/TestEngine/TestImpactTestEngineJob.cpp @@ -0,0 +1,40 @@ +/* + * All or portions of this file Copyright (c) Amazon.com, Inc. or its affiliates or + * its licensors. + * + * For complete copyright and license terms please see the LICENSE at the root of this + * distribution (the "License"). All use of this software is governed by the License, + * or, if provided, by the license below or the license accompanying this file. Do not + * remove or modify any license notices. This file is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * + */ + +#include +#include + +namespace TestImpact +{ + TestEngineJob::TestEngineJob(const TestTarget* testTarget, const AZStd::string& commandString, const JobMeta& jobMeta, Client::TestRunResult testResult) + : JobMetaWrapper(jobMeta) + , m_testTarget(testTarget) + , m_commandString(commandString) + , m_testResult(testResult) + { + } + + const TestTarget* TestEngineJob::GetTestTarget() const + { + return m_testTarget; + } + + const AZStd::string& TestEngineJob::GetCommandString() const + { + return m_commandString; + } + + Client::TestRunResult TestEngineJob::GetTestResult() const + { + return m_testResult; + } +} // namespace TestImpact diff --git a/Code/Tools/TestImpactFramework/Runtime/Code/Source/TestEngine/TestImpactTestEngineJob.h b/Code/Tools/TestImpactFramework/Runtime/Code/Source/TestEngine/TestImpactTestEngineJob.h new file mode 100644 index 0000000000..00d0c7d099 --- /dev/null +++ b/Code/Tools/TestImpactFramework/Runtime/Code/Source/TestEngine/TestImpactTestEngineJob.h @@ -0,0 +1,41 @@ +/* + * All or portions of this file Copyright (c) Amazon.com, Inc. or its affiliates or + * its licensors. + * + * For complete copyright and license terms please see the LICENSE at the root of this + * distribution (the "License"). All use of this software is governed by the License, + * or, if provided, by the license below or the license accompanying this file. Do not + * remove or modify any license notices. This file is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * + */ + +#pragma once + +#include + +namespace TestImpact +{ + class TestTarget; + + //! Represents the meta-data describing a test engine run. + class TestEngineJob + : public JobMetaWrapper + { + public: + TestEngineJob(const TestTarget* testTarget, const AZStd::string& commandString, const JobMeta& jobMeta, Client::TestRunResult testResult); + + //! Returns the test target that was run for this job. + const TestTarget* GetTestTarget() const; + + //! Returns the result of the job that was run. + Client::TestRunResult GetTestResult() const; + + //! Returns the command string that was used to execute this job. + const AZStd::string& GetCommandString() const; + private: + const TestTarget* m_testTarget; + AZStd::string m_commandString; + Client::TestRunResult m_testResult; + }; +} // namespace TestImpact diff --git a/Code/Tools/TestImpactFramework/Runtime/Code/Source/TestEngine/Enumeration/TestImpactTestEnumerationException.h b/Code/Tools/TestImpactFramework/Runtime/Code/Source/TestEngine/TestImpactTestEngineRegularRun.cpp similarity index 61% rename from Code/Tools/TestImpactFramework/Runtime/Code/Source/TestEngine/Enumeration/TestImpactTestEnumerationException.h rename to Code/Tools/TestImpactFramework/Runtime/Code/Source/TestEngine/TestImpactTestEngineRegularRun.cpp index 0c9d27df7a..0090689e8b 100644 --- a/Code/Tools/TestImpactFramework/Runtime/Code/Source/TestEngine/Enumeration/TestImpactTestEnumerationException.h +++ b/Code/Tools/TestImpactFramework/Runtime/Code/Source/TestEngine/TestImpactTestEngineRegularRun.cpp @@ -10,17 +10,18 @@ * */ -#pragma once - -#include +#include namespace TestImpact { - //! Exception for test enumerations and test enumeration related operations. - class TestEnumerationException - : public Exception + TestEngineRegularRun::TestEngineRegularRun(TestEngineJob&& testJob, AZStd::optional&& testRun) + : TestEngineJob(AZStd::move(testJob)) + , m_testRun(AZStd::move(testRun)) { - public: - using Exception::Exception; - }; + } + + const AZStd::optional& TestEngineRegularRun::GetTestRun() const + { + return m_testRun; + } } // namespace TestImpact diff --git a/Code/Tools/TestImpactFramework/Runtime/Code/Source/TestEngine/TestImpactTestEngineRegularRun.h b/Code/Tools/TestImpactFramework/Runtime/Code/Source/TestEngine/TestImpactTestEngineRegularRun.h new file mode 100644 index 0000000000..3c65830094 --- /dev/null +++ b/Code/Tools/TestImpactFramework/Runtime/Code/Source/TestEngine/TestImpactTestEngineRegularRun.h @@ -0,0 +1,34 @@ +/* + * All or portions of this file Copyright (c) Amazon.com, Inc. or its affiliates or + * its licensors. + * + * For complete copyright and license terms please see the LICENSE at the root of this + * distribution (the "License"). All use of this software is governed by the License, + * or, if provided, by the license below or the license accompanying this file. Do not + * remove or modify any license notices. This file is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * + */ + +#pragma once + +#include + +#include +#include + +namespace TestImpact +{ + //! Represents the generated test run data for a regular test engine run. + class TestEngineRegularRun + : public TestEngineJob + { + public: + TestEngineRegularRun(TestEngineJob&& testJob, AZStd::optional&& testRun); + + //! Returns the test run payload for this job (if any). + const AZStd::optional& GetTestRun() const; + private: + AZStd::optional m_testRun; + }; +} // namespace TestImpact