diff --git a/Code/Tools/TestImpactFramework/Runtime/Code/Source/Artifact/Dynamic/TestImpactCoverage.h b/Code/Tools/TestImpactFramework/Runtime/Code/Source/Artifact/Dynamic/TestImpactCoverage.h index 54b78d682e..4e1c435841 100644 --- a/Code/Tools/TestImpactFramework/Runtime/Code/Source/Artifact/Dynamic/TestImpactCoverage.h +++ b/Code/Tools/TestImpactFramework/Runtime/Code/Source/Artifact/Dynamic/TestImpactCoverage.h @@ -12,6 +12,8 @@ #pragma once +#include + #include #include @@ -27,14 +29,14 @@ namespace TestImpact //! Coverage information about a particular source file. struct SourceCoverage { - AZStd::string m_path; //!< Source file path. + RepoPath m_path; //!< Source file path. AZStd::vector m_coverage; //!< Source file line coverage (empty if source level coverage only). }; //! Coverage information about a particular module (executable, shared library). struct ModuleCoverage { - AZStd::string m_path; //!< Module path. + RepoPath m_path; //!< Module path. AZStd::vector m_sources; //!< Sources of this module that are covered. }; } // namespace TestImpact diff --git a/Code/Tools/TestImpactFramework/Runtime/Code/Source/Test/Job/TestImpactTestJobCommon.h b/Code/Tools/TestImpactFramework/Runtime/Code/Source/Test/Job/TestImpactTestJobCommon.h deleted file mode 100644 index 03bbd91454..0000000000 --- a/Code/Tools/TestImpactFramework/Runtime/Code/Source/Test/Job/TestImpactTestJobCommon.h +++ /dev/null @@ -1,38 +0,0 @@ -/* - * All or portions of this file Copyright (c) Amazon.com, Inc. or its affiliates or - * its licensors. - * - * For complete copyright and license terms please see the LICENSE at the root of this - * distribution (the "License"). All use of this software is governed by the License, - * or, if provided, by the license below or the license accompanying this file. Do not - * remove or modify any license notices. This file is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * - */ - -#include - -#include -#include -#include -#include - -#pragma once - -namespace TestImpact -{ - template - AZStd::string ReadFileContents(const AZ::IO::Path& file) - { - static_assert(AZStd::is_base_of::value, "Exception must be a TestImpact exception or derived type"); - - const auto fileSize = AZ::IO::SystemFile::Length(file.c_str()); - AZ_TestImpact_Eval(fileSize > 0, ExceptionType, AZStd::string::format("File %s does not exist", file.c_str())); - - AZStd::vector buffer(fileSize + 1); - buffer[fileSize] = '\0'; - AZ_TestImpact_Eval(AZ::IO::SystemFile::Read(file.c_str(), buffer.data()), ExceptionType, "Could not read file contents"); - - return AZStd::string(buffer.begin(), buffer.end()); - } -} // namespace TestImpact diff --git a/Code/Tools/TestImpactFramework/Runtime/Code/Source/Test/Job/TestImpactTestJobException.h b/Code/Tools/TestImpactFramework/Runtime/Code/Source/Test/Job/TestImpactTestJobException.h deleted file mode 100644 index 263d70322e..0000000000 --- a/Code/Tools/TestImpactFramework/Runtime/Code/Source/Test/Job/TestImpactTestJobException.h +++ /dev/null @@ -1,26 +0,0 @@ -/* - * All or portions of this file Copyright (c) Amazon.com, Inc. or its affiliates or - * its licensors. - * - * For complete copyright and license terms please see the LICENSE at the root of this - * distribution (the "License"). All use of this software is governed by the License, - * or, if provided, by the license below or the license accompanying this file. Do not - * remove or modify any license notices. This file is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * - */ - -#pragma once - -#include - -namespace TestImpact -{ - //! Exception for test job related operations. - class TestJobException - : public Exception - { - public: - using Exception::Exception; - }; -} // namespace TestImpact diff --git a/Code/Tools/TestImpactFramework/Runtime/Code/Source/Test/Run/TestImpactTestRun.cpp b/Code/Tools/TestImpactFramework/Runtime/Code/Source/Test/Run/TestImpactTestRun.cpp deleted file mode 100644 index 1ba2a3e849..0000000000 --- a/Code/Tools/TestImpactFramework/Runtime/Code/Source/Test/Run/TestImpactTestRun.cpp +++ /dev/null @@ -1,70 +0,0 @@ -/* - * All or portions of this file Copyright (c) Amazon.com, Inc. or its affiliates or - * its licensors. - * - * For complete copyright and license terms please see the LICENSE at the root of this - * distribution (the "License"). All use of this software is governed by the License, - * or, if provided, by the license below or the license accompanying this file. Do not - * remove or modify any license notices. This file is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * - */ - -#include "TestImpactTestRun.h" - -namespace TestImpact -{ - TestRun::TestRun(AZStd::vector&& testSuites, AZStd::chrono::milliseconds duration) - : TestSuiteContainer(AZStd::move(testSuites)) - , m_duration(duration) - { - for (const auto& suite : m_testSuites) - { - for (const auto& test : suite.m_tests) - { - if (test.m_status == TestRunStatus::Run) - { - m_numRuns++; - - if (test.m_result.value() == TestRunResult::Passed) - { - m_numPasses++; - } - else - { - m_numFailures++; - } - } - else - { - m_numNotRuns++; - } - } - } - } - - size_t TestRun::GetNumRuns() const - { - return m_numRuns; - } - - size_t TestRun::GetNumNotRuns() const - { - return m_numNotRuns; - } - - size_t TestRun::GetNumPasses() const - { - return m_numPasses; - } - - size_t TestRun::GetNumFailures() const - { - return m_numFailures; - } - - AZStd::chrono::milliseconds TestRun::GetDuration() const - { - return m_duration; - } -} // namespace TestImpact diff --git a/Code/Tools/TestImpactFramework/Runtime/Code/Source/Test/Enumeration/TestImpactTestEnumeration.h b/Code/Tools/TestImpactFramework/Runtime/Code/Source/TestEngine/Enumeration/TestImpactTestEnumeration.h similarity index 93% rename from Code/Tools/TestImpactFramework/Runtime/Code/Source/Test/Enumeration/TestImpactTestEnumeration.h rename to Code/Tools/TestImpactFramework/Runtime/Code/Source/TestEngine/Enumeration/TestImpactTestEnumeration.h index c12abec006..709c5d27d0 100644 --- a/Code/Tools/TestImpactFramework/Runtime/Code/Source/Test/Enumeration/TestImpactTestEnumeration.h +++ b/Code/Tools/TestImpactFramework/Runtime/Code/Source/TestEngine/Enumeration/TestImpactTestEnumeration.h @@ -13,7 +13,7 @@ #pragma once #include -#include +#include namespace TestImpact { diff --git a/Code/Tools/TestImpactFramework/Runtime/Code/Source/Test/Enumeration/TestImpactTestEnumerationException.h b/Code/Tools/TestImpactFramework/Runtime/Code/Source/TestEngine/Enumeration/TestImpactTestEnumerationException.h similarity index 100% rename from Code/Tools/TestImpactFramework/Runtime/Code/Source/Test/Enumeration/TestImpactTestEnumerationException.h rename to Code/Tools/TestImpactFramework/Runtime/Code/Source/TestEngine/Enumeration/TestImpactTestEnumerationException.h diff --git a/Code/Tools/TestImpactFramework/Runtime/Code/Source/Test/Enumeration/TestImpactTestEnumerationSerializer.cpp b/Code/Tools/TestImpactFramework/Runtime/Code/Source/TestEngine/Enumeration/TestImpactTestEnumerationSerializer.cpp similarity index 96% rename from Code/Tools/TestImpactFramework/Runtime/Code/Source/Test/Enumeration/TestImpactTestEnumerationSerializer.cpp rename to Code/Tools/TestImpactFramework/Runtime/Code/Source/TestEngine/Enumeration/TestImpactTestEnumerationSerializer.cpp index efed04970d..05a13a0703 100644 --- a/Code/Tools/TestImpactFramework/Runtime/Code/Source/Test/Enumeration/TestImpactTestEnumerationSerializer.cpp +++ b/Code/Tools/TestImpactFramework/Runtime/Code/Source/TestEngine/Enumeration/TestImpactTestEnumerationSerializer.cpp @@ -10,8 +10,8 @@ * */ -#include -#include +#include +#include #include #include diff --git a/Code/Tools/TestImpactFramework/Runtime/Code/Source/Test/Enumeration/TestImpactTestEnumerationSerializer.h b/Code/Tools/TestImpactFramework/Runtime/Code/Source/TestEngine/Enumeration/TestImpactTestEnumerationSerializer.h similarity index 94% rename from Code/Tools/TestImpactFramework/Runtime/Code/Source/Test/Enumeration/TestImpactTestEnumerationSerializer.h rename to Code/Tools/TestImpactFramework/Runtime/Code/Source/TestEngine/Enumeration/TestImpactTestEnumerationSerializer.h index 422308d862..06bd224b50 100644 --- a/Code/Tools/TestImpactFramework/Runtime/Code/Source/Test/Enumeration/TestImpactTestEnumerationSerializer.h +++ b/Code/Tools/TestImpactFramework/Runtime/Code/Source/TestEngine/Enumeration/TestImpactTestEnumerationSerializer.h @@ -12,7 +12,7 @@ #pragma once -#include +#include #include diff --git a/Code/Tools/TestImpactFramework/Runtime/Code/Source/Test/Enumeration/TestImpactTestEnumerator.cpp b/Code/Tools/TestImpactFramework/Runtime/Code/Source/TestEngine/Enumeration/TestImpactTestEnumerator.cpp similarity index 87% rename from Code/Tools/TestImpactFramework/Runtime/Code/Source/Test/Enumeration/TestImpactTestEnumerator.cpp rename to Code/Tools/TestImpactFramework/Runtime/Code/Source/TestEngine/Enumeration/TestImpactTestEnumerator.cpp index 1964f8876e..882c65c516 100644 --- a/Code/Tools/TestImpactFramework/Runtime/Code/Source/Test/Enumeration/TestImpactTestEnumerator.cpp +++ b/Code/Tools/TestImpactFramework/Runtime/Code/Source/TestEngine/Enumeration/TestImpactTestEnumerator.cpp @@ -10,11 +10,13 @@ * */ +#include + #include -#include -#include -#include -#include +#include +#include +#include +#include #include @@ -23,7 +25,7 @@ namespace TestImpact namespace { void WriteCacheFile( - const TestEnumeration& enumeration, const AZ::IO::Path& path, Bitwise::CacheExceptionPolicy cacheExceptionPolicy) + const TestEnumeration& enumeration, const RepoPath& path, Bitwise::CacheExceptionPolicy cacheExceptionPolicy) { const AZStd::string cacheJSON = SerializeTestEnumeration(enumeration); const AZStd::vector cacheBytes(cacheJSON.begin(), cacheJSON.end()); @@ -48,7 +50,7 @@ namespace TestImpact } } - AZStd::optional ReadCacheFile(const AZ::IO::Path& path, Bitwise::CacheExceptionPolicy cacheExceptionPolicy) + AZStd::optional ReadCacheFile(const RepoPath& path, Bitwise::CacheExceptionPolicy cacheExceptionPolicy) { AZ::IO::SystemFile cacheFile; AZStd::string cacheJSON; @@ -83,18 +85,18 @@ namespace TestImpact } } // namespace - TestEnumeration ParseTestEnumerationFile(const AZ::IO::Path& enumerationFile) + TestEnumeration ParseTestEnumerationFile(const RepoPath& enumerationFile) { return TestEnumeration(GTest::TestEnumerationSuitesFactory(ReadFileContents(enumerationFile))); } - TestEnumerationJobData::TestEnumerationJobData(const AZ::IO::Path& enumerationArtifact, AZStd::optional&& cache) + TestEnumerationJobData::TestEnumerationJobData(const RepoPath& enumerationArtifact, AZStd::optional&& cache) : m_enumerationArtifact(enumerationArtifact) , m_cache(AZStd::move(cache)) { } - const AZ::IO::Path& TestEnumerationJobData::GetEnumerationArtifactPath() const + const RepoPath& TestEnumerationJobData::GetEnumerationArtifactPath() const { return m_enumerationArtifact; } @@ -163,7 +165,7 @@ namespace TestImpact const auto& [meta, jobInfo] = jobData; if (meta.m_result == JobResult::ExecutedWithSuccess) { - const auto& enumeration = enumerations[jobId] = ParseTestEnumerationFile(jobInfo->GetEnumerationArtifactPath()); + const auto& enumeration = (enumerations[jobId] = ParseTestEnumerationFile(jobInfo->GetEnumerationArtifactPath())); // Write out the enumeration to a cache file if we have a cache write policy for this job if (jobInfo->GetCache().has_value() && jobInfo->GetCache()->m_policy == JobData::CachePolicy::Write) diff --git a/Code/Tools/TestImpactFramework/Runtime/Code/Source/Test/Enumeration/TestImpactTestEnumerator.h b/Code/Tools/TestImpactFramework/Runtime/Code/Source/TestEngine/Enumeration/TestImpactTestEnumerator.h similarity index 90% rename from Code/Tools/TestImpactFramework/Runtime/Code/Source/Test/Enumeration/TestImpactTestEnumerator.h rename to Code/Tools/TestImpactFramework/Runtime/Code/Source/TestEngine/Enumeration/TestImpactTestEnumerator.h index a005a32b0d..63d0b82d3e 100644 --- a/Code/Tools/TestImpactFramework/Runtime/Code/Source/Test/Enumeration/TestImpactTestEnumerator.h +++ b/Code/Tools/TestImpactFramework/Runtime/Code/Source/TestEngine/Enumeration/TestImpactTestEnumerator.h @@ -12,11 +12,11 @@ #pragma once -#include -#include -#include +#include -#include +#include +#include +#include namespace TestImpact { @@ -35,19 +35,19 @@ namespace TestImpact struct Cache { CachePolicy m_policy; - AZ::IO::Path m_file; + RepoPath m_file; }; - TestEnumerationJobData(const AZ::IO::Path& enumerationArtifact, AZStd::optional&& cache); + TestEnumerationJobData(const RepoPath& enumerationArtifact, AZStd::optional&& cache); //! Returns the path to the enumeration artifact produced by the test target. - const AZ::IO::Path& GetEnumerationArtifactPath() const; + const RepoPath& GetEnumerationArtifactPath() const; //! Returns the cache details for this job. const AZStd::optional& GetCache() const; private: - AZ::IO::Path m_enumerationArtifact; //!< Path to enumeration artifact to be processed. + RepoPath m_enumerationArtifact; //!< Path to enumeration artifact to be processed. AZStd::optional m_cache = AZStd::nullopt; //!< No caching takes place if cache is empty. }; diff --git a/Code/Tools/TestImpactFramework/Runtime/Code/Source/Test/Job/TestImpactTestJobRunner.h b/Code/Tools/TestImpactFramework/Runtime/Code/Source/TestEngine/JobRunner/TestImpactTestJobRunner.h similarity index 92% rename from Code/Tools/TestImpactFramework/Runtime/Code/Source/Test/Job/TestImpactTestJobRunner.h rename to Code/Tools/TestImpactFramework/Runtime/Code/Source/TestEngine/JobRunner/TestImpactTestJobRunner.h index a4a0b19845..c60a93e21d 100644 --- a/Code/Tools/TestImpactFramework/Runtime/Code/Source/Test/Job/TestImpactTestJobRunner.h +++ b/Code/Tools/TestImpactFramework/Runtime/Code/Source/TestEngine/JobRunner/TestImpactTestJobRunner.h @@ -12,11 +12,10 @@ #pragma once -#include +#include #include #include -#include #include #include @@ -45,6 +44,7 @@ namespace TestImpact public: using JobData = AdditionalInfo; using JobInfo = JobInfo; + using Command = typename JobInfo::Command; using JobPayload = Payload; using Job = Job; using ClientJobCallback = AZStd::function; @@ -111,20 +111,22 @@ namespace TestImpact // Callback to handle job exception policies and client/derived callbacks const auto jobCallback = [this, &jobExceptionPolicy](const JobInfo& jobInfo, const JobMeta& meta, StdContent&& std) { + auto callbackResult = ProcessCallbackResult::Continue; if (meta.m_result == JobResult::FailedToExecute && IsFlagSet(jobExceptionPolicy, JobExceptionPolicy::OnFailedToExecute)) { - throw TestJobException("Job failed to execute"); + callbackResult = ProcessCallbackResult::Abort; } else if (meta.m_result == JobResult::ExecutedWithFailure && IsFlagSet(jobExceptionPolicy, JobExceptionPolicy::OnExecutedWithFailure)) { - throw TestJobException("Job executed with failure"); + callbackResult = ProcessCallbackResult::Abort; } if (m_derivedJobCallback.has_value()) { - if (const auto result = (*m_derivedJobCallback)(jobInfo, meta, AZStd::move(std)); result != CallbackResult::Continue) + if (const auto result = (*m_derivedJobCallback)(jobInfo, meta, AZStd::move(std)); + result == ProcessCallbackResult::Abort) { - return result; + callbackResult = ProcessCallbackResult::Abort; } } @@ -133,7 +135,7 @@ namespace TestImpact (*m_clientJobCallback)(jobInfo, meta); } - return CallbackResult::Continue; + return callbackResult; }; return m_jobRunner.Execute(jobInfos, jobCallback, payloadMapProducer); diff --git a/Code/Tools/TestImpactFramework/Runtime/Code/Source/Test/Run/TestImpactInstrumentedTestRunner.cpp b/Code/Tools/TestImpactFramework/Runtime/Code/Source/TestEngine/Run/TestImpactInstrumentedTestRunner.cpp similarity index 69% rename from Code/Tools/TestImpactFramework/Runtime/Code/Source/Test/Run/TestImpactInstrumentedTestRunner.cpp rename to Code/Tools/TestImpactFramework/Runtime/Code/Source/TestEngine/Run/TestImpactInstrumentedTestRunner.cpp index 3b7a088fb4..0d27d56af1 100644 --- a/Code/Tools/TestImpactFramework/Runtime/Code/Source/Test/Run/TestImpactInstrumentedTestRunner.cpp +++ b/Code/Tools/TestImpactFramework/Runtime/Code/Source/TestEngine/Run/TestImpactInstrumentedTestRunner.cpp @@ -10,31 +10,32 @@ * */ +#include + #include #include -#include -#include -#include -#include +#include +#include +#include #include namespace TestImpact { - InstrumentedTestRunJobData::InstrumentedTestRunJobData(const AZ::IO::Path& resultsArtifact, const AZ::IO::Path& coverageArtifact) + InstrumentedTestRunJobData::InstrumentedTestRunJobData(const RepoPath& resultsArtifact, const RepoPath& coverageArtifact) : TestRunJobData(resultsArtifact) , m_coverageArtifact(coverageArtifact) { } - const AZ::IO::Path& InstrumentedTestRunJobData::GetCoverageArtifactPath() const + const RepoPath& InstrumentedTestRunJobData::GetCoverageArtifactPath() const { return m_coverageArtifact; } InstrumentedTestRunner::JobPayload ParseTestRunAndCoverageFiles( - const AZ::IO::Path& runFile, - const AZ::IO::Path& coverageFile, + const RepoPath& runFile, + const RepoPath& coverageFile, AZStd::chrono::milliseconds duration, InstrumentedTestRunner::CoverageExceptionPolicy coverageExceptionPolicy) { @@ -44,7 +45,7 @@ namespace TestImpact { AZ_TestImpact_Eval( !IsFlagSet(coverageExceptionPolicy, Bitwise::CoverageExceptionPolicy::OnEmptyCoverage), TestRunException, - "No coverage data generated"); + AZStd::string::format("No coverage data generated for '%s'", coverageFile.c_str())); } TestCoverage coverage(AZStd::move(moduleCoverages)); @@ -73,11 +74,24 @@ namespace TestImpact const auto& [meta, jobInfo] = jobData; if (meta.m_result == JobResult::ExecutedWithSuccess || meta.m_result == JobResult::ExecutedWithFailure) { - runs[jobId] = ParseTestRunAndCoverageFiles( - jobInfo->GetRunArtifactPath(), - jobInfo->GetCoverageArtifactPath(), - meta.m_duration.value(), - coverageExceptionPolicy); + try + { + runs[jobId] = ParseTestRunAndCoverageFiles( + jobInfo->GetRunArtifactPath(), + jobInfo->GetCoverageArtifactPath(), + meta.m_duration.value(), + coverageExceptionPolicy); + } + catch (const Exception& e) + { + AZ_Warning("RunInstrumentedTests", false, e.what()); + runs[jobId] = AZStd::nullopt; + + if (coverageExceptionPolicy == CoverageExceptionPolicy::OnEmptyCoverage) + { + break; + } + } } } diff --git a/Code/Tools/TestImpactFramework/Runtime/Code/Source/Test/Run/TestImpactInstrumentedTestRunner.h b/Code/Tools/TestImpactFramework/Runtime/Code/Source/TestEngine/Run/TestImpactInstrumentedTestRunner.h similarity index 82% rename from Code/Tools/TestImpactFramework/Runtime/Code/Source/Test/Run/TestImpactInstrumentedTestRunner.h rename to Code/Tools/TestImpactFramework/Runtime/Code/Source/TestEngine/Run/TestImpactInstrumentedTestRunner.h index e6f059b5d3..3b47f53128 100644 --- a/Code/Tools/TestImpactFramework/Runtime/Code/Source/Test/Run/TestImpactInstrumentedTestRunner.h +++ b/Code/Tools/TestImpactFramework/Runtime/Code/Source/TestEngine/Run/TestImpactInstrumentedTestRunner.h @@ -12,24 +12,25 @@ #pragma once -#include -#include -#include -#include +#include +#include +#include +#include namespace TestImpact { //! Per-job data for instrumented test runs. - class InstrumentedTestRunJobData : public TestRunJobData + class InstrumentedTestRunJobData + : public TestRunJobData { public: - InstrumentedTestRunJobData(const AZ::IO::Path& resultsArtifact, const AZ::IO::Path& coverageArtifact); + InstrumentedTestRunJobData(const RepoPath& resultsArtifact, const RepoPath& coverageArtifact); //! Returns the path to the coverage artifact produced by the test target. - const AZ::IO::Path& GetCoverageArtifactPath() const; + const RepoPath& GetCoverageArtifactPath() const; private: - AZ::IO::Path m_coverageArtifact; //!< Path to coverage data. + RepoPath m_coverageArtifact; //!< Path to coverage data. }; namespace Bitwise @@ -43,7 +44,8 @@ namespace TestImpact } // namespace Bitwise //! Runs a batch of test targets to determine the test coverage and passes/failures. - class InstrumentedTestRunner : public TestJobRunner> + class InstrumentedTestRunner + : public TestJobRunner> { using JobRunner = TestJobRunner>; diff --git a/Code/Tools/TestImpactFramework/Runtime/Code/Source/Test/Run/TestImpactTestCoverage.cpp b/Code/Tools/TestImpactFramework/Runtime/Code/Source/TestEngine/Run/TestImpactTestCoverage.cpp similarity index 58% rename from Code/Tools/TestImpactFramework/Runtime/Code/Source/Test/Run/TestImpactTestCoverage.cpp rename to Code/Tools/TestImpactFramework/Runtime/Code/Source/TestEngine/Run/TestImpactTestCoverage.cpp index 0d50dbbf9a..453b1fa962 100644 --- a/Code/Tools/TestImpactFramework/Runtime/Code/Source/Test/Run/TestImpactTestCoverage.cpp +++ b/Code/Tools/TestImpactFramework/Runtime/Code/Source/TestEngine/Run/TestImpactTestCoverage.cpp @@ -10,16 +10,68 @@ * */ -#include +#include #include #include namespace TestImpact { - TestCoverage::TestCoverage(AZStd::vector&& moduleCoverages) + TestCoverage::TestCoverage(const TestCoverage& other) + : m_modules(other.m_modules) + , m_sourcesCovered(other.m_sourcesCovered) + , m_coverageLevel(other.m_coverageLevel) + { + } + + TestCoverage::TestCoverage(TestCoverage&& other) noexcept + : m_modules(AZStd::move(other.m_modules)) + , m_sourcesCovered(AZStd::move(other.m_sourcesCovered)) + { + AZStd::swap(m_coverageLevel, other.m_coverageLevel); + } + + TestCoverage::TestCoverage(const AZStd::vector& moduleCoverages) + : m_modules(moduleCoverages) + { + CalculateTestMetrics(); + } + + TestCoverage::TestCoverage(AZStd::vector&& moduleCoverages) noexcept : m_modules(AZStd::move(moduleCoverages)) { + CalculateTestMetrics(); + } + + TestCoverage& TestCoverage::operator=(const TestCoverage& other) + { + if (this != &other) + { + m_modules = other.m_modules; + m_sourcesCovered = other.m_sourcesCovered; + m_coverageLevel = other.m_coverageLevel; + } + + return *this; + } + + TestCoverage& TestCoverage::operator=(TestCoverage&& other) noexcept + { + if (this != &other) + { + m_modules = AZStd::move(other.m_modules); + m_sourcesCovered = other.m_sourcesCovered; + m_coverageLevel = other.m_coverageLevel; + } + + return *this; + } + + void TestCoverage::CalculateTestMetrics() + { + m_coverageLevel.reset(); + m_sourcesCovered.clear(); + for (const auto& moduleCovered : m_modules) { for (const auto& sourceCovered : moduleCovered.m_sources) diff --git a/Code/Tools/TestImpactFramework/Runtime/Code/Source/Test/Run/TestImpactTestCoverage.h b/Code/Tools/TestImpactFramework/Runtime/Code/Source/TestEngine/Run/TestImpactTestCoverage.h similarity index 85% rename from Code/Tools/TestImpactFramework/Runtime/Code/Source/Test/Run/TestImpactTestCoverage.h rename to Code/Tools/TestImpactFramework/Runtime/Code/Source/TestEngine/Run/TestImpactTestCoverage.h index 7253a29df0..e7599fb145 100644 --- a/Code/Tools/TestImpactFramework/Runtime/Code/Source/Test/Run/TestImpactTestCoverage.h +++ b/Code/Tools/TestImpactFramework/Runtime/Code/Source/TestEngine/Run/TestImpactTestCoverage.h @@ -29,7 +29,13 @@ namespace TestImpact class TestCoverage { public: - TestCoverage(AZStd::vector&& moduleCoverages); + TestCoverage(const TestCoverage&); + TestCoverage(TestCoverage&&) noexcept; + TestCoverage(AZStd::vector&& moduleCoverages) noexcept; + TestCoverage(const AZStd::vector& moduleCoverages); + + TestCoverage& operator=(const TestCoverage&); + TestCoverage& operator=(TestCoverage&&) noexcept; //! Returns the number of unique sources covered. size_t GetNumSourcesCovered() const; @@ -47,6 +53,8 @@ namespace TestImpact AZStd::optional GetCoverageLevel() const; private: + void CalculateTestMetrics(); + AZStd::vector m_modules; AZStd::vector m_sourcesCovered; AZStd::optional m_coverageLevel; diff --git a/Code/Tools/TestImpactFramework/Runtime/Code/Source/TestEngine/Run/TestImpactTestRun.cpp b/Code/Tools/TestImpactFramework/Runtime/Code/Source/TestEngine/Run/TestImpactTestRun.cpp new file mode 100644 index 0000000000..6a361ca9d0 --- /dev/null +++ b/Code/Tools/TestImpactFramework/Runtime/Code/Source/TestEngine/Run/TestImpactTestRun.cpp @@ -0,0 +1,138 @@ +/* + * All or portions of this file Copyright (c) Amazon.com, Inc. or its affiliates or + * its licensors. + * + * For complete copyright and license terms please see the LICENSE at the root of this + * distribution (the "License"). All use of this software is governed by the License, + * or, if provided, by the license below or the license accompanying this file. Do not + * remove or modify any license notices. This file is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * + */ + +#include "TestImpactTestRun.h" + +namespace TestImpact +{ + TestRun::TestRun(const TestRun& other) + : TestSuiteContainer(other) + , m_numRuns(other.m_numRuns) + , m_numNotRuns(other.m_numNotRuns) + , m_numPasses(other.m_numPasses) + , m_numFailures(other.m_numFailures) + , m_duration(other.m_duration) + { + CalculateTestMetrics(); + } + + TestRun::TestRun(TestRun&& other) noexcept + : TestSuiteContainer(AZStd::move(other)) + , m_numRuns(other.m_numRuns) + , m_numNotRuns(other.m_numNotRuns) + , m_numPasses(other.m_numPasses) + , m_numFailures(other.m_numFailures) + , m_duration(other.m_duration) + { + } + + TestRun::TestRun(AZStd::vector&& testSuites, AZStd::chrono::milliseconds duration) noexcept + : TestSuiteContainer(AZStd::move(testSuites)) + , m_duration(duration) + { + CalculateTestMetrics(); + } + + TestRun::TestRun(const AZStd::vector& testSuites, AZStd::chrono::milliseconds duration) + : TestSuiteContainer(testSuites) + , m_duration(duration) + { + CalculateTestMetrics(); + } + + TestRun& TestRun::operator=(TestRun&& other) noexcept + { + if (this != &other) + { + TestSuiteContainer::operator=(AZStd::move(other)); + m_numRuns = other.m_numRuns; + m_numNotRuns = other.m_numNotRuns; + m_numPasses = other.m_numPasses; + m_numFailures = other.m_numFailures; + m_duration = other.m_duration; + } + + return *this; + } + + TestRun& TestRun::operator=(const TestRun& other) + { + if (this != &other) + { + TestSuiteContainer::operator=(other); + m_numRuns = other.m_numRuns; + m_numNotRuns = other.m_numNotRuns; + m_numPasses = other.m_numPasses; + m_numFailures = other.m_numFailures; + m_duration = other.m_duration; + } + + return *this; + } + + void TestRun::CalculateTestMetrics() + { + m_numRuns = 0; + m_numNotRuns = 0; + m_numPasses = 0; + m_numFailures = 0; + + for (const auto& suite : m_testSuites) + { + for (const auto& test : suite.m_tests) + { + if (test.m_status == TestRunStatus::Run) + { + m_numRuns++; + + if (test.m_result.value() == TestRunResult::Passed) + { + m_numPasses++; + } + else + { + m_numFailures++; + } + } + else + { + m_numNotRuns++; + } + } + } + } + + size_t TestRun::GetNumRuns() const + { + return m_numRuns; + } + + size_t TestRun::GetNumNotRuns() const + { + return m_numNotRuns; + } + + size_t TestRun::GetNumPasses() const + { + return m_numPasses; + } + + size_t TestRun::GetNumFailures() const + { + return m_numFailures; + } + + AZStd::chrono::milliseconds TestRun::GetDuration() const + { + return m_duration; + } +} // namespace TestImpact diff --git a/Code/Tools/TestImpactFramework/Runtime/Code/Source/Test/Run/TestImpactTestRun.h b/Code/Tools/TestImpactFramework/Runtime/Code/Source/TestEngine/Run/TestImpactTestRun.h similarity index 81% rename from Code/Tools/TestImpactFramework/Runtime/Code/Source/Test/Run/TestImpactTestRun.h rename to Code/Tools/TestImpactFramework/Runtime/Code/Source/TestEngine/Run/TestImpactTestRun.h index 75d2140a68..f3a1ba93de 100644 --- a/Code/Tools/TestImpactFramework/Runtime/Code/Source/Test/Run/TestImpactTestRun.h +++ b/Code/Tools/TestImpactFramework/Runtime/Code/Source/TestEngine/Run/TestImpactTestRun.h @@ -13,7 +13,7 @@ #pragma once #include -#include +#include namespace TestImpact { @@ -24,7 +24,13 @@ namespace TestImpact using TestSuiteContainer = TestSuiteContainer; public: - TestRun(AZStd::vector&& testSuites, AZStd::chrono::milliseconds duration); + TestRun(const TestRun&); + TestRun(TestRun&&) noexcept; + TestRun(const AZStd::vector& testSuites, AZStd::chrono::milliseconds duration); + TestRun(AZStd::vector&& testSuites, AZStd::chrono::milliseconds duration) noexcept; + + TestRun& operator=(const TestRun&); + TestRun& operator=(TestRun&&) noexcept; //! Returns the total number of tests that were run. size_t GetNumRuns() const; @@ -42,6 +48,8 @@ namespace TestImpact AZStd::chrono::milliseconds GetDuration() const; private: + void CalculateTestMetrics(); + size_t m_numRuns = 0; size_t m_numNotRuns = 0; size_t m_numPasses = 0; diff --git a/Code/Tools/TestImpactFramework/Runtime/Code/Source/Test/Run/TestImpactTestRunException.h b/Code/Tools/TestImpactFramework/Runtime/Code/Source/TestEngine/Run/TestImpactTestRunException.h similarity index 100% rename from Code/Tools/TestImpactFramework/Runtime/Code/Source/Test/Run/TestImpactTestRunException.h rename to Code/Tools/TestImpactFramework/Runtime/Code/Source/TestEngine/Run/TestImpactTestRunException.h diff --git a/Code/Tools/TestImpactFramework/Runtime/Code/Source/Test/Run/TestImpactTestRunJobData.cpp b/Code/Tools/TestImpactFramework/Runtime/Code/Source/TestEngine/Run/TestImpactTestRunJobData.cpp similarity index 78% rename from Code/Tools/TestImpactFramework/Runtime/Code/Source/Test/Run/TestImpactTestRunJobData.cpp rename to Code/Tools/TestImpactFramework/Runtime/Code/Source/TestEngine/Run/TestImpactTestRunJobData.cpp index fa40b30263..a7ed446a25 100644 --- a/Code/Tools/TestImpactFramework/Runtime/Code/Source/Test/Run/TestImpactTestRunJobData.cpp +++ b/Code/Tools/TestImpactFramework/Runtime/Code/Source/TestEngine/Run/TestImpactTestRunJobData.cpp @@ -10,16 +10,16 @@ * */ -#include +#include namespace TestImpact { - TestRunJobData::TestRunJobData(const AZ::IO::Path& resultsArtifact) + TestRunJobData::TestRunJobData(const RepoPath& resultsArtifact) : m_runArtifact(resultsArtifact) { } - const AZ::IO::Path& TestRunJobData::GetRunArtifactPath() const + const RepoPath& TestRunJobData::GetRunArtifactPath() const { return m_runArtifact; } diff --git a/Code/Tools/TestImpactFramework/Runtime/Code/Source/Test/Run/TestImpactTestRunJobData.h b/Code/Tools/TestImpactFramework/Runtime/Code/Source/TestEngine/Run/TestImpactTestRunJobData.h similarity index 77% rename from Code/Tools/TestImpactFramework/Runtime/Code/Source/Test/Run/TestImpactTestRunJobData.h rename to Code/Tools/TestImpactFramework/Runtime/Code/Source/TestEngine/Run/TestImpactTestRunJobData.h index 802aa0d30f..a2033b5945 100644 --- a/Code/Tools/TestImpactFramework/Runtime/Code/Source/Test/Run/TestImpactTestRunJobData.h +++ b/Code/Tools/TestImpactFramework/Runtime/Code/Source/TestEngine/Run/TestImpactTestRunJobData.h @@ -12,7 +12,7 @@ #pragma once -#include +#include namespace TestImpact { @@ -20,12 +20,12 @@ namespace TestImpact class TestRunJobData { public: - TestRunJobData(const AZ::IO::Path& resultsArtifact); + TestRunJobData(const RepoPath& resultsArtifact); //! Returns the path to the test run artifact produced by the test target. - const AZ::IO::Path& GetRunArtifactPath() const; + const RepoPath& GetRunArtifactPath() const; private: - AZ::IO::Path m_runArtifact; //!< Path to results data. + RepoPath m_runArtifact; //!< Path to results data. }; } // namespace TestImpact diff --git a/Code/Tools/TestImpactFramework/Runtime/Code/Source/Test/Run/TestImpactTestRunSerializer.cpp b/Code/Tools/TestImpactFramework/Runtime/Code/Source/TestEngine/Run/TestImpactTestRunSerializer.cpp similarity index 98% rename from Code/Tools/TestImpactFramework/Runtime/Code/Source/Test/Run/TestImpactTestRunSerializer.cpp rename to Code/Tools/TestImpactFramework/Runtime/Code/Source/TestEngine/Run/TestImpactTestRunSerializer.cpp index 7f29dd77a9..7fe8789d22 100644 --- a/Code/Tools/TestImpactFramework/Runtime/Code/Source/Test/Run/TestImpactTestRunSerializer.cpp +++ b/Code/Tools/TestImpactFramework/Runtime/Code/Source/TestEngine/Run/TestImpactTestRunSerializer.cpp @@ -10,8 +10,8 @@ * */ -#include -#include +#include +#include #include #include diff --git a/Code/Tools/TestImpactFramework/Runtime/Code/Source/Test/Run/TestImpactTestRunSerializer.h b/Code/Tools/TestImpactFramework/Runtime/Code/Source/TestEngine/Run/TestImpactTestRunSerializer.h similarity index 95% rename from Code/Tools/TestImpactFramework/Runtime/Code/Source/Test/Run/TestImpactTestRunSerializer.h rename to Code/Tools/TestImpactFramework/Runtime/Code/Source/TestEngine/Run/TestImpactTestRunSerializer.h index 5978b27105..a73e675559 100644 --- a/Code/Tools/TestImpactFramework/Runtime/Code/Source/Test/Run/TestImpactTestRunSerializer.h +++ b/Code/Tools/TestImpactFramework/Runtime/Code/Source/TestEngine/Run/TestImpactTestRunSerializer.h @@ -12,7 +12,7 @@ #pragma once -#include +#include #include diff --git a/Code/Tools/TestImpactFramework/Runtime/Code/Source/Test/Run/TestImpactTestRunner.cpp b/Code/Tools/TestImpactFramework/Runtime/Code/Source/TestEngine/Run/TestImpactTestRunner.cpp similarity index 73% rename from Code/Tools/TestImpactFramework/Runtime/Code/Source/Test/Run/TestImpactTestRunner.cpp rename to Code/Tools/TestImpactFramework/Runtime/Code/Source/TestEngine/Run/TestImpactTestRunner.cpp index dde7dd7d47..ebb846f406 100644 --- a/Code/Tools/TestImpactFramework/Runtime/Code/Source/Test/Run/TestImpactTestRunner.cpp +++ b/Code/Tools/TestImpactFramework/Runtime/Code/Source/TestEngine/Run/TestImpactTestRunner.cpp @@ -10,17 +10,18 @@ * */ +#include + #include -#include -#include -#include -#include +#include +#include +#include #include namespace TestImpact { - TestRun ParseTestRunFile(const AZ::IO::Path& runFile, AZStd::chrono::milliseconds duration) + TestRun ParseTestRunFile(const RepoPath& runFile, AZStd::chrono::milliseconds duration) { return TestRun(GTest::TestRunSuitesFactory(ReadFileContents(runFile)), duration); } @@ -46,7 +47,15 @@ namespace TestImpact const auto& [meta, jobInfo] = jobData; if (meta.m_result == JobResult::ExecutedWithSuccess || meta.m_result == JobResult::ExecutedWithFailure) { - runs[jobId] = ParseTestRunFile(jobInfo->GetRunArtifactPath(), meta.m_duration.value()); + try + { + runs[jobId] = ParseTestRunFile(jobInfo->GetRunArtifactPath(), meta.m_duration.value()); + } + catch (const Exception& e) + { + AZ_Warning("RunTests", false, e.what()); + runs[jobId] = AZStd::nullopt; + } } } diff --git a/Code/Tools/TestImpactFramework/Runtime/Code/Source/Test/Run/TestImpactTestRunner.h b/Code/Tools/TestImpactFramework/Runtime/Code/Source/TestEngine/Run/TestImpactTestRunner.h similarity index 93% rename from Code/Tools/TestImpactFramework/Runtime/Code/Source/Test/Run/TestImpactTestRunner.h rename to Code/Tools/TestImpactFramework/Runtime/Code/Source/TestEngine/Run/TestImpactTestRunner.h index 0bf66601fe..379c6b424e 100644 --- a/Code/Tools/TestImpactFramework/Runtime/Code/Source/Test/Run/TestImpactTestRunner.h +++ b/Code/Tools/TestImpactFramework/Runtime/Code/Source/TestEngine/Run/TestImpactTestRunner.h @@ -12,9 +12,9 @@ #pragma once -#include -#include -#include +#include +#include +#include namespace TestImpact { diff --git a/Code/Tools/TestImpactFramework/Runtime/Code/Source/Test/TestImpactTestSuiteContainer.h b/Code/Tools/TestImpactFramework/Runtime/Code/Source/TestEngine/TestImpactTestSuiteContainer.h similarity index 59% rename from Code/Tools/TestImpactFramework/Runtime/Code/Source/Test/TestImpactTestSuiteContainer.h rename to Code/Tools/TestImpactFramework/Runtime/Code/Source/TestEngine/TestImpactTestSuiteContainer.h index c41d76bd04..5991640c46 100644 --- a/Code/Tools/TestImpactFramework/Runtime/Code/Source/Test/TestImpactTestSuiteContainer.h +++ b/Code/Tools/TestImpactFramework/Runtime/Code/Source/TestEngine/TestImpactTestSuiteContainer.h @@ -22,7 +22,13 @@ namespace TestImpact class TestSuiteContainer { public: - TestSuiteContainer(AZStd::vector&& testSuites); + TestSuiteContainer(const TestSuiteContainer&); + TestSuiteContainer(TestSuiteContainer&&) noexcept; + TestSuiteContainer(const AZStd::vector& testSuites); + TestSuiteContainer(AZStd::vector&& testSuites) noexcept; + + TestSuiteContainer& operator=(const TestSuiteContainer&); + TestSuiteContainer& operator=(TestSuiteContainer&&) noexcept; //! Returns the test suites in this container. const AZStd::vector& GetTestSuites() const; @@ -39,6 +45,9 @@ namespace TestImpact //! Returns the total number of disabled tests across all test suites. size_t GetNumDisabledTests() const; + private: + void CalculateTestMetrics(); + protected: AZStd::vector m_testSuites; size_t m_numDisabledTests = 0; @@ -46,9 +55,67 @@ namespace TestImpact }; template - TestSuiteContainer::TestSuiteContainer(AZStd::vector&& testSuites) + TestSuiteContainer::TestSuiteContainer(TestSuiteContainer&& other) noexcept + : m_testSuites(AZStd::move(other.m_testSuites)) + , m_numDisabledTests(other.m_numDisabledTests) + , m_numEnabledTests(other.m_numEnabledTests) + { + } + + template + TestSuiteContainer::TestSuiteContainer(const TestSuiteContainer& other) + : m_testSuites(other.m_testSuites.begin(), other.m_testSuites.end()) + , m_numDisabledTests(other.m_numDisabledTests) + , m_numEnabledTests(other.m_numEnabledTests) + { + } + + template + TestSuiteContainer::TestSuiteContainer(AZStd::vector&& testSuites) noexcept : m_testSuites(std::move(testSuites)) { + CalculateTestMetrics(); + } + + template + TestSuiteContainer::TestSuiteContainer(const AZStd::vector& testSuites) + : m_testSuites(testSuites) + { + CalculateTestMetrics(); + } + + template + TestSuiteContainer& TestSuiteContainer::operator=(TestSuiteContainer&& other) noexcept + { + if (this != &other) + { + m_testSuites = AZStd::move(other.m_testSuites); + m_numDisabledTests = other.m_numDisabledTests; + m_numEnabledTests = other.m_numEnabledTests; + } + + return *this; + } + + template + TestSuiteContainer& TestSuiteContainer::operator=(const TestSuiteContainer& other) + { + if (this != &other) + { + m_testSuites = other.m_testSuites; + m_numDisabledTests = other.m_numDisabledTests; + m_numEnabledTests = other.m_numEnabledTests; + } + + return *this; + } + + template + void TestSuiteContainer::CalculateTestMetrics() + { + m_numDisabledTests = 0; + m_numEnabledTests = 0; + for (const auto& suite : m_testSuites) { if (suite.m_enabled)