From f9425992f86fe20fa822c761233cd71e34020c52 Mon Sep 17 00:00:00 2001 From: jonawals Date: Thu, 27 May 2021 11:37:34 +0100 Subject: [PATCH] Runtime refactor --- .../TestImpactClientFailureReport.h | 72 +++----------- .../TestImpactClientTestSelection.h | 6 +- ...estImpactUtils.h => TestImpactFileUtils.h} | 22 +++++ .../TestImpactFramework/TestImpactRepoPath.h | 68 ++++++++++++-- .../TestImpactFramework/TestImpactRuntime.h | 77 +++++++++++---- .../TestImpactTestSequence.h | 18 ++-- .../TestImpactDynamicDependencyMap.cpp | 66 +++++++++++-- .../TestImpactDynamicDependencyMap.h | 14 ++- .../TestImpactSourceCoveringTestsList.cpp | 2 +- ...estImpactSourceCoveringTestsSerializer.cpp | 93 +++++++++++++++++++ .../TestImpactSourceCoveringTestsSerializer.h | 26 ++++++ .../TestImpactTestSelectorAndPrioritizer.cpp | 4 +- .../TestImpactTestSelectorAndPrioritizer.h | 13 +-- .../TestImpactWin32_TestEngineJobFailure.cpp | 35 +++++++ .../TestImpactTestEngineJobFailure.cpp | 82 ++++++++++++++++ .../TestImpactTestEngineJobFailure.h | 29 ++++++ 16 files changed, 513 insertions(+), 114 deletions(-) rename Code/Tools/TestImpactFramework/Runtime/Code/Include/TestImpactFramework/{TestImpactUtils.h => TestImpactFileUtils.h} (77%) create mode 100644 Code/Tools/TestImpactFramework/Runtime/Code/Source/Dependency/TestImpactSourceCoveringTestsSerializer.cpp create mode 100644 Code/Tools/TestImpactFramework/Runtime/Code/Source/Dependency/TestImpactSourceCoveringTestsSerializer.h create mode 100644 Code/Tools/TestImpactFramework/Runtime/Code/Source/Platform/Windows/TestEngine/TestImpactWin32_TestEngineJobFailure.cpp create mode 100644 Code/Tools/TestImpactFramework/Runtime/Code/Source/TestEngine/TestImpactTestEngineJobFailure.cpp create mode 100644 Code/Tools/TestImpactFramework/Runtime/Code/Source/TestEngine/TestImpactTestEngineJobFailure.h diff --git a/Code/Tools/TestImpactFramework/Runtime/Code/Include/TestImpactFramework/TestImpactClientFailureReport.h b/Code/Tools/TestImpactFramework/Runtime/Code/Include/TestImpactFramework/TestImpactClientFailureReport.h index 02b1169e93..19e98a7d1f 100644 --- a/Code/Tools/TestImpactFramework/Runtime/Code/Include/TestImpactFramework/TestImpactClientFailureReport.h +++ b/Code/Tools/TestImpactFramework/Runtime/Code/Include/TestImpactFramework/TestImpactClientFailureReport.h @@ -44,19 +44,6 @@ namespace TestImpact AZStd::string m_commandString; }; - //! Represents a test target that terminated abnormally. - class LauncherFailure - : public ExecutionFailure - { - public: - LauncherFailure(const AZStd::string& targetName, const AZStd::string& command, int returnCode); - - //! The return code of the test target that terminated abnormally. - int GetReturnCode() const; - private: - int m_returnCode; - }; - //! Represents an individual test of a test target that failed. class TestFailure { @@ -107,6 +94,7 @@ namespace TestImpact private: AZStd::vector m_testCaseFailures; + size_t m_numTestFailures = 0; }; //! Base class for reporting failing test sequences. @@ -115,63 +103,27 @@ namespace TestImpact public: SequenceFailure( AZStd::vector&& executionFailures, - AZStd::vector&& launcherFailures, + AZStd::vector&& testRunFailures, + AZStd::vector&& timedOutTests, AZStd::vector&& unexecutedTests); //! Returns the test targets in this sequence that failed to execute. const AZStd::vector& GetExecutionFailures() const; - //! Returns the test targets in this sequence that terminated abnormally. - const AZStd::vector& GetLauncherFailures() const; - - //! Returns the test targets in this sequence that were not executed due to the sequence terminating prematurely. - const AZStd::vector& GetUnexecutedTest() const; - - private: - AZStd::vector m_executionFailures; - AZStd::vector m_launcherFailures; - AZStd::vector m_unexecutedTests; - }; - - //! Represents the report for a failed regular test sequence run without test impact analysis. - class RegularSequenceFailure - : public SequenceFailure - { - public: - RegularSequenceFailure( - AZStd::vector&& executionFailures, - AZStd::vector&& launcherFailures, - AZStd::vector&& testRunFailures, - AZStd::vector&& unexecutedTests); - //! Returns the test targets that contain failing tests. const AZStd::vector& GetTestRunFailures() const; + //! Returns the test targets in this sequence that were terminated for exceeding their allotted flight time. + const AZStd::vector& GetTimedOutTests() const; + + //! Returns the test targets in this sequence that were not executed due to the sequence terminating prematurely. + const AZStd::vector& GetUnexecutedTests() const; + private: + AZStd::vector m_executionFailures; AZStd::vector m_testRunFailures; - }; - - //! Represents the report for a failed test sequence run with test impact analysis. - class ImpactAnalysisSequenceFailure - : public SequenceFailure - { - public: - ImpactAnalysisSequenceFailure( - AZStd::vector&& executionFailures, - AZStd::vector&& launcherFailures, - AZStd::vector&& selectedTestRunFailures, - AZStd::vector&& discardedTestRunFailures, - AZStd::vector&& unexecutedTests); - - //! Returns the test targets that were selected to run but contain failing tests. - const AZStd::vector GetSelectedTestRunFailures() const; - - //! Returns the test targets that were not selected but still run but contain failing tests. - const AZStd::vector GetDiscardedTestRunFailures() const; - - private: - AZStd::vector m_selectedTestRunFailures; - AZStd::vector m_discardedTestRunFailures; + AZStd::vector m_timedOutTests; + AZStd::vector m_unexecutedTests; }; } // namespace Client } // namespace TestImpact diff --git a/Code/Tools/TestImpactFramework/Runtime/Code/Include/TestImpactFramework/TestImpactClientTestSelection.h b/Code/Tools/TestImpactFramework/Runtime/Code/Include/TestImpactFramework/TestImpactClientTestSelection.h index f251e7117d..76e6461206 100644 --- a/Code/Tools/TestImpactFramework/Runtime/Code/Include/TestImpactFramework/TestImpactClientTestSelection.h +++ b/Code/Tools/TestImpactFramework/Runtime/Code/Include/TestImpactFramework/TestImpactClientTestSelection.h @@ -25,6 +25,7 @@ namespace TestImpact class TestRunSelection { public: + TestRunSelection(const AZStd::vector& includedTests, const AZStd::vector& excludedTests); TestRunSelection(AZStd::vector&& includedTests, AZStd::vector&& excludedTests); //! Returns the test runs that were selected to be run and will actually be run. @@ -37,7 +38,10 @@ namespace TestImpact size_t GetNumIncludedTestRuns() const; //! Returns the number of selected test runs that will not be run. - size_t GetNumNumExcludedTestRuns() const; + size_t GetNumExcludedTestRuns() const; + + //! Returns the total number of tests runs selected regardless of whether or not they will actually be run. + size_t GetTotalNumTests() const; private: AZStd::vector m_includedTestRuns; diff --git a/Code/Tools/TestImpactFramework/Runtime/Code/Include/TestImpactFramework/TestImpactUtils.h b/Code/Tools/TestImpactFramework/Runtime/Code/Include/TestImpactFramework/TestImpactFileUtils.h similarity index 77% rename from Code/Tools/TestImpactFramework/Runtime/Code/Include/TestImpactFramework/TestImpactUtils.h rename to Code/Tools/TestImpactFramework/Runtime/Code/Include/TestImpactFramework/TestImpactFileUtils.h index d734d7a70e..aa64840e78 100644 --- a/Code/Tools/TestImpactFramework/Runtime/Code/Include/TestImpactFramework/TestImpactUtils.h +++ b/Code/Tools/TestImpactFramework/Runtime/Code/Include/TestImpactFramework/TestImpactFileUtils.h @@ -59,4 +59,26 @@ namespace TestImpact AZ_TestImpact_Eval( file.Write(bytes.data(), bytes.size()), ExceptionType, AZStd::string::format("Couldn't write contents for file %s", path.c_str())); } + + //! Delete the files that match the pattern from the specified directory. + //! @param path The path to the directory to pattern match the files for deletion. + //! @param pattern The pattern to match files for deletion. + inline void DeleteFiles(const RepoPath& path, const AZStd::string& pattern) + { + AZ::IO::SystemFile::FindFiles(AZStd::string::format("%s/%s", path.c_str(), pattern.c_str()).c_str(), + [&path](const char* file, bool isFile) + { + if (isFile) + { + AZ::IO::SystemFile::Delete(AZStd::string::format("%s/%s", path.c_str(), file).c_str()); + } + + return true; + }); + } + + inline void DeleteFile(const RepoPath& path) + { + DeleteFiles(path.ParentPath(), path.Filename().Native()); + } } // namespace TestImpact diff --git a/Code/Tools/TestImpactFramework/Runtime/Code/Include/TestImpactFramework/TestImpactRepoPath.h b/Code/Tools/TestImpactFramework/Runtime/Code/Include/TestImpactFramework/TestImpactRepoPath.h index ccba3f677d..80f82e4785 100644 --- a/Code/Tools/TestImpactFramework/Runtime/Code/Include/TestImpactFramework/TestImpactRepoPath.h +++ b/Code/Tools/TestImpactFramework/Runtime/Code/Include/TestImpactFramework/TestImpactRepoPath.h @@ -21,22 +21,78 @@ namespace TestImpact //! to the test impact analysis data as otherwise querying/retrieving test impact analysis data for the same source albeit //! with different path separators will be considered different files entirely. class RepoPath - : public AZ::IO::Path { public: + using string_type = AZ::IO::Path::string_type; + using string_view_type = AZ::IO::Path::string_view_type; + using value_type = AZ::IO::Path::value_type; + constexpr RepoPath() = default; constexpr RepoPath(const RepoPath&) = default; constexpr RepoPath(RepoPath&&) noexcept = default; - constexpr RepoPath(const string_type&) noexcept; - constexpr RepoPath(const value_type*) noexcept; - constexpr RepoPath(const AZ::IO::PathView&); - constexpr RepoPath(const AZ::IO::Path&); + constexpr RepoPath::RepoPath(const string_type & path) noexcept; + constexpr RepoPath::RepoPath(const string_view_type& path) noexcept; + constexpr RepoPath::RepoPath(const value_type* path) noexcept; + constexpr RepoPath::RepoPath(const AZ::IO::PathView& path); + constexpr RepoPath::RepoPath(const AZ::IO::Path& path); RepoPath& operator=(const RepoPath&) noexcept = default; RepoPath& operator=(const string_type&) noexcept; RepoPath& operator=(const value_type*) noexcept; RepoPath& operator=(const AZ::IO::Path& str) noexcept; - using AZ::IO::Path::operator AZ::IO::PathView; + const char* c_str() const { return m_path.c_str(); } + AZStd::string String() const { return m_path.String(); } + constexpr AZ::IO::PathView Stem() const { return m_path.Stem(); } + constexpr AZ::IO::PathView Extension() const { return m_path.Extension(); } + constexpr bool empty() const { return m_path.empty(); } + constexpr AZ::IO::PathView ParentPath() const { return m_path.ParentPath(); } + constexpr AZ::IO::PathView Filename() const { return m_path.Filename(); } + AZ::IO::Path LexicallyRelative(const RepoPath& base) const { return m_path.LexicallyRelative(base.m_path); } + [[nodiscard]] bool IsRelativeTo(const RepoPath& base) const { return m_path.IsRelativeTo(base.m_path); } + constexpr AZ::IO::PathView RootName() const { return m_path.RootName(); } + constexpr AZ::IO::PathView RelativePath() const { return m_path.RelativePath(); } + + friend RepoPath operator/(const RepoPath& lhs, const AZ::IO::PathView& rhs); + friend RepoPath operator/(const RepoPath& lhs, AZStd::string_view rhs); + friend RepoPath operator/(const RepoPath& lhs, const typename value_type* rhs); + friend RepoPath operator/(const RepoPath& lhs, const RepoPath& rhs); + + RepoPath& operator/=(const AZ::IO::PathView& rhs); + RepoPath& operator/=(AZStd::string_view rhs); + RepoPath& operator/=(const typename value_type* rhs); + RepoPath& operator/=(const RepoPath& rhs); + + friend bool operator==(const RepoPath& lhs, const RepoPath& rhs) noexcept; + friend bool operator!=(const RepoPath& lhs, const RepoPath& rhs) noexcept; + friend bool operator<(const RepoPath& lhs, const RepoPath& rhs) noexcept; + + private: + AZ::IO::Path m_path; }; + + constexpr RepoPath::RepoPath(const string_type& path) noexcept + : m_path(AZ::IO::Path(path).MakePreferred()) + { + } + + constexpr RepoPath::RepoPath(const string_view_type& path) noexcept + : m_path(AZ::IO::Path(path).MakePreferred()) + { + } + + constexpr RepoPath::RepoPath(const value_type* path) noexcept + : m_path(AZ::IO::Path(path).MakePreferred()) + { + } + + constexpr RepoPath::RepoPath(const AZ::IO::PathView& path) + : m_path(AZ::IO::Path(path).MakePreferred()) + { + } + + constexpr RepoPath::RepoPath(const AZ::IO::Path& path) + : m_path(AZ::IO::Path(path).MakePreferred()) + { + } } // namespace TestImpact diff --git a/Code/Tools/TestImpactFramework/Runtime/Code/Include/TestImpactFramework/TestImpactRuntime.h b/Code/Tools/TestImpactFramework/Runtime/Code/Include/TestImpactFramework/TestImpactRuntime.h index 8221eba834..31b2d1edce 100644 --- a/Code/Tools/TestImpactFramework/Runtime/Code/Include/TestImpactFramework/TestImpactRuntime.h +++ b/Code/Tools/TestImpactFramework/Runtime/Code/Include/TestImpactFramework/TestImpactRuntime.h @@ -29,9 +29,12 @@ namespace TestImpact { + class ChangeDependencyList; class DynamicDependencyMap; + class TestSelectorAndPrioritizer; class TestEngine; class TestTarget; + class SourceCoveringTestsList; //! Callback for a test sequence that isn't using test impact analysis to determine selected tests. //! @param tests The tests that will be run for this sequence. @@ -66,16 +69,22 @@ namespace TestImpact //! Callback for end of a test sequence. //! @param failureReport The test runs that failed for any reason during this sequence. //! @param duration The total duration of this test sequence. - using TestSequenceCompleteCallback = AZStd::function; + using TestSequenceCompleteCallback = AZStd::function; //! Callback for end of a test impact analysis test sequence. - //! @param failureReport The test runs that failed for any reason during this sequence. + //! @param selectedFailureReport The selected test runs that failed for any reason during this sequence. + //! @param discardedFailureReport The discarded test runs that failed for any reason during this sequence. //! @param duration The total duration of this test sequence. - using ImpactAnalysisTestSequenceCompleteCallback = AZStd::function; + using SafeTestSequenceCompleteCallback = AZStd::function; //! Callback for test runs that have completed for any reason. - //! test The test that has completed. - using TestCompleteCallback = AZStd::function; + //! selectedTests The test that has completed. + using TestRunCompleteCallback = AZStd::function; //! The API exposed to the client responsible for all test runs and persistent data management. class Runtime @@ -95,7 +104,7 @@ namespace TestImpact Policy::TestFailure testFailurePolicy, Policy::IntegrityFailure integrationFailurePolicy, Policy::TestSharding testShardingPolicy, - TargetOutputCapture targetOutputCapture, + Policy::TargetOutputCapture targetOutputCapture, AZStd::optional maxConcurrency = AZStd::nullopt); ~Runtime(); @@ -103,78 +112,110 @@ namespace TestImpact //! Runs a test sequence where all tests with a matching suite in the suite filter and also not on the excluded list are selected. //! @param suitesFilter The test suites that will be included in the test selection. //! @param testTargetTimeout The maximum duration individual test targets may be in flight for (infinite if empty). - //! @param testTargetTimeout The maximum duration the entire test sequence may run for (infinite if empty). + //! @param globalTimeout The maximum duration the entire test sequence may run for (infinite if empty). //! @param testSequenceStartCallback The client function to be called after the test targets have been selected but prior to running the tests. //! @param testSequenceCompleteCallback The client function to be called after the test sequence has completed. //! @param testRunCompleteCallback The client function to be called after an individual test run has completed. + //! @returns TestSequenceResult RegularTestSequence( const AZStd::unordered_set suitesFilter, AZStd::optional testTargetTimeout, AZStd::optional globalTimeout, AZStd::optional testSequenceStartCallback, AZStd::optional testSequenceCompleteCallback, - AZStd::optional testRunCompleteCallback); + AZStd::optional testRunCompleteCallback); //! Runs a test sequence where tests are selected according to test impact analysis so long as they are not on the excluded list. //! @param changeList The change list used to determine the tests to select. //! @param testPrioritizationPolicy Determines how selected tests will be prioritized. //! @param testTargetTimeout The maximum duration individual test targets may be in flight for (infinite if empty). - //! @param testTargetTimeout The maximum duration the entire test sequence may run for (infinite if empty). + //! @param globalTimeout The maximum duration the entire test sequence may run for (infinite if empty). //! @param testSequenceStartCallback The client function to be called after the test targets have been selected but prior to running the tests. //! @param testSequenceCompleteCallback The client function to be called after the test sequence has completed. //! @param testRunCompleteCallback The client function to be called after an individual test run has completed. + //! @returns TestSequenceResult ImpactAnalysisTestSequence( const ChangeList& changeList, Policy::TestPrioritization testPrioritizationPolicy, AZStd::optional testTargetTimeout, AZStd::optional globalTimeout, AZStd::optional testSequenceStartCallback, - AZStd::optional testSequenceCompleteCallback, - AZStd::optional testRunCompleteCallback); + AZStd::optional testSequenceCompleteCallback, + AZStd::optional testRunCompleteCallback); //! Runs a test sequence as per the ImpactAnalysisTestSequence where the tests not selected are also run (albeit without instrumentation). //! @param changeList The change list used to determine the tests to select. //! @param suitesFilter The test suites that will be included in the test selection. //! @param testPrioritizationPolicy Determines how selected tests will be prioritized. //! @param testTargetTimeout The maximum duration individual test targets may be in flight for (infinite if empty). - //! @param testTargetTimeout The maximum duration the entire test sequence may run for (infinite if empty). + //! @param globalTimeout The maximum duration the entire test sequence may run for (infinite if empty). //! @param testSequenceStartCallback The client function to be called after the test targets have been selected but prior to running the tests. //! @param testSequenceCompleteCallback The client function to be called after the test sequence has completed. //! @param testRunCompleteCallback The client function to be called after an individual test run has completed. - TestSequenceResult SafeImpactAnalysisTestSequence( + //! @returns + AZStd::pair SafeImpactAnalysisTestSequence( const ChangeList& changeList, const AZStd::unordered_set suitesFilter, Policy::TestPrioritization testPrioritizationPolicy, AZStd::optional testTargetTimeout, AZStd::optional globalTimeout, AZStd::optional testSequenceStartCallback, - AZStd::optional testSequenceCompleteCallback, - AZStd::optional testRunCompleteCallback); + AZStd::optional testSequenceCompleteCallback, + AZStd::optional testRunCompleteCallback); //! Runs all tests not on the excluded list and uses their coverage data to seed the test impact analysis data (ant existing data will be overwritten). - //! @param testTargetTimeout The maximum duration the entire test sequence may run for (infinite if empty). + //! @param testTargetTimeout The maximum duration individual test targets may be in flight for (infinite if empty). + //! @param globalTimeout The maximum duration the entire test sequence may run for (infinite if empty). //! @param testSequenceStartCallback The client function to be called after the test targets have been selected but prior to running the tests. //! @param testSequenceCompleteCallback The client function to be called after the test sequence has completed. //! @param testRunCompleteCallback The client function to be called after an individual test run has completed. + //! TestSequenceResult SeededTestSequence( + AZStd::optional testTargetTimeout, AZStd::optional globalTimeout, AZStd::optional testSequenceStartCallback, AZStd::optional testSequenceCompleteCallback, - AZStd::optional testRunCompleteCallback); + AZStd::optional testRunCompleteCallback); //! Returns true if the runtime has test impact analysis data (either preexisting or generated). bool HasImpactAnalysisData() const; private: + //! Updates the test enumeration cache for test targets that had sources modified by a given change list. + //! @param changeDependencyList The resolved change dependency list generated for the change list. + void EnumerateMutatedTestTargets(const ChangeDependencyList& changeDependencyList); + + //! Selects the test targets covering a given change list and updates the enumeration cache of the test targets with sources + //! modified in that change list. + //! @param changeList The change list for which the covering tests and enumeration cache updates will be generated for. + //! @param testPrioritizationPolicy The test prioritization strategy to use for the selected test targets. + //! @returns The pair of selected test targets and discarded test targets. + AZStd::pair, AZStd::vector> SelectCoveringTestTargetsAndUpdateEnumerationCache( + const ChangeList& changeList, + Policy::TestPrioritization testPrioritizationPolicy); + + //! Selects the test targets from the specified list of test targets that are not on the test target exclusion list. + //! @param testTargets The list of test targets to select from. + //! @returns The subset of test targets in the specified list that are not on the target exclude list. + AZStd::pair, AZStd::vector> SelectTestTargetsByExcludeList( + AZStd::vector testTargets) const; + + //! Prepares the dynamic dependency map for a seed update by clearing all existing data and deleting the file that will be serialized. + void ClearDynamicDependencyMapAndRemoveExistingFile(); + + //! Updates the dynamic dependency map and serializes the entire map to disk. + void UpdateAndSerializeDynamicDependencyMap(const SourceCoveringTestsList& sourceCoverageTestsList); + RuntimeConfig m_config; Policy::ExecutionFailure m_executionFailurePolicy; Policy::ExecutionFailureDrafting m_executionFailureDraftingPolicy; Policy::TestFailure m_testFailurePolicy; Policy::IntegrityFailure m_integrationFailurePolicy; Policy::TestSharding m_testShardingPolicy; - TargetOutputCapture m_targetOutputCapture; + Policy::TargetOutputCapture m_targetOutputCapture; size_t m_maxConcurrency = 0; AZStd::unique_ptr m_dynamicDependencyMap; + AZStd::unique_ptr m_testSelectorAndPrioritizer; AZStd::unique_ptr m_testEngine; AZStd::unordered_set m_testTargetExcludeList; AZStd::unordered_set m_testTargetShardList; diff --git a/Code/Tools/TestImpactFramework/Runtime/Code/Include/TestImpactFramework/TestImpactTestSequence.h b/Code/Tools/TestImpactFramework/Runtime/Code/Include/TestImpactFramework/TestImpactTestSequence.h index 751db43987..594c997c45 100644 --- a/Code/Tools/TestImpactFramework/Runtime/Code/Include/TestImpactFramework/TestImpactTestSequence.h +++ b/Code/Tools/TestImpactFramework/Runtime/Code/Include/TestImpactFramework/TestImpactTestSequence.h @@ -61,16 +61,16 @@ namespace TestImpact Never, //!< Do not shard any test targets. Always //!< Shard all test targets that have been marked for test sharding. }; - } - //! Standard output capture of test target runs. - enum class TargetOutputCapture - { - None, //!< Do not capture any output. - StdOut, //!< Send captured output to standard output - File, //!< Write captured output to file. - StdOutAndFile //!< Send captured output to standard output and write to file. - }; + //! Standard output capture of test target runs. + enum class TargetOutputCapture + { + None, //!< Do not capture any output. + StdOut, //!< Send captured output to standard output + File, //!< Write captured output to file. + StdOutAndFile //!< Send captured output to standard output and write to file. + }; + } //! Configuration for test targets that opt in to test sharding. enum class ShardConfiguration diff --git a/Code/Tools/TestImpactFramework/Runtime/Code/Source/Dependency/TestImpactDynamicDependencyMap.cpp b/Code/Tools/TestImpactFramework/Runtime/Code/Source/Dependency/TestImpactDynamicDependencyMap.cpp index 31b3485c70..919c10d049 100644 --- a/Code/Tools/TestImpactFramework/Runtime/Code/Source/Dependency/TestImpactDynamicDependencyMap.cpp +++ b/Code/Tools/TestImpactFramework/Runtime/Code/Source/Dependency/TestImpactDynamicDependencyMap.cpp @@ -34,7 +34,7 @@ namespace TestImpact else { // This is a new entry on the dependency map so create an entry with this parent target and no covering targets - m_sourceDependencyMap.emplace(source, DependencyData{ {target}, {} }); + m_sourceDependencyMap.emplace(source.String(), DependencyData{ {target}, {} }); } } @@ -56,6 +56,7 @@ namespace TestImpact for (const auto& target : m_testTargets.GetTargets()) { mapBuildTargetSources(&target); + m_testTargetSourceCoverageCount[&target] = 0; } } @@ -142,10 +143,21 @@ namespace TestImpact DependencyException, AZStd::string::format("Couldn't replace source coverage for %s, source file is an autogen input file", sourceCoverage.GetPath().c_str()).c_str()); - auto [it, inserted] = m_sourceDependencyMap.insert(sourceCoverage.GetPath().String()); - auto& [key, sourceDependency] = *it; + auto [sourceDependencyIt, inserted] = m_sourceDependencyMap.insert(sourceCoverage.GetPath().String()); + auto& [key, sourceDependency] = *sourceDependencyIt; - // Clear any existing coverage for the delta + // Knock down the source coverage count for the test targets and clear any existing coverage for the delta + for (const auto& testTarget : sourceDependency.m_coveringTestTargets) + { + if (auto coveringTestTargetIt = m_testTargetSourceCoverageCount.find(testTarget); + coveringTestTargetIt != m_testTargetSourceCoverageCount.end()) + { + if (coveringTestTargetIt->second > 0) + { + coveringTestTargetIt->second--; + } + } + } sourceDependency.m_coveringTestTargets.clear(); // Update the dependency with any new coverage data @@ -157,10 +169,13 @@ namespace TestImpact // Source to covering test target mapping sourceDependency.m_coveringTestTargets.insert(testTarget); + // Test target covering sources count + m_testTargetSourceCoverageCount[testTarget]++; + // Build target to covering test target mapping for (const auto& parentTarget : sourceDependency.m_parentTargets) - { - m_buildTargetCoverage[parentTarget.GetBuildTarget()].insert(testTarget); + { + m_buildTargetCoverage[parentTarget.GetBuildTarget()].insert(testTarget); } } else @@ -173,7 +188,7 @@ namespace TestImpact // If the new coverage data results in a parentless and coverageless entry, consider it a dead entry and remove accordingly if (sourceDependency.m_coveringTestTargets.empty() && sourceDependency.m_parentTargets.empty()) { - m_sourceDependencyMap.erase(it); + m_sourceDependencyMap.erase(sourceDependencyIt); } } } @@ -198,6 +213,14 @@ namespace TestImpact } } + void DynamicDependencyMap::ClearAllSourceCoverage() + { + for (const auto& [path, coverage] : m_sourceDependencyMap) + { + ReplaceSourceCoverage(SourceCoveringTestsList(AZStd::vector{ SourceCoveringTests(RepoPath(path)) })); + } + } + const ProductionTargetList& DynamicDependencyMap::GetProductionTargetList() const { return m_productionTargets; @@ -405,4 +428,33 @@ namespace TestImpact return ChangeDependencyList(AZStd::move(createDependencies), AZStd::move(updateDependencies), AZStd::move(deleteDependencies)); } + + AZStd::vector DynamicDependencyMap::GetCoveringTests() const + { + AZStd::vector covering; + for (const auto& [testTarget, coveringSources] : m_testTargetSourceCoverageCount) + { + if (coveringSources > 0) + { + covering.push_back(testTarget); + } + } + + return covering; + } + + AZStd::vector DynamicDependencyMap::GetNotCoveringTests() const + { + AZStd::vector notCovering; + for(const auto& [testTarget, coveringSources] : m_testTargetSourceCoverageCount) + { + if(coveringSources == 0) + { + notCovering.push_back(testTarget); + } + } + + return notCovering; + } + } // namespace TestImpact diff --git a/Code/Tools/TestImpactFramework/Runtime/Code/Source/Dependency/TestImpactDynamicDependencyMap.h b/Code/Tools/TestImpactFramework/Runtime/Code/Source/Dependency/TestImpactDynamicDependencyMap.h index 6faef76452..f209e22823 100644 --- a/Code/Tools/TestImpactFramework/Runtime/Code/Source/Dependency/TestImpactDynamicDependencyMap.h +++ b/Code/Tools/TestImpactFramework/Runtime/Code/Source/Dependency/TestImpactDynamicDependencyMap.h @@ -85,18 +85,27 @@ namespace TestImpact //! @param sourceCoverageDelta The source coverage delta to replace in the dependency map. void ReplaceSourceCoverage(const SourceCoveringTestsList& sourceCoverageDelta); + //! Clears all of the existing source coverage in the dependency map. + void ClearAllSourceCoverage(); + //! Exports the coverage of all sources in the dependency map. SourceCoveringTestsList ExportSourceCoverage() const; //! Gets the list of orphaned source files in the dependency map that have coverage data but belong to no parent build targets. AZStd::vector GetOrphanSourceFiles() const; - //! Applies the specified change list to the dynamic dependency map and resolves the change list to a change dependency list + //! Applies the specified change list to the dependency map and resolves the change list to a change dependency list //! containing the updated source dependencies for each source file in the change list. //! @param changeList The change list to apply and resolve. //! @returns The change list as resolved to the appropriate source dependencies. [[nodiscard]] ChangeDependencyList ApplyAndResoveChangeList(const ChangeList& changeList); + //! Returns the test targets that cover one or more sources in the repository. + AZStd::vector GetCoveringTests() const; + + //! Returns the test targets that do not cover any sources in the repository. + AZStd::vector GetNotCoveringTests() const; + private: //! Clears the source coverage of the specified sources. //! @note The covering targets for the parent test target(s) will not be pruned if those covering targets are removed. @@ -116,5 +125,8 @@ namespace TestImpact //! Mapping of autogen input sources to their generated output sources. AZStd::unordered_map> m_autogenInputToOutputMap; + + //! Number of sources that each test target in the repository covers. + AZStd::unordered_map m_testTargetSourceCoverageCount; }; } // namespace TestImpact diff --git a/Code/Tools/TestImpactFramework/Runtime/Code/Source/Dependency/TestImpactSourceCoveringTestsList.cpp b/Code/Tools/TestImpactFramework/Runtime/Code/Source/Dependency/TestImpactSourceCoveringTestsList.cpp index 101bdf862c..ce8bf64678 100644 --- a/Code/Tools/TestImpactFramework/Runtime/Code/Source/Dependency/TestImpactSourceCoveringTestsList.cpp +++ b/Code/Tools/TestImpactFramework/Runtime/Code/Source/Dependency/TestImpactSourceCoveringTestsList.cpp @@ -65,7 +65,7 @@ namespace TestImpact { AZStd::sort(m_coverage.begin(), m_coverage.end(), [](const SourceCoveringTests& lhs, const SourceCoveringTests& rhs) { - return lhs.GetPath() < rhs.GetPath(); + return lhs.GetPath().String() < rhs.GetPath().String(); }); } diff --git a/Code/Tools/TestImpactFramework/Runtime/Code/Source/Dependency/TestImpactSourceCoveringTestsSerializer.cpp b/Code/Tools/TestImpactFramework/Runtime/Code/Source/Dependency/TestImpactSourceCoveringTestsSerializer.cpp new file mode 100644 index 0000000000..7b89f3bda9 --- /dev/null +++ b/Code/Tools/TestImpactFramework/Runtime/Code/Source/Dependency/TestImpactSourceCoveringTestsSerializer.cpp @@ -0,0 +1,93 @@ +/* + * All or portions of this file Copyright (c) Amazon.com, Inc. or its affiliates or + * its licensors. + * + * For complete copyright and license terms please see the LICENSE at the root of this + * distribution (the "License"). All use of this software is governed by the License, + * or, if provided, by the license below or the license accompanying this file. Do not + * remove or modify any license notices. This file is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * + */ + +#include +#include + +#include +#include +#include +#include + +namespace TestImpact +{ + // Tag used to indicate that a given line is the name o a covering test target + constexpr char TargetTag = '-'; + + AZStd::string SerializeSourceCoveringTestsList(const SourceCoveringTestsList& sourceCoveringTestsList) + { + AZStd::string output; + output.reserve(1U << 24); // Reserve approx. 16Mib as the outputs can be quite large + + for (const auto& source : sourceCoveringTestsList.GetCoverage()) + { + // Source file + output += source.GetPath().String(); + output += "\n"; + + // Covering test targets + for (const auto& testTarget : source.GetCoveringTestTargets()) + { + output += AZStd::string::format("%c%s\n", TargetTag, testTarget.c_str()); + } + } + + // Add the newline so the deserializer can properly terminate on the last read line + output += "\n"; + + return output; + } + + SourceCoveringTestsList DeserializeSourceCoveringTestsList(const AZStd::string& sourceCoveringTestsListString) + { + AZStd::vector sourceCoveringTests; + AZStd::string source; + AZStd::vector coveringTests; + sourceCoveringTests.reserve(1U << 16); // Reserve for approx. 65k source files + const AZStd::string delim = "\n"; + auto start = 0U; + auto end = sourceCoveringTestsListString.find(delim); + + while (end != AZStd::string::npos) + { + const auto line = sourceCoveringTestsListString.substr(start, end - start); + if (line.starts_with(TargetTag)) + { + // This is a test target covering the most recent source discovered + coveringTests.push_back(line.substr(1, line.length() - 1)); + } + else + { + // This is a new source file so assign the accumulated test targets to the current source file before proceeding + if (!coveringTests.empty()) + { + sourceCoveringTests.push_back(SourceCoveringTests(source, AZStd::move(coveringTests))); + coveringTests.clear(); + } + + source = line; + } + + start = end + delim.length(); + end = sourceCoveringTestsListString.find(delim, start); + } + + // Ensure we properly assign the accumulated test targets to the most recent source discovered + if (!coveringTests.empty()) + { + sourceCoveringTests.push_back(SourceCoveringTests(source, AZStd::move(coveringTests))); + coveringTests.clear(); + } + + return SourceCoveringTestsList(AZStd::move(sourceCoveringTests)); + } +} // namespace TestImpact diff --git a/Code/Tools/TestImpactFramework/Runtime/Code/Source/Dependency/TestImpactSourceCoveringTestsSerializer.h b/Code/Tools/TestImpactFramework/Runtime/Code/Source/Dependency/TestImpactSourceCoveringTestsSerializer.h new file mode 100644 index 0000000000..901e43233c --- /dev/null +++ b/Code/Tools/TestImpactFramework/Runtime/Code/Source/Dependency/TestImpactSourceCoveringTestsSerializer.h @@ -0,0 +1,26 @@ +/* + * All or portions of this file Copyright (c) Amazon.com, Inc. or its affiliates or + * its licensors. + * + * For complete copyright and license terms please see the LICENSE at the root of this + * distribution (the "License"). All use of this software is governed by the License, + * or, if provided, by the license below or the license accompanying this file. Do not + * remove or modify any license notices. This file is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * + */ + +#pragma once + +#include + +#include + +namespace TestImpact +{ + //! Serializes the specified source covering tests list to plain text format. + AZStd::string SerializeSourceCoveringTestsList(const SourceCoveringTestsList& sourceCoveringTestsList); + + //! Deserializes a source covering tests list from the specified source covering tests data in plain text format. + SourceCoveringTestsList DeserializeSourceCoveringTestsList(const AZStd::string& sourceCoveringTestsListString); +} // namespace TestImpact diff --git a/Code/Tools/TestImpactFramework/Runtime/Code/Source/Dependency/TestImpactTestSelectorAndPrioritizer.cpp b/Code/Tools/TestImpactFramework/Runtime/Code/Source/Dependency/TestImpactTestSelectorAndPrioritizer.cpp index 549ef0c108..35e6679fdc 100644 --- a/Code/Tools/TestImpactFramework/Runtime/Code/Source/Dependency/TestImpactTestSelectorAndPrioritizer.cpp +++ b/Code/Tools/TestImpactFramework/Runtime/Code/Source/Dependency/TestImpactTestSelectorAndPrioritizer.cpp @@ -25,7 +25,7 @@ namespace TestImpact } AZStd::vector TestSelectorAndPrioritizer::SelectTestTargets( - const ChangeDependencyList& changeDependencyList, TestSelectionStrategy testSelectionStrategy) + const ChangeDependencyList& changeDependencyList, Policy::TestPrioritization testSelectionStrategy) { const auto selectedTestTargetAndDependerMap = SelectTestTargets(changeDependencyList); const auto prioritizedSelectedTests = PrioritizeSelectedTestTargets(selectedTestTargetAndDependerMap, testSelectionStrategy); @@ -210,7 +210,7 @@ namespace TestImpact AZStd::vector TestSelectorAndPrioritizer::PrioritizeSelectedTestTargets( const SelectedTestTargetAndDependerMap& selectedTestTargetAndDependerMap, - [[maybe_unused]]TestSelectionStrategy testSelectionStrategy) + [[maybe_unused]] Policy::TestPrioritization testSelectionStrategy) { AZStd::vector selectedTestTargets; diff --git a/Code/Tools/TestImpactFramework/Runtime/Code/Source/Dependency/TestImpactTestSelectorAndPrioritizer.h b/Code/Tools/TestImpactFramework/Runtime/Code/Source/Dependency/TestImpactTestSelectorAndPrioritizer.h index c742176073..6ee3a616c4 100644 --- a/Code/Tools/TestImpactFramework/Runtime/Code/Source/Dependency/TestImpactTestSelectorAndPrioritizer.h +++ b/Code/Tools/TestImpactFramework/Runtime/Code/Source/Dependency/TestImpactTestSelectorAndPrioritizer.h @@ -12,6 +12,8 @@ #pragma once +#include + #include #include @@ -25,13 +27,6 @@ namespace TestImpact class BuildTarget; class TestTarget; - //! Strategy for selecting tests given a set of source changes. - enum class TestSelectionStrategy : bool - { - SelectOnly, //!< Select tests only, do not attempt prioritization of those selected tests. - SelectAndPriotitize //!< Select tests and prioritize according to dependency graph locality of coverer and coveree. - }; - //! Map of build targets and their dependency graph data. //! For test targets, the dependency graph data is that of the build targets which the test target depends on. //! For production targets, the dependency graph is that of the build targets that depend on it (dependers). @@ -52,7 +47,7 @@ namespace TestImpact //! Select the covering test targets for the given set of source changes and optionally prioritizes said test selection. //! @param changeDependencyList The resolved list of source dependencies for the CRUD source changes. //! @param testSelectionStrategy The test selection and prioritization strategy to apply to the given CRUD source changes. - AZStd::vector SelectTestTargets(const ChangeDependencyList& changeDependencyList, TestSelectionStrategy testSelectionStrategy); + AZStd::vector SelectTestTargets(const ChangeDependencyList& changeDependencyList, Policy::TestPrioritization testSelectionStrategy); private: //! Map of selected test targets and the production targets they cover for the given set of source changes. @@ -69,7 +64,7 @@ namespace TestImpact //! @param testSelectionStrategy The test selection strategy to prioritize the selected tests. //! @returns The selected tests either in either arbitrary order or in prioritized with highest priority first. AZStd::vector PrioritizeSelectedTestTargets( - const SelectedTestTargetAndDependerMap& selectedTestTargetAndDependerMap, TestSelectionStrategy testSelectionStrategy); + const SelectedTestTargetAndDependerMap& selectedTestTargetAndDependerMap, Policy::TestPrioritization testSelectionStrategy); const DynamicDependencyMap* m_dynamicDependencyMap; DependencyGraphDataMap m_dependencyGraphDataMap; diff --git a/Code/Tools/TestImpactFramework/Runtime/Code/Source/Platform/Windows/TestEngine/TestImpactWin32_TestEngineJobFailure.cpp b/Code/Tools/TestImpactFramework/Runtime/Code/Source/Platform/Windows/TestEngine/TestImpactWin32_TestEngineJobFailure.cpp new file mode 100644 index 0000000000..578130882d --- /dev/null +++ b/Code/Tools/TestImpactFramework/Runtime/Code/Source/Platform/Windows/TestEngine/TestImpactWin32_TestEngineJobFailure.cpp @@ -0,0 +1,35 @@ +/* + * All or portions of this file Copyright (c) Amazon.com, Inc. or its affiliates or + * its licensors. + * + * For complete copyright and license terms please see the LICENSE at the root of this + * distribution (the "License"). All use of this software is governed by the License, + * or, if provided, by the license below or the license accompanying this file. Do not + * remove or modify any license notices. This file is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * + */ + +#include + +namespace TestImpact +{ + // Known error codes for test instrumentation + namespace ErrorCodes + { + namespace OpenCppCoverage + { + static constexpr ReturnCode InvalidArgs = 0x9F8C8E5C; + } + } + + AZStd::optional CheckForKnownTestInstrumentErrorCode(ReturnCode returnCode) + { + if (returnCode == ErrorCodes::OpenCppCoverage::InvalidArgs) + { + return Client::TestRunResult::FailedToExecute; + } + + return AZStd::nullopt; + } +} diff --git a/Code/Tools/TestImpactFramework/Runtime/Code/Source/TestEngine/TestImpactTestEngineJobFailure.cpp b/Code/Tools/TestImpactFramework/Runtime/Code/Source/TestEngine/TestImpactTestEngineJobFailure.cpp new file mode 100644 index 0000000000..debf1406f2 --- /dev/null +++ b/Code/Tools/TestImpactFramework/Runtime/Code/Source/TestEngine/TestImpactTestEngineJobFailure.cpp @@ -0,0 +1,82 @@ +/* + * All or portions of this file Copyright (c) Amazon.com, Inc. or its affiliates or + * its licensors. + * + * For complete copyright and license terms please see the LICENSE at the root of this + * distribution (the "License"). All use of this software is governed by the License, + * or, if provided, by the license below or the license accompanying this file. Do not + * remove or modify any license notices. This file is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * + */ + +#include + +namespace TestImpact +{ + // Known error codes for test runner and test library + namespace ErrorCodes + { + namespace GTest + { + static constexpr ReturnCode Unsuccessful = 1; + } + + namespace AZTestRunner + { + static constexpr ReturnCode InvalidArgs = 101; + static constexpr ReturnCode FailedToFindTargetBinary = 102; + static constexpr ReturnCode SymbolNotFound = 103; + static constexpr ReturnCode ModuleSkipped = 104; + } + } + + AZStd::optional CheckForKnownTestRunnerErrorCode(int returnCode) + { + switch (returnCode) + { + // We will consider test targets that technically execute but their launcher or unit test library return a know error + // code that pertains to incorrect argument usage as test targets that failed to execute + case ErrorCodes::AZTestRunner::InvalidArgs: + case ErrorCodes::AZTestRunner::FailedToFindTargetBinary: + case ErrorCodes::AZTestRunner::ModuleSkipped: + case ErrorCodes::AZTestRunner::SymbolNotFound: + return Client::TestRunResult::FailedToExecute; + default: + return AZStd::nullopt; + } + } + + AZStd::optional CheckForKnownTestLibraryErrorCode(int returnCode) + { + if (returnCode == ErrorCodes::GTest::Unsuccessful) + { + return Client::TestRunResult::TestFailures; + } + + return AZStd::nullopt; + } + + AZStd::optional CheckForAnyKnownErrorCode(ReturnCode returnCode) + { + if (const auto result = CheckForKnownTestInstrumentErrorCode(returnCode); + result != AZStd::nullopt) + { + return result.value(); + } + + if (const auto result = CheckForKnownTestRunnerErrorCode(returnCode); + result != AZStd::nullopt) + { + return result.value(); + } + + if (const auto result = CheckForKnownTestLibraryErrorCode(returnCode); + result != AZStd::nullopt) + { + return result.value(); + } + + return AZStd::nullopt; + } +} diff --git a/Code/Tools/TestImpactFramework/Runtime/Code/Source/TestEngine/TestImpactTestEngineJobFailure.h b/Code/Tools/TestImpactFramework/Runtime/Code/Source/TestEngine/TestImpactTestEngineJobFailure.h new file mode 100644 index 0000000000..aef4e962c5 --- /dev/null +++ b/Code/Tools/TestImpactFramework/Runtime/Code/Source/TestEngine/TestImpactTestEngineJobFailure.h @@ -0,0 +1,29 @@ +/* + * All or portions of this file Copyright (c) Amazon.com, Inc. or its affiliates or + * its licensors. + * + * For complete copyright and license terms please see the LICENSE at the root of this + * distribution (the "License"). All use of this software is governed by the License, + * or, if provided, by the license below or the license accompanying this file. Do not + * remove or modify any license notices. This file is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * + */ + +#pragma once + +#include + +#include + +#include + +namespace TestImpact +{ + AZStd::optional CheckForKnownTestInstrumentErrorCode(ReturnCode returnCode); + AZStd::optional CheckForKnownTestRunnerErrorCode(ReturnCode returnCode); + AZStd::optional CheckForKnownTestLibraryErrorCode(ReturnCode returnCode); + + AZStd::optional CheckForAnyKnownErrorCode(ReturnCode returnCode); + +}