diff --git a/Code/Tools/PythonBindingsExample/CMakeLists.txt b/Code/Tools/PythonBindingsExample/CMakeLists.txt index 3abd24b5fc..90cb718ce4 100644 --- a/Code/Tools/PythonBindingsExample/CMakeLists.txt +++ b/Code/Tools/PythonBindingsExample/CMakeLists.txt @@ -114,6 +114,6 @@ if(PAL_TRAIT_BUILD_TESTS_SUPPORTED) ly_add_googletest( NAME AZ::PythonBindingsExample.Tests - TEST_COMMAND $ + TEST_COMMAND $ --unittest ) endif() diff --git a/Code/Tools/TestImpactFramework/Runtime/Code/Include/TestImpactFramework/TestImpactBitwise.h b/Code/Tools/TestImpactFramework/Runtime/Code/Include/TestImpactFramework/TestImpactBitwise.h deleted file mode 100644 index 73278f6be6..0000000000 --- a/Code/Tools/TestImpactFramework/Runtime/Code/Include/TestImpactFramework/TestImpactBitwise.h +++ /dev/null @@ -1,35 +0,0 @@ -/* - * All or portions of this file Copyright (c) Amazon.com, Inc. or its affiliates or - * its licensors. - * - * For complete copyright and license terms please see the LICENSE at the root of this - * distribution (the "License"). All use of this software is governed by the License, - * or, if provided, by the license below or the license accompanying this file. Do not - * remove or modify any license notices. This file is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * - */ - -#pragma once - -namespace TestImpact -{ - //! Convinience namespace for allowing bitwise operations on enum classes. - //! @note Any types declared in this namespace will have the bitwise operator overloads declared within. - namespace Bitwise - { - template - Flags operator|(Flags lhs, Flags rhs) - { - return static_cast( - static_cast::type>(lhs) | static_cast::type>(rhs)); - } - - template - bool IsFlagSet(Flags flags, Flags flag) - { - return static_cast( - static_cast::type>(flags) & static_cast::type>(flag)); - } - } // namespace Bitwise -} // namespace TestImpact diff --git a/Code/Tools/TestImpactFramework/Runtime/Code/Include/TestImpactFramework/TestImpactCallback.h b/Code/Tools/TestImpactFramework/Runtime/Code/Include/TestImpactFramework/TestImpactCallback.h deleted file mode 100644 index a8c9542b47..0000000000 --- a/Code/Tools/TestImpactFramework/Runtime/Code/Include/TestImpactFramework/TestImpactCallback.h +++ /dev/null @@ -1,23 +0,0 @@ -/* - * All or portions of this file Copyright (c) Amazon.com, Inc. or its affiliates or - * its licensors. - * - * For complete copyright and license terms please see the LICENSE at the root of this - * distribution (the "License"). All use of this software is governed by the License, - * or, if provided, by the license below or the license accompanying this file. Do not - * remove or modify any license notices. This file is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * - */ - -#pragma once - -namespace TestImpact -{ - //! Generic callback result used by test impact systems. - enum class CallbackResult : bool - { - Continue, - Abort - }; -} // namespace TestImpact diff --git a/Code/Tools/TestImpactFramework/Runtime/Code/Include/TestImpactFramework/TestImpactFrameworkPath.h b/Code/Tools/TestImpactFramework/Runtime/Code/Include/TestImpactFramework/TestImpactFrameworkPath.h deleted file mode 100644 index affab6daa7..0000000000 --- a/Code/Tools/TestImpactFramework/Runtime/Code/Include/TestImpactFramework/TestImpactFrameworkPath.h +++ /dev/null @@ -1,41 +0,0 @@ -/* - * All or portions of this file Copyright (c) Amazon.com, Inc. or its affiliates or - * its licensors. - * - * For complete copyright and license terms please see the LICENSE at the root of this - * distribution (the "License"). All use of this software is governed by the License, - * or, if provided, by the license below or the license accompanying this file. Do not - * remove or modify any license notices. This file is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * - */ - -#pragma once - -#include - -namespace TestImpact -{ - //! Wrapper for OS paths relative to a specified parent path. - //! @note Mimics path semantics only, makes no guarantees about validity. - class FrameworkPath - { - public: - FrameworkPath() = default; - //! Creates a path with no parent. - explicit FrameworkPath(const AZ::IO::Path& absolutePath); - //! Creates a path with an absolute path and path relative to the specified parent path. - explicit FrameworkPath(const AZ::IO::Path& absolutePath, const FrameworkPath& relativeTo); - - //! Retrieves the absolute path. - const AZ::IO::Path& Absolute() const; - //! Retrieves the path relative to the specified parent path. - const AZ::IO::Path& Relative() const; - - private: - //! The absolute path value. - AZ::IO::Path m_absolutePath; - //! The path value relative to the specified parent path. - AZ::IO::Path m_relativePath; - }; -} // namespace TestImpact diff --git a/Code/Tools/TestImpactFramework/Runtime/Code/Source/Artifact/Dynamic/TestImpactChangeList.h b/Code/Tools/TestImpactFramework/Runtime/Code/Source/Artifact/Dynamic/TestImpactChangeList.h deleted file mode 100644 index 6d966385ac..0000000000 --- a/Code/Tools/TestImpactFramework/Runtime/Code/Source/Artifact/Dynamic/TestImpactChangeList.h +++ /dev/null @@ -1,27 +0,0 @@ -/* - * All or portions of this file Copyright (c) Amazon.com, Inc. or its affiliates or - * its licensors. - * - * For complete copyright and license terms please see the LICENSE at the root of this - * distribution (the "License"). All use of this software is governed by the License, - * or, if provided, by the license below or the license accompanying this file. Do not - * remove or modify any license notices. This file is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * - */ - -#pragma once - -#include -#include - -namespace TestImpact -{ - //! Artifact produced by the unified diff parsing process representing the file CRUD operations of a given diff. - struct ChangeList - { - AZStd::vector m_createdFiles; - AZStd::vector m_updatedFiles; - AZStd::vector m_deletedFiles; - }; -} // namespace TestImpact diff --git a/Code/Tools/TestImpactFramework/Runtime/Code/Source/Artifact/Factory/TestImpactChangeListFactory.cpp b/Code/Tools/TestImpactFramework/Runtime/Code/Source/Artifact/Factory/TestImpactChangeListFactory.cpp deleted file mode 100644 index 4b2a33355b..0000000000 --- a/Code/Tools/TestImpactFramework/Runtime/Code/Source/Artifact/Factory/TestImpactChangeListFactory.cpp +++ /dev/null @@ -1,175 +0,0 @@ -/* - * All or portions of this file Copyright (c) Amazon.com, Inc. or its affiliates or - * its licensors. - * - * For complete copyright and license terms please see the LICENSE at the root of this - * distribution (the "License"). All use of this software is governed by the License, - * or, if provided, by the license below or the license accompanying this file. Do not - * remove or modify any license notices. This file is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * - */ - -#include -#include - -#include - -namespace TestImpact -{ - namespace Utils - { - template - auto split(C&& str, const AZStd::string& delimiter) - { - AZStd::vector strings; - - for (auto p = str.data(), end = p + str.length(); p != end; p += ((p == end) ? 0 : delimiter.length())) - { - const auto pre = p; - p = AZStd::search(pre, end, delimiter.cbegin(), delimiter.cend()); - - if (p != pre) - { - strings.emplace_back(pre, p - pre); - } - } - - return strings; - } - } // namespace Utils - - namespace UnifiedDiff - { - class UnifiedDiffParser - { - public: - ChangeList Parse(const AZStd::string& unifiedDiff); - - private: - AZStd::optional GetTargetFile(const AZStd::string_view& targetFile); - ChangeList GenerateChangelist( - const AZStd::vector>& src, - const AZStd::vector>& dst); - - const AZStd::string m_srcFilePrefix = "--- "; - const AZStd::string m_dstFilePrefix = "+++ "; - const AZStd::string m_gitTargetPrefix = "b/"; - const AZStd::string m_perforceTargetPrefix = "/b/"; - const AZStd::string m_renameFromPrefix = "rename from "; - const AZStd::string m_renameToPrefix = "rename to "; - const AZStd::string m_nullFile = "/dev/null"; - bool m_hasGitHeader = false; - }; - - AZStd::optional UnifiedDiffParser::GetTargetFile(const AZStd::string_view& targetFile) - { - size_t startIndex = 0; - - if (targetFile.starts_with(m_renameFromPrefix)) - { - startIndex = m_renameFromPrefix.length(); - } - else if (targetFile.starts_with(m_renameToPrefix)) - { - startIndex = m_renameToPrefix.length(); - } - else if (targetFile.find(m_nullFile) != AZStd::string::npos) - { - return AZStd::nullopt; - } - else - { - startIndex = m_hasGitHeader ? m_gitTargetPrefix.size() + m_dstFilePrefix.size() - : m_perforceTargetPrefix.size() + m_dstFilePrefix.size(); - } - - const auto endIndex = targetFile.find('\t'); - - if (endIndex != AZStd::string::npos) - { - return targetFile.substr(startIndex, endIndex - startIndex); - } - - return targetFile.substr(startIndex); - } - - ChangeList UnifiedDiffParser::GenerateChangelist( - const AZStd::vector>& src, const AZStd::vector>& dst) - { - AZ_TestImpact_Eval(src.size() == dst.size(), ArtifactException, "Change list source and destination file count mismatch"); - ChangeList changelist; - - for (size_t i = 0; i < src.size(); i++) - { - if (!src[i].has_value()) - { - changelist.m_createdFiles.emplace_back(dst[i].value()); - } - else if (!dst[i].has_value()) - { - changelist.m_deletedFiles.emplace_back(src[i].value()); - } - else if (src[i] != dst[i]) - { - changelist.m_deletedFiles.emplace_back(src[i].value()); - changelist.m_createdFiles.emplace_back(dst[i].value()); - } - else - { - changelist.m_updatedFiles.emplace_back(src[i].value()); - } - } - - return changelist; - } - - ChangeList UnifiedDiffParser::Parse(const AZStd::string& unifiedDiff) - { - const AZStd::string GitHeader = "diff --git"; - const auto lines = Utils::split(unifiedDiff, "\n"); - - AZStd::vector> src; - AZStd::vector> dst; - - for (const auto& line : lines) - { - if (line.starts_with(GitHeader)) - { - m_hasGitHeader = true; - } - else if (line.starts_with(m_srcFilePrefix)) - { - src.emplace_back(GetTargetFile(line)); - } - else if (line.starts_with(m_dstFilePrefix)) - { - dst.emplace_back(GetTargetFile(line)); - } - else if (line.starts_with(m_renameFromPrefix)) - { - src.emplace_back(GetTargetFile(line)); - } - else if (line.starts_with(m_renameToPrefix)) - { - dst.emplace_back(GetTargetFile(line)); - } - } - - return GenerateChangelist(src, dst); - } - - ChangeList ChangeListFactory(const AZStd::string& unifiedDiff) - { - AZ_TestImpact_Eval(!unifiedDiff.empty(), ArtifactException, "Unified diff is empty"); - UnifiedDiffParser diff; - ChangeList changeList = diff.Parse(unifiedDiff); - AZ_TestImpact_Eval( - !changeList.m_createdFiles.empty() || - !changeList.m_updatedFiles.empty() || - !changeList.m_deletedFiles.empty(), - ArtifactException, "The unified diff contained no changes"); - return changeList; - } - } // namespace UnifiedDiff -} // namespace TestImpact diff --git a/Code/Tools/TestImpactFramework/Runtime/Code/Source/Artifact/Factory/TestImpactChangeListFactory.h b/Code/Tools/TestImpactFramework/Runtime/Code/Source/Artifact/Factory/TestImpactChangeListFactory.h deleted file mode 100644 index 82074a66ef..0000000000 --- a/Code/Tools/TestImpactFramework/Runtime/Code/Source/Artifact/Factory/TestImpactChangeListFactory.h +++ /dev/null @@ -1,29 +0,0 @@ -/* - * All or portions of this file Copyright (c) Amazon.com, Inc. or its affiliates or - * its licensors. - * - * For complete copyright and license terms please see the LICENSE at the root of this - * distribution (the "License"). All use of this software is governed by the License, - * or, if provided, by the license below or the license accompanying this file. Do not - * remove or modify any license notices. This file is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * - */ - -#pragma once - -#include - -#include -#include - -namespace TestImpact -{ - namespace UnifiedDiff - { - //! Constructs a change list artifact from the specified unified diff data. - //! @param unifiedDiffData The raw change list data in unified diff format. - //! @return The constructed change list artifact. - ChangeList ChangeListFactory(const AZStd::string& unifiedDiffData); - } // namespace UnifiedDiff -} // namespace TestImpact diff --git a/Code/Tools/TestImpactFramework/Runtime/Code/Source/TestImpactFrameworkPath.cpp b/Code/Tools/TestImpactFramework/Runtime/Code/Source/TestImpactFrameworkPath.cpp deleted file mode 100644 index 6b5d671776..0000000000 --- a/Code/Tools/TestImpactFramework/Runtime/Code/Source/TestImpactFrameworkPath.cpp +++ /dev/null @@ -1,38 +0,0 @@ -/* - * All or portions of this file Copyright (c) Amazon.com, Inc. or its affiliates or - * its licensors. - * - * For complete copyright and license terms please see the LICENSE at the root of this - * distribution (the "License"). All use of this software is governed by the License, - * or, if provided, by the license below or the license accompanying this file. Do not - * remove or modify any license notices. This file is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * - */ - -#include - -namespace TestImpact -{ - FrameworkPath::FrameworkPath(const AZ::IO::Path& absolutePath) - { - m_absolutePath = AZ::IO::Path(absolutePath).MakePreferred(); - m_relativePath = m_absolutePath.LexicallyRelative(m_absolutePath); - } - - FrameworkPath::FrameworkPath(const AZ::IO::Path& absolutePath, const FrameworkPath& relativeTo) - { - m_absolutePath = AZ::IO::Path(absolutePath).MakePreferred(); - m_relativePath = m_absolutePath.LexicallyRelative(relativeTo.Absolute()); - } - - const AZ::IO::Path& FrameworkPath::Absolute() const - { - return m_absolutePath; - } - - const AZ::IO::Path& FrameworkPath::Relative() const - { - return m_relativePath; - } -} // namespace TestImpact diff --git a/Code/Tools/TestImpactFramework/Runtime/Code/Tests/Artifact/TestImpactChangeListFactoryTest.cpp b/Code/Tools/TestImpactFramework/Runtime/Code/Tests/Artifact/TestImpactChangeListFactoryTest.cpp deleted file mode 100644 index aeb8f118a4..0000000000 --- a/Code/Tools/TestImpactFramework/Runtime/Code/Tests/Artifact/TestImpactChangeListFactoryTest.cpp +++ /dev/null @@ -1,288 +0,0 @@ -/* - * All or portions of this file Copyright (c) Amazon.com, Inc. or its affiliates or - * its licensors. - * - * For complete copyright and license terms please see the LICENSE at the root of this - * distribution (the "License"). All use of this software is governed by the License, - * or, if provided, by the license below or the license accompanying this file. Do not - * remove or modify any license notices. This file is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * - */ - -#include -#include - -#include -#include - -namespace UnitTest -{ - TEST(ChangeListFactoryTest, NoRawData_ExpectArtifactException) - { - // Given an empty unified diff string - const AZStd::string unifiedDiff; - - try - { - // When attempting to construct the change list - const TestImpact::ChangeList changeList = TestImpact::UnifiedDiff::ChangeListFactory(unifiedDiff); - - // Do not expect this statement to be reachable - FAIL(); - } - catch ([[maybe_unused]] const TestImpact::ArtifactException& e) - { - // Expect an artifact exception - SUCCEED(); - } - catch (...) - { - // Do not expect any other exceptions - FAIL(); - } - } - - TEST(ChangeListFactoryTest, NoChanges_ExpectArtifactException) - { - // Given a unified diff string with no changes - const AZStd::string unifiedDiff = "On this day in 1738 absolutely nothing happened"; - - try - { - // When attempting to construct the change list - const TestImpact::ChangeList changeList = TestImpact::UnifiedDiff::ChangeListFactory(unifiedDiff); - - // Do not expect this statement to be reachable - FAIL(); - } - catch ([[maybe_unused]] const TestImpact::ArtifactException& e) - { - // Expect an artifact exception - SUCCEED(); - } - catch (...) - { - // Do not expect any other exceptions - FAIL(); - } - } - - TEST(ChangeListFactoryTest, CreateOnly_ExpectValidChangeListWithFileCreateOperations) - { - // Given a unified diff with only one file creation and no file updates or deletions - const AZStd::string unifiedDiff = - "From f642a2f698452fc18484758b0046132415f09467 Mon Sep 17 00:00:00 2001\n" - "From: user \n" - "Date: Sat, 13 Mar 2021 22:58:07 +0000\n" - "Subject: Test\n" - "\n" - "---\n" - " New.txt | 1 +\n" - " create mode 100644 New.txt\n" - "diff --git a/New.txt b/New.txt\n" - "new file mode 100644\n" - "index 0000000..30d74d2\n" - "--- /dev/null\n" - "+++ b/New.txt\n" - "@@ -0,0 +1 @@\n" - "+test\n" - "\\ No newline at end of file\n" - "-- \n" - "2.30.0.windows.2\n" - "\n" - "\n"; - - // When attempting to construct the change list - const TestImpact::ChangeList changeList = TestImpact::UnifiedDiff::ChangeListFactory(unifiedDiff); - - // Expect the change list to contain the 1 created file - EXPECT_EQ(changeList.m_createdFiles.size(), 1); - EXPECT_TRUE( - AZStd::find(changeList.m_createdFiles.begin(), changeList.m_createdFiles.end(), "New.txt") != changeList.m_createdFiles.end()); - - // Expect the change list to contain no updated files - EXPECT_TRUE(changeList.m_updatedFiles.empty()); - - // Expect the change list to contain no deleted files - EXPECT_TRUE(changeList.m_deletedFiles.empty()); - } - - TEST(ChangeListFactoryTest, UpdateOnly_ExpectValidChangeListWithFileUpdateOperations) - { - // Given a unified diff with only one file update and no file creations or deletions - const AZStd::string unifiedDiff = - "From f642a2f698452fc18484758b0046132415f09467 Mon Sep 17 00:00:00 2001\n" - "From: user \n" - "Date: Sat, 13 Mar 2021 22:58:07 +0000\n" - "Subject: Test\n" - "\n" - "---\n" - " A.txt | 2 +-\n" - "diff --git a/A.txt b/A.txt\n" - "index 7c4a013..e132db2 100644\n" - "--- a/A.txt\n" - "+++ b/A.txt\n" - "@@ -1 +1 @@\n" - "-aaa\n" - "\\ No newline at end of file\n" - "+zzz\n" - "\\ No newline at end of file\n" - "-- \n" - "2.30.0.windows.2\n" - "\n" - "\n"; - - // When attempting to construct the change list - const TestImpact::ChangeList changeList = TestImpact::UnifiedDiff::ChangeListFactory(unifiedDiff); - - // Expect the change list to contain no created files - EXPECT_TRUE(changeList.m_createdFiles.empty()); - - // Expect the change list to contain one updated file - EXPECT_EQ(changeList.m_updatedFiles.size(), 1); - EXPECT_TRUE( - AZStd::find(changeList.m_updatedFiles.begin(), changeList.m_updatedFiles.end(), "A.txt") != changeList.m_updatedFiles.end()); - - // Expect the change list to contain no deleted files - EXPECT_TRUE(changeList.m_deletedFiles.empty()); - } - - TEST(ChangeListFactoryTest, DeleteOnly_ExpectValidChangeListWithFileDeleteOperations) - { - // Given a unified diff with only one file deletion and no file creations or updates - const AZStd::string unifiedDiff = - "From f642a2f698452fc18484758b0046132415f09467 Mon Sep 17 00:00:00 2001\n" - "From: user \n" - "Date: Sat, 13 Mar 2021 22:58:07 +0000\n" - "Subject: Test\n" - "\n" - "---\n" - " B.txt | 1 -\n" - " delete mode 100644 B.txt\n" - "diff --git a/B.txt b/B.txt\n" - "deleted file mode 100644\n" - "index 01f02e3..0000000\n" - "--- a/B.txt\n" - "+++ /dev/null\n" - "@@ -1 +0,0 @@\n" - "-bbb\n" - "\\ No newline at end of file\n" - "-- \n" - "2.30.0.windows.2\n" - "\n" - "\n"; - - // When attempting to construct the change list - const TestImpact::ChangeList changeList = TestImpact::UnifiedDiff::ChangeListFactory(unifiedDiff); - - // Expect the change list to contain no created files - EXPECT_TRUE(changeList.m_createdFiles.empty()); - - // Expect the change list to contain no updated files - EXPECT_TRUE(changeList.m_updatedFiles.empty()); - - // Expect the change list to contain one deleted file - EXPECT_EQ(changeList.m_deletedFiles.size(), 1); - EXPECT_TRUE( - AZStd::find(changeList.m_deletedFiles.begin(), changeList.m_deletedFiles.end(), "B.txt") != changeList.m_deletedFiles.end()); - } - - TEST(ChangeListFactoryTest, ParseUnifiedDiffWithAllPossibleOperations_ExpectChangeListMatchingOperations) - { - // Given a unified diff with created files, updated files, deleted files, renamed files and moved files - const AZStd::string unifiedDiff = - "From f642a2f698452fc18484758b0046132415f09467 Mon Sep 17 00:00:00 2001\n" - "From: user \n" - "Date: Sat, 13 Mar 2021 22:58:07 +0000\n" - "Subject: Test\n" - "\n" - "---\n" - " A.txt | 2 +-\n" - " B.txt | 1 -\n" - " D.txt => Foo/D.txt | 0\n" - " E.txt => Foo/Y.txt | 0\n" - " New.txt | 1 +\n" - " C.txt => X.txt | 0\n" - " 6 files changed, 2 insertions(+), 2 deletions(-)\n" - " delete mode 100644 B.txt\n" - " rename D.txt => Foo/D.txt (100%)\n" - " rename E.txt => Foo/Y.txt (100%)\n" - " create mode 100644 New.txt\n" - " rename C.txt => X.txt (100%)\n" - "\n" - "diff --git a/A.txt b/A.txt\n" - "index 7c4a013..e132db2 100644\n" - "--- a/A.txt\n" - "+++ b/A.txt\n" - "@@ -1 +1 @@\n" - "-aaa\n" - "\\ No newline at end of file\n" - "+zzz\n" - "\\ No newline at end of file\n" - "diff --git a/B.txt b/B.txt\n" - "deleted file mode 100644\n" - "index 01f02e3..0000000\n" - "--- a/B.txt\n" - "+++ /dev/null\n" - "@@ -1 +0,0 @@\n" - "-bbb\n" - "\\ No newline at end of file\n" - "diff --git a/D.txt b/Foo/D.txt\n" - "similarity index 100%\n" - "rename from D.txt\n" - "rename to Foo/D.txt\n" - "diff --git a/E.txt b/Foo/Y.txt\n" - "similarity index 100%\n" - "rename from E.txt\n" - "rename to Foo/Y.txt\n" - "diff --git a/New.txt b/New.txt\n" - "new file mode 100644\n" - "index 0000000..30d74d2\n" - "--- /dev/null\n" - "+++ b/New.txt\n" - "@@ -0,0 +1 @@\n" - "+test\n" - "\\ No newline at end of file\n" - "diff --git a/C.txt b/X.txt\n" - "similarity index 100%\n" - "rename from C.txt\n" - "rename to X.txt\n" - "-- \n" - "2.30.0.windows.2\n" - "\n" - "\n"; - - // When attempting to construct the change list - const TestImpact::ChangeList changeList = TestImpact::UnifiedDiff::ChangeListFactory(unifiedDiff); - - // Expect the change list to contain the 4 created files - EXPECT_EQ(changeList.m_createdFiles.size(), 4); - EXPECT_TRUE( - AZStd::find(changeList.m_createdFiles.begin(), changeList.m_createdFiles.end(), "Foo/D.txt") != - changeList.m_createdFiles.end()); - EXPECT_TRUE( - AZStd::find(changeList.m_createdFiles.begin(), changeList.m_createdFiles.end(), "Foo/Y.txt") != - changeList.m_createdFiles.end()); - EXPECT_TRUE( - AZStd::find(changeList.m_createdFiles.begin(), changeList.m_createdFiles.end(), "X.txt") != changeList.m_createdFiles.end()); - EXPECT_TRUE( - AZStd::find(changeList.m_createdFiles.begin(), changeList.m_createdFiles.end(), "New.txt") != changeList.m_createdFiles.end()); - - // Expect the change list to contain the 1 updated file - EXPECT_EQ(changeList.m_updatedFiles.size(), 1); - EXPECT_TRUE( - AZStd::find(changeList.m_updatedFiles.begin(), changeList.m_updatedFiles.end(), "A.txt") != changeList.m_updatedFiles.end()); - - // Expect the change list to contain the 4 deleted files - EXPECT_EQ(changeList.m_deletedFiles.size(), 4); - EXPECT_TRUE( - AZStd::find(changeList.m_deletedFiles.begin(), changeList.m_deletedFiles.end(), "B.txt") != changeList.m_deletedFiles.end()); - EXPECT_TRUE( - AZStd::find(changeList.m_deletedFiles.begin(), changeList.m_deletedFiles.end(), "D.txt") != changeList.m_deletedFiles.end()); - EXPECT_TRUE( - AZStd::find(changeList.m_deletedFiles.begin(), changeList.m_deletedFiles.end(), "E.txt") != changeList.m_deletedFiles.end()); - EXPECT_TRUE( - AZStd::find(changeList.m_deletedFiles.begin(), changeList.m_deletedFiles.end(), "C.txt") != changeList.m_deletedFiles.end()); - } -} // namespace UnitTest diff --git a/Code/Tools/TestImpactFramework/Runtime/Code/Tests/Test/TestImpactInstrumentedTestRunnerTest.cpp b/Code/Tools/TestImpactFramework/Runtime/Code/Tests/Test/TestImpactInstrumentedTestRunnerTest.cpp deleted file mode 100644 index 897eda012c..0000000000 --- a/Code/Tools/TestImpactFramework/Runtime/Code/Tests/Test/TestImpactInstrumentedTestRunnerTest.cpp +++ /dev/null @@ -1,822 +0,0 @@ -/* - * All or portions of this file Copyright (c) Amazon.com, Inc. or its affiliates or - * its licensors. - * - * For complete copyright and license terms please see the LICENSE at the root of this - * distribution (the "License"). All use of this software is governed by the License, - * or, if provided, by the license below or the license accompanying this file. Do not - * remove or modify any license notices. This file is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * - */ - -#include -#include - -#include -#include -#include - -#include -#include -#include -#include -#include -#include -#include -#include -#include - -namespace UnitTest -{ - using JobExceptionPolicy = TestImpact::InstrumentedTestRunner::JobExceptionPolicy; - using CoverageExceptionPolicy = TestImpact::InstrumentedTestRunner::CoverageExceptionPolicy; - - struct TargetPaths - { - AZ::IO::Path m_targetBinary; - AZ::IO::Path m_testRunArtifact; - AZ::IO::Path m_testCoverageArtifact; - }; - - // Indices for looking up job command arguments for the different coverage levels - enum CoverageLevel : uint8_t - { - LineLevel = 0, - SourceLevel - }; - - // Get the job command for an instrumented test run - AZStd::string GetRunCommandForTarget(const TargetPaths& testTarget, CoverageLevel coverageLevel, const char* sourcesFilter) - { - AZStd::string args = AZStd::string::format( - "%s " // 1. Instrumented test runner - "--coverage_level %s " // 2. Coverage level - "--export_type cobertura:\"%s\" " // 3. Test coverage artifact path - "--modules \"%s\" " // 4. Modules path - "--excluded_modules \"%s\" " // 5. Exclude modules - "--sources \"%s\" -- " // 6. Sources path - "\"%s\" " // 7. Test runner binary - "\"%s\" " // 8. Test target bin - "AzRunUnitTests " - "--gtest_output=xml:%s", // 9. Test run result artifact - - LY_TEST_IMPACT_INSTRUMENTATION_BIN, // 1. - (coverageLevel == CoverageLevel::LineLevel ? "line" : "source"), // 2. - testTarget.m_testCoverageArtifact.c_str(), // 3. - LY_TEST_IMPACT_MODULES_DIR, // 4. - LY_TEST_IMPACT_AZ_TESTRUNNER_BIN, // 5. - sourcesFilter, // 6. - LY_TEST_IMPACT_AZ_TESTRUNNER_BIN, // 7. - testTarget.m_targetBinary.c_str(), // 8. - testTarget.m_testRunArtifact.c_str() // 9. - ); - - // OpenCppCoverage doesn't support forward slash directory separators so replace all with escaped backslashes - return AZStd::regex_replace(args, AZStd::regex("/"), "\\"); - } - - // Get the job command for an instrumented test run with valid source filters to produce coverage artifact - AZStd::string GetRunCommandForTargetWithSources(const TargetPaths& testTarget, CoverageLevel coverageLevel) - { - return GetRunCommandForTarget(testTarget, coverageLevel, LY_TEST_IMPACT_COVERAGE_SOURCES_DIR); - } - - // Get the job command for an instrumented test run without valid source filters to produce empty coverage artifact - AZStd::string GetRunCommandForTargetWithoutSources(const TargetPaths& testTarget, CoverageLevel coverageLevel) - { - return GetRunCommandForTarget(testTarget, coverageLevel, "C:\\No\\Sources\\Here\\At\\All\\Ever\\Ever\\Ever"); - } - - class InstrumentedTestRunnerFixture - : public AllocatorsTestFixture - { - public: - void SetUp() override; - void TearDown() override; - - protected: - using JobInfo = TestImpact::InstrumentedTestRunner::JobInfo; - using JobData = TestImpact::InstrumentedTestRunner::JobData; - - AZStd::vector m_jobInfos; - AZStd::unique_ptr m_testRunner; - AZStd::vector> m_testTargetJobArgs; - AZStd::vector m_testTargetPaths; - AZStd::vector m_expectedTestTargetRuns; - AZStd::vector> m_expectedTestTargetCoverages; - AZStd::vector m_expectedTestTargetResult; - size_t m_maxConcurrency = 0; - CoverageLevel m_coverageLevel = CoverageLevel::LineLevel; - inline static AZ::u32 s_uniqueTestCaseId = 0; // Unique id for each test case to be used as the suffix for the written files - }; - - void InstrumentedTestRunnerFixture::SetUp() - { - AllocatorsTestFixture::SetUp(); - - // Suffix each artifact file with the unique test case id to ensure that there are no possible race conditions between cleaning - // up the artifact files after each test case and those artifact files being written and read again in future test cases - const AZStd::string fileExtension = AZStd::string::format(".%u.xml", s_uniqueTestCaseId++); - - const AZStd::string runPath = AZStd::string(LY_TEST_IMPACT_TEST_TARGET_RESULTS_DIR) + "/%s.Run" + fileExtension; - const AZStd::string coveragePath = AZStd::string(LY_TEST_IMPACT_TEST_TARGET_COVERAGE_DIR) + "/%s.Coverage" + fileExtension; - - // TestTargetA - m_testTargetPaths.emplace_back(TargetPaths{ - LY_TEST_IMPACT_TEST_TARGET_A_BIN, AZStd::string::format(runPath.c_str(), LY_TEST_IMPACT_TEST_TARGET_A_BASE_NAME), - AZStd::string::format(coveragePath.c_str(), LY_TEST_IMPACT_TEST_TARGET_A_BASE_NAME)}); - - // TestTargetB - m_testTargetPaths.emplace_back(TargetPaths{ - LY_TEST_IMPACT_TEST_TARGET_B_BIN, AZStd::string::format(runPath.c_str(), LY_TEST_IMPACT_TEST_TARGET_B_BASE_NAME), - AZStd::string::format(coveragePath.c_str(), LY_TEST_IMPACT_TEST_TARGET_B_BASE_NAME)}); - - // TestTargetC - m_testTargetPaths.emplace_back(TargetPaths{ - LY_TEST_IMPACT_TEST_TARGET_C_BIN, AZStd::string::format(runPath.c_str(), LY_TEST_IMPACT_TEST_TARGET_C_BASE_NAME), - AZStd::string::format(coveragePath.c_str(), LY_TEST_IMPACT_TEST_TARGET_C_BASE_NAME)}); - - // TestTargetD - m_testTargetPaths.emplace_back(TargetPaths{ - LY_TEST_IMPACT_TEST_TARGET_D_BIN, AZStd::string::format(runPath.c_str(), LY_TEST_IMPACT_TEST_TARGET_D_BASE_NAME), - AZStd::string::format(coveragePath.c_str(), LY_TEST_IMPACT_TEST_TARGET_D_BASE_NAME)}); - - m_expectedTestTargetRuns.emplace_back(GetTestTargetATestRunSuites(), AZStd::chrono::milliseconds{500}); // TestTargetA - m_expectedTestTargetRuns.emplace_back(GetTestTargetBTestRunSuites(), AZStd::chrono::milliseconds{500}); // TestTargetB - m_expectedTestTargetRuns.emplace_back(GetTestTargetCTestRunSuites(), AZStd::chrono::milliseconds{500}); // TestTargetC - m_expectedTestTargetRuns.emplace_back(GetTestTargetDTestRunSuites(), AZStd::chrono::milliseconds{500}); // TestTargetD - - // TestTargetA - m_expectedTestTargetCoverages.emplace_back(AZStd::array{ - TestImpact::TestCoverage(GetTestTargetALineModuleCoverages()), - TestImpact::TestCoverage(GetTestTargetASourceModuleCoverages())}); - - // TestTargetB - m_expectedTestTargetCoverages.emplace_back(AZStd::array{ - TestImpact::TestCoverage(GetTestTargetBLineModuleCoverages()), - TestImpact::TestCoverage(GetTestTargetBSourceModuleCoverages())}); - - // TestTargetC - m_expectedTestTargetCoverages.emplace_back(AZStd::array{ - TestImpact::TestCoverage(GetTestTargetCLineModuleCoverages()), - TestImpact::TestCoverage(GetTestTargetCSourceModuleCoverages())}); - - // TestTargetD - m_expectedTestTargetCoverages.emplace_back(AZStd::array{ - TestImpact::TestCoverage(GetTestTargetDLineModuleCoverages()), - TestImpact::TestCoverage(GetTestTargetDSourceModuleCoverages())}); - - m_expectedTestTargetResult.emplace_back(TestImpact::TestRunResult::Failed); // TestTargetA - m_expectedTestTargetResult.emplace_back(TestImpact::TestRunResult::Passed); // TestTargetB - m_expectedTestTargetResult.emplace_back(TestImpact::TestRunResult::Passed); // TestTargetC - m_expectedTestTargetResult.emplace_back(TestImpact::TestRunResult::Passed); // TestTargetD - - // Generate the job command arguments for both line level and source level coverage permutations - for (const auto& testTarget : m_testTargetPaths) - { - m_testTargetJobArgs.emplace_back(AZStd::array{ - GetRunCommandForTargetWithSources(testTarget, CoverageLevel::LineLevel), - GetRunCommandForTargetWithSources(testTarget, CoverageLevel::SourceLevel)}); - } - } - - void InstrumentedTestRunnerFixture::TearDown() - { - DeleteFiles(LY_TEST_IMPACT_TEST_TARGET_COVERAGE_DIR, "*.xml"); - DeleteFiles(LY_TEST_IMPACT_TEST_TARGET_RESULTS_DIR, "*.xml"); - - AllocatorsTestFixture::TearDown(); - } - - using ConcurrencyAndCoveragePermutation = AZStd::tuple - < - size_t, // Max number of concurrent processes - CoverageLevel // Coverage level - >; - - // Fixture parameterized for different max number of concurrent jobs and coverage levels - class InstrumentedTestRunnerFixtureWithConcurrencyAndCoverageParams - : public InstrumentedTestRunnerFixture - , public ::testing::WithParamInterface - { - public: - void SetUp() override - { - InstrumentedTestRunnerFixture::SetUp(); - auto [maxConcurrency, coverageLevel] = GetParam(); - m_maxConcurrency = maxConcurrency; - m_coverageLevel = coverageLevel; - } - }; - - using ConcurrencyAndJobExceptionPermutation = AZStd::tuple - < - size_t, // Max number of concurrent processes - CoverageLevel, // Coverage level - JobExceptionPolicy // Test job exception policy - >; - - // Fixture parameterized for different max number of concurrent jobs, coverage levels and different job exception policies - class InstrumentedTestRunnerFixtureWithConcurrencyAndCoverageAndJobExceptionParams - : public InstrumentedTestRunnerFixture - , public ::testing::WithParamInterface - { - public: - void SetUp() override - { - InstrumentedTestRunnerFixture::SetUp(); - const auto& [maxConcurrency, coverageLevel, jobExceptionPolicy] = GetParam(); - m_maxConcurrency = maxConcurrency; - m_coverageLevel = coverageLevel; - m_jobExceptionPolicy = jobExceptionPolicy; - } - - protected: - JobExceptionPolicy m_jobExceptionPolicy = JobExceptionPolicy::Never; - }; - - class InstrumentedTestRunnerFixtureWithConcurrencyAndCoverageAndFailedToLaunchExceptionParams - : public InstrumentedTestRunnerFixtureWithConcurrencyAndCoverageAndJobExceptionParams - { - }; - - class InstrumentedTestRunnerFixtureWithConcurrencyAndCoverageAndExecutedWithFailureExceptionParams - : public InstrumentedTestRunnerFixtureWithConcurrencyAndCoverageAndJobExceptionParams - { - }; - - using ConcurrencyAndCoverageExceptionPermutation = AZStd::tuple - < - size_t, // Max number of concurrent processes - CoverageExceptionPolicy // Test coverage exception policy - >; - - // Fixture parameterized for different max number of concurrent jobs and different coverage exception policies - class InstrumentedTestRunnerFixtureWithConcurrencyAndCoverageAndCoverageExceptionParams - : public InstrumentedTestRunnerFixture - , public ::testing::WithParamInterface - { - public: - void SetUp() override - { - InstrumentedTestRunnerFixture::SetUp(); - const auto& [maxConcurrency, coverageExceptionPolicy] = GetParam(); - m_maxConcurrency = maxConcurrency; - m_coverageExceptionPolicy = coverageExceptionPolicy; - } - - protected: - CoverageExceptionPolicy m_coverageExceptionPolicy = CoverageExceptionPolicy::Never; - }; - - namespace - { - AZStd::array MaxConcurrentRuns = {1, 2, 3, 4}; - - AZStd::array CoverageLevels = {CoverageLevel::LineLevel, CoverageLevel::SourceLevel}; - - AZStd::array FailedToLaunchExceptionPolicies = { - JobExceptionPolicy::Never, JobExceptionPolicy::OnFailedToExecute}; - - AZStd::array ExecutedWithFailureExceptionPolicies = { - JobExceptionPolicy::Never, JobExceptionPolicy::OnExecutedWithFailure}; - } // namespace - - // Validates that the specified test coverage matches the expected output - void ValidateTestTargetCoverage(const TestImpact::TestCoverage& actualResult, const TestImpact::TestCoverage& expectedResult) - { - EXPECT_TRUE(actualResult == expectedResult); - } - - // Validates that the specified test coverage is empty - void ValidateEmptyTestTargetCoverage(const TestImpact::TestCoverage& actualResult) - { - EXPECT_TRUE(actualResult.GetSourcesCovered().empty()); - EXPECT_TRUE(actualResult.GetModuleCoverages().empty()); - EXPECT_EQ(actualResult.GetNumSourcesCovered(), 0); - EXPECT_EQ(actualResult.GetNumModulesCovered(), 0); - } - - TEST_P( - InstrumentedTestRunnerFixtureWithConcurrencyAndCoverageAndCoverageExceptionParams, - EmptyTestCoverages_ExpectEmptyTestCoveragesOrTestRunException) - { - // Given a test runner with no client callback or run timeout or runner timeout - m_testRunner = - AZStd::make_unique(AZStd::nullopt, m_maxConcurrency, AZStd::nullopt, AZStd::nullopt); - - // Given a mixture of instrumented test run jobs with and without coverage sources - for (size_t jobId = 0; jobId < m_testTargetJobArgs.size(); jobId++) - { - const AZStd::string args = (jobId % 2) ? GetRunCommandForTargetWithoutSources(m_testTargetPaths[jobId], m_coverageLevel) - : m_testTargetJobArgs[jobId][m_coverageLevel]; - - JobData jobData(m_testTargetPaths[jobId].m_testRunArtifact, m_testTargetPaths[jobId].m_testCoverageArtifact); - m_jobInfos.emplace_back(JobInfo({jobId}, args, AZStd::move(jobData))); - } - - try - { - // When the instrumented test run jobs are executed with different exception policies - const auto runnerJobs = m_testRunner->RunInstrumentedTests(m_jobInfos, m_coverageExceptionPolicy, JobExceptionPolicy::Never); - - // Expect this statement to be reachable only if no exception policy for empty coverages - EXPECT_FALSE(::IsFlagSet(m_coverageExceptionPolicy, CoverageExceptionPolicy::OnEmptyCoverage)); - - for (const auto& job : runnerJobs) - { - const JobInfo::IdType jobId = job.GetJobInfo().GetId().m_value; - ValidateTestRunCompleted(job, m_expectedTestTargetResult[jobId]); - ValidateTestTargetRun(job.GetPayload().value().first, m_expectedTestTargetRuns[jobId]); - - if (jobId % 2) - { - // Expect jobs have a empty test coverages - ValidateEmptyTestTargetCoverage(job.GetPayload().value().second); - } - else - { - // Expect the jobs to successfully result in a test run and coverage that matches the expected test run data - ValidateTestTargetCoverage(job.GetPayload().value().second, m_expectedTestTargetCoverages[jobId][m_coverageLevel]); - } - } - } - catch ([[maybe_unused]] const TestImpact::TestRunException& e) - { - // Expect this statement to be reachable only if there is an exception policy for empty coverages - EXPECT_TRUE(::IsFlagSet(m_coverageExceptionPolicy, CoverageExceptionPolicy::OnEmptyCoverage)); - } - catch (...) - { - // Do not expect any other exceptions - FAIL(); - } - } - - TEST_P( - InstrumentedTestRunnerFixtureWithConcurrencyAndCoverageAndFailedToLaunchExceptionParams, - InvalidCommandArgument_ExpectJobResulFailedToExecuteeOrTestJobException) - { - // Given a test runner with no client callback or run timeout or runner timeout - m_testRunner = - AZStd::make_unique(AZStd::nullopt, m_maxConcurrency, AZStd::nullopt, AZStd::nullopt); - - // Given a mixture of instrumented test run jobs with valid and invalid command arguments - for (size_t jobId = 0; jobId < m_testTargetJobArgs.size(); jobId++) - { - const AZStd::string args = (jobId % 2) ? InvalidProcessPath : m_testTargetJobArgs[jobId][m_coverageLevel]; - JobData jobData(m_testTargetPaths[jobId].m_testRunArtifact, m_testTargetPaths[jobId].m_testCoverageArtifact); - m_jobInfos.emplace_back(JobInfo({jobId}, args, AZStd::move(jobData))); - } - - try - { - // When the instrumented test run jobs are executed with different exception policies - const auto runnerJobs = m_testRunner->RunInstrumentedTests(m_jobInfos, CoverageExceptionPolicy::Never, m_jobExceptionPolicy); - - // Expect this statement to be reachable only if no exception policy for launch failures - EXPECT_FALSE(::IsFlagSet(m_jobExceptionPolicy, JobExceptionPolicy::OnFailedToExecute)); - - for (const auto& job : runnerJobs) - { - const JobInfo::IdType jobId = job.GetJobInfo().GetId().m_value; - if (jobId % 2) - { - // Expect invalid jobs have a job result of FailedToExecute - ValidateJobFailedToExecute(job); - } - else - { - // Expect the valid jobs to successfully result in a test run that matches the expected test run data - ValidateTestRunCompleted(job, m_expectedTestTargetResult[jobId]); - ValidateTestTargetRun(job.GetPayload().value().first, m_expectedTestTargetRuns[jobId]); - ValidateTestTargetCoverage(job.GetPayload().value().second, m_expectedTestTargetCoverages[jobId][m_coverageLevel]); - } - } - } - catch ([[maybe_unused]] const TestImpact::TestJobException& e) - { - // Expect this statement to be reachable only if there is an exception policy for launch failures - EXPECT_TRUE(::IsFlagSet(m_jobExceptionPolicy, JobExceptionPolicy::OnFailedToExecute)); - } - catch (...) - { - // Do not expect any other exceptions - FAIL(); - } - } - - TEST_P( - InstrumentedTestRunnerFixtureWithConcurrencyAndCoverageAndExecutedWithFailureExceptionParams, - ErroneousReturnCode_ExpectJobResultExecutedWithFailureOrTestJobException) - { - // Given a test runner with no client callback or run timeout or runner timeout - m_testRunner = - AZStd::make_unique(AZStd::nullopt, m_maxConcurrency, AZStd::nullopt, AZStd::nullopt); - - // Given a mixture of instrumented test run jobs that execute and return either successfully or with failure - for (size_t jobId = 0; jobId < m_testTargetJobArgs.size(); jobId++) - { - JobData jobData(m_testTargetPaths[jobId].m_testRunArtifact, m_testTargetPaths[jobId].m_testCoverageArtifact); - m_jobInfos.emplace_back(JobInfo({jobId}, m_testTargetJobArgs[jobId][m_coverageLevel], AZStd::move(jobData))); - } - - try - { - // When the instrumented test run jobs are executed with different exception policies - const auto runnerJobs = m_testRunner->RunInstrumentedTests(m_jobInfos, CoverageExceptionPolicy::Never, m_jobExceptionPolicy); - - // Expect this statement to be reachable only if no exception policy for jobs that return with error - EXPECT_FALSE(::IsFlagSet(m_jobExceptionPolicy, JobExceptionPolicy::OnExecutedWithFailure)); - - for (const auto& job : runnerJobs) - { - const JobInfo::IdType jobId = job.GetJobInfo().GetId().m_value; - - // Expect the valid jobs to successfully result in a test run that matches the expected test run data - ValidateTestRunCompleted(job, m_expectedTestTargetResult[jobId]); - ValidateTestTargetRun(job.GetPayload().value().first, m_expectedTestTargetRuns[jobId]); - ValidateTestTargetCoverage(job.GetPayload().value().second, m_expectedTestTargetCoverages[jobId][m_coverageLevel]); - } - } - catch ([[maybe_unused]] const TestImpact::TestJobException& e) - { - // Expect this statement to be reachable only if there is an exception policy for jobs that return with error - EXPECT_TRUE(::IsFlagSet(m_jobExceptionPolicy, JobExceptionPolicy::OnExecutedWithFailure)); - } - catch ([[maybe_unused]] const TestImpact::Exception& e) - { - FAIL(); - } - catch (...) - { - // Do not expect any other exceptions - FAIL(); - } - } - - TEST_F(InstrumentedTestRunnerFixture, EmptyRunRawData_ExpectTestRunnerException) - { - // Given a test runner with no client callback, concurrency, run timeout or runner timeout - m_testRunner = - AZStd::make_unique(AZStd::nullopt, OneConcurrentProcess, AZStd::nullopt, AZStd::nullopt); - - // Given an test runner job that will return successfully but with an empty artifact string - JobData jobData("", m_testTargetPaths[TestTargetA].m_testCoverageArtifact); - m_jobInfos.emplace_back(JobInfo({TestTargetA}, m_testTargetJobArgs[TestTargetA][LineLevel], AZStd::move(jobData))); - - try - { - // When the test runner job is executed - const auto runnerJobs = - m_testRunner->RunInstrumentedTests(m_jobInfos, CoverageExceptionPolicy::Never, JobExceptionPolicy::Never); - - // Do not expect this statement to be reachable - FAIL(); - } - catch ([[maybe_unused]] const TestImpact::TestRunException& e) - { - // Expect an runner exception - SUCCEED(); - } - catch (...) - { - // Do not expect any other exceptions - FAIL(); - } - } - - TEST_F(InstrumentedTestRunnerFixture, EmptyCoverageRawData_ExpectTestRunnerException) - { - // Given a test runner with no client callback, concurrency, run timeout or runner timeout - m_testRunner = - AZStd::make_unique(AZStd::nullopt, OneConcurrentProcess, AZStd::nullopt, AZStd::nullopt); - - // Given an test runner job that will return successfully but with an empty artifact string - JobData jobData(m_testTargetPaths[TestTargetA].m_testRunArtifact, ""); - m_jobInfos.emplace_back(JobInfo({TestTargetA}, m_testTargetJobArgs[TestTargetA][LineLevel], AZStd::move(jobData))); - - try - { - // When the test runner job is executed - const auto runnerJobs = - m_testRunner->RunInstrumentedTests(m_jobInfos, CoverageExceptionPolicy::Never, JobExceptionPolicy::Never); - - // Do not expect this statement to be reachable - FAIL(); - } - catch ([[maybe_unused]] const TestImpact::TestRunException& e) - { - // Expect an runner exception - SUCCEED(); - } - catch (...) - { - // Do not expect any other exceptions - FAIL(); - } - } - - TEST_F(InstrumentedTestRunnerFixture, InvalidRunArtifact_ExpectArtifactException) - { - // Given a run artifact with invalid contents - WriteTextToFile("There is nothing valid here", m_testTargetPaths[TestTargetA].m_testRunArtifact); - - // Given a job command that will write the run artifact to a different location that what we will read from - TargetPaths invalidRunArtifact = m_testTargetPaths[TestTargetA]; - invalidRunArtifact.m_testRunArtifact /= ".xml"; - const AZStd::string args = GetRunCommandForTargetWithSources(invalidRunArtifact, LineLevel); - - // Given a test runner with no client callback, concurrency, run timeout or runner timeout - m_testRunner = - AZStd::make_unique(AZStd::nullopt, OneConcurrentProcess, AZStd::nullopt, AZStd::nullopt); - - // Given an test runner job that will return successfully but not produce a run artifact - JobData jobData(m_testTargetPaths[TestTargetA].m_testRunArtifact, m_testTargetPaths[TestTargetA].m_testCoverageArtifact); - m_jobInfos.emplace_back(JobInfo({TestTargetA}, args, AZStd::move(jobData))); - - try - { - // When the test runner job is executed - const auto runnerJobs = - m_testRunner->RunInstrumentedTests(m_jobInfos, CoverageExceptionPolicy::Never, JobExceptionPolicy::Never); - - // Do not expect this statement to be reachable - FAIL(); - } - catch ([[maybe_unused]] const TestImpact::ArtifactException& e) - { - // Expect an runner exception - SUCCEED(); - } - catch (...) - { - // Do not expect any other exceptions - FAIL(); - } - } - - TEST_F(InstrumentedTestRunnerFixture, InvalidCoverageArtifact_ExpectArtifactException) - { - // Given a coverage artifact with invalid contents - WriteTextToFile("There is nothing valid here", m_testTargetPaths[TestTargetA].m_testCoverageArtifact); - - // Given a job command that will write the coverage artifact to a different location that what we will read from - TargetPaths invalidCoverageArtifact = m_testTargetPaths[TestTargetA]; - invalidCoverageArtifact.m_testCoverageArtifact /= ".xml"; - const AZStd::string args = GetRunCommandForTargetWithSources(invalidCoverageArtifact, LineLevel); - - // Given a test runner with no client callback, concurrency, run timeout or runner timeout - m_testRunner = - AZStd::make_unique(AZStd::nullopt, OneConcurrentProcess, AZStd::nullopt, AZStd::nullopt); - - // Given an test runner job that will return successfully but not produce a run artifact - JobData jobData(m_testTargetPaths[TestTargetA].m_testRunArtifact, m_testTargetPaths[TestTargetA].m_testCoverageArtifact); - m_jobInfos.emplace_back(JobInfo({TestTargetA}, args, AZStd::move(jobData))); - - try - { - // When the test runner job is executed - const auto runnerJobs = - m_testRunner->RunInstrumentedTests(m_jobInfos, CoverageExceptionPolicy::Never, JobExceptionPolicy::Never); - - // Do not expect this statement to be reachable - FAIL(); - } - catch ([[maybe_unused]] const TestImpact::ArtifactException& e) - { - // Expect an runner exception - SUCCEED(); - } - catch (...) - { - // Do not expect any other exceptions - FAIL(); - } - } - - TEST_P(InstrumentedTestRunnerFixtureWithConcurrencyAndCoverageParams, RunTestTargets_RunsAndCoverageMatchTestSuitesInTarget) - { - // Given a test runner with no client callback, runner timeout or runner timeout - m_testRunner = - AZStd::make_unique(AZStd::nullopt, m_maxConcurrency, AZStd::nullopt, AZStd::nullopt); - - // Given an test runner job for each test target with no runner caching - for (size_t jobId = 0; jobId < m_testTargetJobArgs.size(); jobId++) - { - JobData jobData(m_testTargetPaths[jobId].m_testRunArtifact, m_testTargetPaths[jobId].m_testCoverageArtifact); - m_jobInfos.emplace_back(JobInfo({jobId}, m_testTargetJobArgs[jobId][m_coverageLevel], AZStd::move(jobData))); - } - - // When the test runner jobs are executed - const auto runnerJobs = m_testRunner->RunInstrumentedTests(m_jobInfos, CoverageExceptionPolicy::Never, JobExceptionPolicy::Never); - - // Expect each job to successfully result in a test runner that matches the expected test runner data for that test target - for (const auto& job : runnerJobs) - { - const JobInfo::IdType jobId = job.GetJobInfo().GetId().m_value; - ValidateTestRunCompleted(job, m_expectedTestTargetResult[jobId]); - ValidateTestTargetRun(job.GetPayload().value().first, m_expectedTestTargetRuns[jobId]); - ValidateTestTargetCoverage(job.GetPayload().value().second, m_expectedTestTargetCoverages[jobId][m_coverageLevel]); - } - } - - TEST_P( - InstrumentedTestRunnerFixtureWithConcurrencyAndCoverageParams, - RunTestTargetsWithArbitraryJobIds_RunsAndCoverageMatchTestSuitesInTarget) - { - // Given a set of arbitrary job ids to be used for the test target jobs - enum - { - ArbitraryA = 36, - ArbitraryB = 890, - ArbitraryC = 19, - ArbitraryD = 1 - }; - - const AZStd::unordered_map sequentialToArbitrary = - { - {TestTargetA, ArbitraryA}, - {TestTargetB, ArbitraryB}, - {TestTargetC, ArbitraryC}, - {TestTargetD, ArbitraryD}, - }; - - const AZStd::unordered_map arbitraryToSequential = - { - {ArbitraryA, TestTargetA}, - {ArbitraryB, TestTargetB}, - {ArbitraryC, TestTargetC}, - {ArbitraryD, TestTargetD}, - }; - - // Given a test runner with no client callback, run timeout or runner timeout - m_testRunner = - AZStd::make_unique(AZStd::nullopt, m_maxConcurrency, AZStd::nullopt, AZStd::nullopt); - - // Given an test run job for each test target - for (size_t jobId = 0; jobId < m_testTargetJobArgs.size(); jobId++) - { - JobData jobData(m_testTargetPaths[jobId].m_testRunArtifact, m_testTargetPaths[jobId].m_testCoverageArtifact); - m_jobInfos.emplace_back( - JobInfo({sequentialToArbitrary.at(jobId)}, m_testTargetJobArgs[jobId][m_coverageLevel], AZStd::move(jobData))); - } - - // When the instrumented test run jobs are executed - const auto runnerJobs = m_testRunner->RunInstrumentedTests(m_jobInfos, CoverageExceptionPolicy::Never, JobExceptionPolicy::Never); - - // Expect each job to successfully result in a test run that matches the expected test run data for that test target - for (const auto& job : runnerJobs) - { - const JobInfo::IdType jobId = arbitraryToSequential.at(job.GetJobInfo().GetId().m_value); - ValidateTestRunCompleted(job, m_expectedTestTargetResult[jobId]); - ValidateTestTargetRun(job.GetPayload().value().first, m_expectedTestTargetRuns[jobId]); - ValidateTestTargetCoverage(job.GetPayload().value().second, m_expectedTestTargetCoverages[jobId][m_coverageLevel]); - } - } - - TEST_P(InstrumentedTestRunnerFixtureWithConcurrencyAndCoverageParams, RunTestTargetsWithCallback_RunsAndCoverageMatchTestSuitesInTarget) - { - // Given a client callback function that tracks the number of successful runs - size_t numSuccesses = 0; - const auto jobCallback = - [&numSuccesses]([[maybe_unused]] const TestImpact::InstrumentedTestRunner::JobInfo& jobInfo, const TestImpact::JobMeta& meta) - { - if (meta.m_result == TestImpact::JobResult::ExecutedWithSuccess) - { - numSuccesses++; - } - }; - - // Given a test runner with no run timeout or runner timeout - m_testRunner = - AZStd::make_unique(jobCallback, m_maxConcurrency, AZStd::nullopt, AZStd::nullopt); - - // Given an test run job for each test target - for (size_t jobId = 0; jobId < m_testTargetJobArgs.size(); jobId++) - { - JobData jobData(m_testTargetPaths[jobId].m_testRunArtifact, m_testTargetPaths[jobId].m_testCoverageArtifact); - m_jobInfos.emplace_back(JobInfo({jobId}, m_testTargetJobArgs[jobId][m_coverageLevel], AZStd::move(jobData))); - } - - // When the instrumented test run jobs are executed - const auto runnerJobs = m_testRunner->RunInstrumentedTests(m_jobInfos, CoverageExceptionPolicy::Never, JobExceptionPolicy::Never); - - // Expect the number of successful runs tracked in the callback to match the number of test targets run with no failures - EXPECT_EQ( - numSuccesses, - AZStd::count_if(m_expectedTestTargetResult.begin(), m_expectedTestTargetResult.end(), [](TestImpact::TestRunResult result) { - return result == TestImpact::TestRunResult::Passed; - })); - - // Expect each job to successfully result in a test run that matches the expected test run data for that test target - for (const auto& job : runnerJobs) - { - const JobInfo::IdType jobId = job.GetJobInfo().GetId().m_value; - ValidateTestRunCompleted(job, m_expectedTestTargetResult[jobId]); - ValidateTestTargetRun(job.GetPayload().value().first, m_expectedTestTargetRuns[jobId]); - ValidateTestTargetCoverage(job.GetPayload().value().second, m_expectedTestTargetCoverages[jobId][m_coverageLevel]); - } - } - - TEST_P(InstrumentedTestRunnerFixtureWithConcurrencyAndCoverageParams, JobRunnerTimeout_InFlightJobsTimeoutAndQueuedJobsUnlaunched) - { - // Given a test runner with no client callback or runner timeout and 2 second run timeout - m_testRunner = AZStd::make_unique( - AZStd::nullopt, m_maxConcurrency, AZStd::chrono::seconds(2), AZStd::nullopt); - - // Given an test run job for each test target where half will sleep indefinitely - for (size_t jobId = 0; jobId < m_testTargetJobArgs.size(); jobId++) - { - JobData jobData(m_testTargetPaths[jobId].m_testRunArtifact, m_testTargetPaths[jobId].m_testCoverageArtifact); - const AZStd::string args = (jobId % 2) - ? AZStd::string::format("%s %s", ValidProcessPath, ConstructTestProcessArgs(jobId, LongSleep).c_str()) - : m_testTargetJobArgs[jobId][m_coverageLevel]; - m_jobInfos.emplace_back(JobInfo({jobId}, args, AZStd::move(jobData))); - } - - // When the instrumented test run jobs are executed - const auto runnerJobs = m_testRunner->RunInstrumentedTests(m_jobInfos, CoverageExceptionPolicy::Never, JobExceptionPolicy::Never); - - // Expect half the jobs to successfully result in a test run that matches the expected test run data for that test target - // with the other half having timed out - for (const auto& job : runnerJobs) - { - const JobInfo::IdType jobId = job.GetJobInfo().GetId().m_value; - if (jobId % 2) - { - ValidateJobTimeout(job); - } - else - { - ValidateTestRunCompleted(job, m_expectedTestTargetResult[jobId]); - ValidateTestTargetRun(job.GetPayload().value().first, m_expectedTestTargetRuns[jobId]); - ValidateTestTargetCoverage(job.GetPayload().value().second, m_expectedTestTargetCoverages[jobId][m_coverageLevel]); - } - } - } - - TEST_F(InstrumentedTestRunnerFixture, JobTimeout_InFlightJobTimeoutAndQueuedJobsUnlaunched) - { - // Given a test runner with no client callback or run timeout and a 5 second runner timeout - m_testRunner = AZStd::make_unique( - AZStd::nullopt, FourConcurrentProcesses, AZStd::nullopt, AZStd::chrono::seconds(5)); - - // Given an test run job for each test target where half will sleep indefinitely - for (size_t jobId = 0; jobId < m_testTargetJobArgs.size(); jobId++) - { - JobData jobData(m_testTargetPaths[jobId].m_testRunArtifact, m_testTargetPaths[jobId].m_testCoverageArtifact); - const AZStd::string args = (jobId % 2) - ? AZStd::string::format("%s %s", ValidProcessPath, ConstructTestProcessArgs(jobId, LongSleep).c_str()) - : m_testTargetJobArgs[jobId][m_coverageLevel]; - m_jobInfos.emplace_back(JobInfo({jobId}, args, AZStd::move(jobData))); - } - - // When the instrumented test run jobs are executed - const auto runnerJobs = m_testRunner->RunInstrumentedTests(m_jobInfos, CoverageExceptionPolicy::Never, JobExceptionPolicy::Never); - - // Expect half the jobs to successfully result in a test run that matches the expected test run data for that test target - // with the other half having timed out - for (const auto& job : runnerJobs) - { - const JobInfo::IdType jobId = job.GetJobInfo().GetId().m_value; - if (jobId % 2) - { - ValidateJobTimeout(job); - } - else - { - ValidateTestRunCompleted(job, m_expectedTestTargetResult[jobId]); - ValidateTestTargetRun(job.GetPayload().value().first, m_expectedTestTargetRuns[jobId]); - ValidateTestTargetCoverage(job.GetPayload().value().second, m_expectedTestTargetCoverages[jobId][m_coverageLevel]); - } - } - } - - INSTANTIATE_TEST_CASE_P( - , - InstrumentedTestRunnerFixtureWithConcurrencyAndCoverageAndCoverageExceptionParams, - ::testing::Combine( - ::testing::ValuesIn(MaxConcurrentRuns), - ::testing::Values(CoverageExceptionPolicy::Never, CoverageExceptionPolicy::OnEmptyCoverage))); - - INSTANTIATE_TEST_CASE_P( - , - InstrumentedTestRunnerFixtureWithConcurrencyAndCoverageAndFailedToLaunchExceptionParams, - ::testing::Combine( - ::testing::ValuesIn(MaxConcurrentRuns), ::testing::ValuesIn(CoverageLevels), - ::testing::ValuesIn(FailedToLaunchExceptionPolicies))); - - INSTANTIATE_TEST_CASE_P( - , - InstrumentedTestRunnerFixtureWithConcurrencyAndCoverageAndExecutedWithFailureExceptionParams, - ::testing::Combine( - ::testing::ValuesIn(MaxConcurrentRuns), ::testing::ValuesIn(CoverageLevels), - ::testing::ValuesIn(ExecutedWithFailureExceptionPolicies))); - - INSTANTIATE_TEST_CASE_P( - , - InstrumentedTestRunnerFixtureWithConcurrencyAndCoverageParams, - ::testing::Combine(::testing::ValuesIn(MaxConcurrentRuns), ::testing::ValuesIn(CoverageLevels))); -} // namespace UnitTest diff --git a/Code/Tools/TestImpactFramework/Runtime/Code/Tests/Test/TestImpactTestCoverageTest.cpp b/Code/Tools/TestImpactFramework/Runtime/Code/Tests/Test/TestImpactTestCoverageTest.cpp deleted file mode 100644 index c1d7ba3e4b..0000000000 --- a/Code/Tools/TestImpactFramework/Runtime/Code/Tests/Test/TestImpactTestCoverageTest.cpp +++ /dev/null @@ -1,208 +0,0 @@ -/* - * All or portions of this file Copyright (c) Amazon.com, Inc. or its affiliates or - * its licensors. - * - * For complete copyright and license terms please see the LICENSE at the root of this - * distribution (the "License"). All use of this software is governed by the License, - * or, if provided, by the license below or the license accompanying this file. Do not - * remove or modify any license notices. This file is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * - */ - -#include - -#include -#include - -namespace UnitTest -{ - namespace - { - AZStd::string GenerateSourcePath(AZ::u32 index) - { - return AZStd::string::format("SourceFile%u", index); - } - - AZStd::string GenerateModulePath(AZ::u32 index) - { - return AZStd::string::format("Module%u", index); - } - - AZStd::vector GenerateLineCoverages(AZ::u32 numLines) - { - AZStd::vector lineCoverages; - for (size_t i = 0; i < numLines; i++) - { - // Fudge some superficially different but trivially checkable line coverage data - lineCoverages.emplace_back(TestImpact::LineCoverage{i, i * 2}); - } - - return lineCoverages; - } - - TestImpact::SourceCoverage GenerateSourceCoverage(AZ::u32 index, TestImpact::CoverageLevel coverageLevel) - { - TestImpact::SourceCoverage sourceCoverage; - sourceCoverage.m_path = GenerateSourcePath(index); - if (coverageLevel == TestImpact::CoverageLevel::Line) - { - sourceCoverage.m_coverage = GenerateLineCoverages(index + 1); - } - - return sourceCoverage; - } - - AZStd::vector GenerateSourceCoverages(AZ::u32 numSources, TestImpact::CoverageLevel coverageLevel) - { - AZStd::vector sourceCoverages; - for (AZ::u32 i = 0; i < numSources; i++) - { - sourceCoverages.emplace_back(GenerateSourceCoverage(i, coverageLevel)); - } - - return sourceCoverages; - } - - TestImpact::ModuleCoverage GenerateModuleCoverage(AZ::u32 index, AZ::u32 numSources, TestImpact::CoverageLevel coverageLevel) - { - TestImpact::ModuleCoverage moduleCoverage; - moduleCoverage.m_path = GenerateModulePath(index); - moduleCoverage.m_sources = GenerateSourceCoverages(numSources, coverageLevel); - return moduleCoverage; - } - - AZStd::vector GenerateModuleCoverages(AZ::u32 numModules, TestImpact::CoverageLevel coverageLevel) - { - AZStd::vector moduleCoverages; - for (AZ::u32 i = 0; i < numModules; i++) - { - // Fudge some superficially different but trivially deducible module coverage data - moduleCoverages.emplace_back(GenerateModuleCoverage(i, i + 1, coverageLevel)); - } - - return moduleCoverages; - } - } // namespace - - using CoveragePermutation = AZStd::tuple - < - unsigned, // Number of modules covered - TestImpact::CoverageLevel // Test coverage level - >; - - // Fixture parameterized for different max number of concurrent jobs - class TestCoverageFixtureWithCoverageParams - : public AllocatorsTestFixture - , public ::testing::WithParamInterface - { - public: - void SetUp() override; - - protected: - void ValidateTestCoverage(const TestImpact::TestCoverage& testCoverage); - - size_t m_numModulesCovered; - TestImpact::CoverageLevel m_coverageLevel; - }; - - void TestCoverageFixtureWithCoverageParams::SetUp() - { - AllocatorsTestFixture::SetUp(); - const auto& [numModulesCovered, coverageLevel] = GetParam(); - m_numModulesCovered = numModulesCovered; - m_coverageLevel = coverageLevel; - } - - void TestCoverageFixtureWithCoverageParams::ValidateTestCoverage(const TestImpact::TestCoverage& testCoverage) - { - // Expect the coverage level to match that which was used to generate the module coverages generated - EXPECT_EQ(testCoverage.GetCoverageLevel(), m_coverageLevel); - - // Expect the number of modules covered to match the number of module coverages generated - EXPECT_EQ(testCoverage.GetNumModulesCovered(), m_numModulesCovered); - - // Expect the number of unique sources covered to match the number of modules coverages generated - EXPECT_EQ(testCoverage.GetNumSourcesCovered(), m_numModulesCovered); - - // Expect the unique sources covered to match the procedurally generated source paths - for (size_t sourceIndex = 0; sourceIndex < testCoverage.GetNumSourcesCovered(); sourceIndex++) - { - EXPECT_EQ(testCoverage.GetSourcesCovered()[sourceIndex], GenerateSourcePath(sourceIndex)); - } - - // Expect each module covered to match that of the corresponding procedurally generated modules - for (size_t moduleIndex = 0; moduleIndex < testCoverage.GetNumModulesCovered(); moduleIndex++) - { - const TestImpact::ModuleCoverage& moduleCoverage = testCoverage.GetModuleCoverages()[moduleIndex]; - - // Expect the module path to match that of the corresponding procedurally generated module - EXPECT_EQ(moduleCoverage.m_path, GenerateModulePath(moduleIndex)); - - // Expect the module's number of sources to match that of the corresponding procedurally generated module - EXPECT_EQ(moduleCoverage.m_sources.size(), moduleIndex + 1); - - for (size_t sourceIndex = 0; sourceIndex < moduleCoverage.m_sources.size(); sourceIndex++) - { - const TestImpact::SourceCoverage& sourceCoverage = moduleCoverage.m_sources[sourceIndex]; - - // Expect the source path to match the procedurally generated source path - EXPECT_EQ(sourceCoverage.m_path, GenerateSourcePath(sourceIndex)); - - if (m_coverageLevel == TestImpact::CoverageLevel::Line) - { - // Expect there to actually be line coverage data if this coverage was procedurally generated with line data - EXPECT_FALSE(sourceCoverage.m_coverage.empty()); - - const AZStd::vector& lineCoverages = sourceCoverage.m_coverage; - - // Expect the source's number of lines to match that of the corresponding procedurally generated source - EXPECT_EQ(lineCoverages.size(), sourceIndex + 1); - - for (size_t lineIndex = 0; lineIndex < lineCoverages.size(); lineIndex++) - { - // The expected line number and hit count are deduced as follows: - // Line number: line index - // Hit count: 2x line index - EXPECT_EQ(lineCoverages[lineIndex].m_lineNumber, lineIndex); - EXPECT_EQ(lineCoverages[lineIndex].m_hitCount, lineIndex * 2); - } - } - else - { - // Do not expect there to actually be line coverage data if this coverage was not procedurally generated with line data - EXPECT_TRUE(sourceCoverage.m_coverage.empty()); - } - } - } - } - - TEST(TestCoverage, EmptyCoverage_ExpectTestRunException) - { - // When constructing a test coverage from the empty module coverages - TestImpact::TestCoverage testCoverage(AZStd::vector{}); - - // Expect the test coverage fields to be empty - EXPECT_EQ(testCoverage.GetNumModulesCovered(), 0); - EXPECT_EQ(testCoverage.GetNumSourcesCovered(), 0); - EXPECT_TRUE(testCoverage.GetModuleCoverages().empty()); - EXPECT_TRUE(testCoverage.GetSourcesCovered().empty()); - } - - TEST_P(TestCoverageFixtureWithCoverageParams, AllCoveragePermutations_ExpectTestCoverageMetaDatasToMatchPermutations) - { - // Given a procedurally generated test coverage - const TestImpact::TestCoverage testCoverage(GenerateModuleCoverages(m_numModulesCovered, m_coverageLevel)); - - // Expect the test coverage data and meta-data to match that of the rules used to procedurally generate the coverage data - ValidateTestCoverage(testCoverage); - } - - INSTANTIATE_TEST_CASE_P( - , - TestCoverageFixtureWithCoverageParams, - ::testing::Combine( - ::testing::Range(1u, 11u), - ::testing::Values(TestImpact::CoverageLevel::Line, TestImpact::CoverageLevel::Source)) - ); -} // namespace UnitTest diff --git a/Code/Tools/TestImpactFramework/Runtime/Code/Tests/Test/TestImpactTestEnumeratorTest.cpp b/Code/Tools/TestImpactFramework/Runtime/Code/Tests/Test/TestImpactTestEnumeratorTest.cpp deleted file mode 100644 index 9b67eafa54..0000000000 --- a/Code/Tools/TestImpactFramework/Runtime/Code/Tests/Test/TestImpactTestEnumeratorTest.cpp +++ /dev/null @@ -1,890 +0,0 @@ -/* - * All or portions of this file Copyright (c) Amazon.com, Inc. or its affiliates or - * its licensors. - * - * For complete copyright and license terms please see the LICENSE at the root of this - * distribution (the "License"). All use of this software is governed by the License, - * or, if provided, by the license below or the license accompanying this file. Do not - * remove or modify any license notices. This file is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * - */ - -#include -#include - -#include -#include -#include - -#include -#include -#include -#include -#include -#include -#include -#include - -namespace UnitTest -{ - using JobExceptionPolicy = TestImpact::TestEnumerator::JobExceptionPolicy; - using CacheExceptionPolicy = TestImpact::TestEnumerator::CacheExceptionPolicy; - - // Generates the command to run the given test target through AzTestRunner and get gtest to output the enumeration file - AZStd::string GetEnumerateCommandForTarget(AZStd::pair testTarget) - { - return AZStd::string::format( - "%s %s AzRunUnitTests --gtest_list_tests --gtest_output=xml:%s", - LY_TEST_IMPACT_AZ_TESTRUNNER_BIN, - testTarget.first.c_str(), // Path to test target bin - testTarget.second.c_str()); // Path to test target gtest enumeration file - } - - class TestEnumeratorFixture - : public AllocatorsTestFixture - { - public: - void SetUp() override; - - protected: - using JobInfo = TestImpact::TestEnumerator::JobInfo; - using JobData = TestImpact::TestEnumerator::JobData; - - AZStd::vector m_jobInfos; - AZStd::unique_ptr m_testEnumerator; - AZStd::vector m_testTargetJobArgs; - AZStd::vector> m_testTargetPaths; - AZStd::vector m_expectedTestTargetEnumerations; - AZStd::vector m_cacheFiles; - size_t m_maxConcurrency = 0; - }; - - void TestEnumeratorFixture::SetUp() - { - UnitTest::AllocatorsTestFixture::SetUp(); - - DeleteFiles(LY_TEST_IMPACT_TEST_TARGET_ENUMERATION_DIR, "*.cache"); - DeleteFiles(LY_TEST_IMPACT_TEST_TARGET_ENUMERATION_DIR, "*.xml"); - - // first: path to test target bin - // second: path to test target gtest enumeration file in XML format - const AZStd::string enumPath = AZStd::string(LY_TEST_IMPACT_TEST_TARGET_ENUMERATION_DIR) + "/%s.Enumeration.xml"; - m_testTargetPaths.emplace_back( - LY_TEST_IMPACT_TEST_TARGET_A_BIN, AZStd::string::format(enumPath.c_str(), LY_TEST_IMPACT_TEST_TARGET_A_BASE_NAME)); - m_testTargetPaths.emplace_back( - LY_TEST_IMPACT_TEST_TARGET_B_BIN, AZStd::string::format(enumPath.c_str(), LY_TEST_IMPACT_TEST_TARGET_B_BASE_NAME)); - m_testTargetPaths.emplace_back( - LY_TEST_IMPACT_TEST_TARGET_C_BIN, AZStd::string::format(enumPath.c_str(), LY_TEST_IMPACT_TEST_TARGET_C_BASE_NAME)); - m_testTargetPaths.emplace_back( - LY_TEST_IMPACT_TEST_TARGET_D_BIN, AZStd::string::format(enumPath.c_str(), LY_TEST_IMPACT_TEST_TARGET_D_BASE_NAME)); - - m_expectedTestTargetEnumerations.emplace_back(GetTestTargetATestEnumerationSuites()); - m_expectedTestTargetEnumerations.emplace_back(GetTestTargetBTestEnumerationSuites()); - m_expectedTestTargetEnumerations.emplace_back(GetTestTargetCTestEnumerationSuites()); - m_expectedTestTargetEnumerations.emplace_back(GetTestTargetDTestEnumerationSuites()); - - // Path to enumeration file in TIAF internal JSON format - m_cacheFiles.emplace_back( - AZStd::string::format("%s/%s.cache", LY_TEST_IMPACT_TEST_TARGET_ENUMERATION_DIR, LY_TEST_IMPACT_TEST_TARGET_A_BASE_NAME)); - m_cacheFiles.emplace_back( - AZStd::string::format("%s/%s.cache", LY_TEST_IMPACT_TEST_TARGET_ENUMERATION_DIR, LY_TEST_IMPACT_TEST_TARGET_B_BASE_NAME)); - m_cacheFiles.emplace_back( - AZStd::string::format("%s/%s.cache", LY_TEST_IMPACT_TEST_TARGET_ENUMERATION_DIR, LY_TEST_IMPACT_TEST_TARGET_C_BASE_NAME)); - m_cacheFiles.emplace_back( - AZStd::string::format("%s/%s.cache", LY_TEST_IMPACT_TEST_TARGET_ENUMERATION_DIR, LY_TEST_IMPACT_TEST_TARGET_D_BASE_NAME)); - - for (const auto& testTarget : m_testTargetPaths) - { - m_testTargetJobArgs.emplace_back(GetEnumerateCommandForTarget(testTarget)); - } - } - - // Fixture parameterized for different max number of concurrent jobs - class TestEnumeratorFixtureWithConcurrencyParams - : public TestEnumeratorFixture - , public ::testing::WithParamInterface - { - public: - void SetUp() override - { - TestEnumeratorFixture::SetUp(); - m_maxConcurrency = GetParam(); - } - }; - - using ConcurrencyAndJobExceptionPermutation = AZStd::tuple - < - size_t, // Max number of concurrent processes - JobExceptionPolicy // Test job exception policy - >; - - // Fixture parameterized for different max number of concurrent jobs and different job exception policies - class TestEnumeratorFixtureWithConcurrencyAndJobExceptionParams - : public TestEnumeratorFixture - , public ::testing::WithParamInterface - { - public: - void SetUp() override - { - TestEnumeratorFixture::SetUp(); - const auto& [maxConcurrency, jobExceptionPolicy] = GetParam(); - m_maxConcurrency = maxConcurrency; - m_jobExceptionPolicy = jobExceptionPolicy; - } - - protected: - JobExceptionPolicy m_jobExceptionPolicy = JobExceptionPolicy::Never; - }; - - using ConcurrencyAndCacheExceptionPermutation = AZStd::tuple - < - size_t, // Max number of concurrent processes - CacheExceptionPolicy // Test enumeration exception policy - >; - - // Fixture parameterized for different max number of concurrent jobs and different enumeration exception policies - class TestEnumeratorFixtureWithConcurrencyAndCacheExceptionParams - : public TestEnumeratorFixture - , public ::testing::WithParamInterface - { - public: - void SetUp() override - { - TestEnumeratorFixture::SetUp(); - const auto& [maxConcurrency, cacheExceptionPolicy] = GetParam(); - m_maxConcurrency = maxConcurrency; - m_cacheExceptionPolicy = cacheExceptionPolicy; - } - - protected: - CacheExceptionPolicy m_cacheExceptionPolicy = CacheExceptionPolicy::Never; - }; - - namespace - { - AZStd::array MaxConcurrentEnumerations = {{1, 2, 3, 4}}; - - AZStd::array JobExceptionPolicies = - { - JobExceptionPolicy::Never, - JobExceptionPolicy::OnExecutedWithFailure, - JobExceptionPolicy::OnFailedToExecute - }; - - AZStd::array CacheExceptionPolicies = - { - CacheExceptionPolicy::Never, - CacheExceptionPolicy::OnCacheNotExist, - CacheExceptionPolicy::OnCacheReadFailure, - CacheExceptionPolicy::OnCacheWriteFailure - }; - } - - // Validates that the specified job successfully read from its test enumeration cache - void ValidateJobSuccessfulCacheRead(const TestImpact::TestEnumerator::Job& job) - { - EXPECT_EQ(job.GetResult(), TestImpact::JobResult::NotExecuted); - EXPECT_EQ(job.GetStartTime(), AZStd::chrono::high_resolution_clock::time_point()); - EXPECT_EQ(job.GetEndTime(), AZStd::chrono::high_resolution_clock::time_point()); - EXPECT_EQ(job.GetDuration(), AZStd::chrono::milliseconds(0)); - EXPECT_FALSE(job.GetReturnCode().has_value()); - EXPECT_TRUE(job.GetPayload().has_value()); - } - - // Validates that the specified test enumeration matched the expected output - void ValidateTestTargetEnumeration(const TestImpact::TestEnumeration& actualResult, const TestImpact::TestEnumeration& expectedResult) - { - EXPECT_TRUE(actualResult == expectedResult); - EXPECT_EQ(actualResult.GetNumTestSuites(), CalculateNumTestSuites(expectedResult.GetTestSuites())); - EXPECT_EQ(actualResult.GetNumTests(), CalculateNumTests(expectedResult.GetTestSuites())); - EXPECT_EQ(actualResult.GetNumEnabledTests(), CalculateNumEnabledTests(expectedResult.GetTestSuites())); - EXPECT_EQ(actualResult.GetNumDisabledTests(), CalculateNumDisabledTests(expectedResult.GetTestSuites())); - } - - // Validates that the specified test enumeration cache matches the expected output - void ValidateTestEnumerationCache(const AZ::IO::Path& cacheFile, const TestImpact::TestEnumeration& expectedEnumeration) - { - // Cache file must exist - const auto fileSize = AZ::IO::SystemFile::Length(cacheFile.c_str()); - EXPECT_GT(fileSize, 0); - - // Read raw byte data from cache - AZStd::vector buffer(fileSize + 1); - buffer[fileSize] = 0; - EXPECT_TRUE(AZ::IO::SystemFile::Read(cacheFile.c_str(), buffer.data())); - - // Transform raw byte data to raw string data and attempt to construct the test enumeration - AZStd::string rawEnum(buffer.begin(), buffer.end()); - TestImpact::TestEnumeration actualEnumeration = TestImpact::DeserializeTestEnumeration(rawEnum); - - // Check that the constructed test enumeration matches the expected enumeration - ValidateTestTargetEnumeration(actualEnumeration, expectedEnumeration); - } - - // Validates that the specified cache file does not exist - void ValidateInvalidTestEnumerationCache(const AZ::IO::Path& cacheFile) - { - EXPECT_FALSE(AZ::IO::SystemFile::Exists(cacheFile.c_str())); - } - - TEST_P( - TestEnumeratorFixtureWithConcurrencyAndJobExceptionParams, InvalidCommandArgument_ExpectJobResulFailedToExecuteeOrTestJobException) - { - // Given a test enumerator with no client callback or enumeration timeout or enumerator timeout - m_testEnumerator = AZStd::make_unique( - AZStd::nullopt, - m_maxConcurrency, - AZStd::nullopt, - AZStd::nullopt); - - // Given a mixture of test enumeration jobs with valid and invalid command arguments - for (size_t jobId = 0; jobId < m_testTargetJobArgs.size(); jobId++) - { - const AZStd::string args = (jobId % 2) ? InvalidProcessPath : m_testTargetJobArgs[jobId]; - JobData jobData(m_testTargetPaths[jobId].second, AZStd::nullopt); - m_jobInfos.emplace_back(JobInfo({jobId}, args, AZStd::move(jobData))); - } - - try - { - // When the test enumeration jobs are executed with different exception policies - const auto enumerationJobs = m_testEnumerator->Enumerate(m_jobInfos, CacheExceptionPolicy::Never, m_jobExceptionPolicy); - - // Expect this statement to be reachable only if no exception policy for launch failures - EXPECT_FALSE(::IsFlagSet(m_jobExceptionPolicy, JobExceptionPolicy::OnFailedToExecute)); - - for (const auto& job : enumerationJobs) - { - const JobInfo::IdType jobId = job.GetJobInfo().GetId().m_value; - if (jobId % 2) - { - // Expect invalid jobs have a job result of FailedToExecute - ValidateJobFailedToExecute(job); - } - else - { - // Expect the valid jobs job to successfully result in a test enumeration that matches the expected test enumeration data - ValidateJobExecutedSuccessfully(job); - ValidateTestTargetEnumeration(job.GetPayload().value(), m_expectedTestTargetEnumerations[jobId]); - } - } - } - catch ([[maybe_unused]] const TestImpact::TestJobException& e) - { - // Expect this statement to be reachable only if there is an exception policy for launch failures - EXPECT_TRUE(::IsFlagSet(m_jobExceptionPolicy, JobExceptionPolicy::OnFailedToExecute)); - } - catch (...) - { - // Do not expect any other exceptions - FAIL(); - } - } - - TEST_P( - TestEnumeratorFixtureWithConcurrencyAndJobExceptionParams, ErroneousReturnCode_ExpectJobResultExecutedWithFailureOrTestJobException) - { - // Given a test enumerator with no client callback or enumeration timeout or enumerator timeout - m_testEnumerator = AZStd::make_unique( - AZStd::nullopt, - m_maxConcurrency, - AZStd::nullopt, - AZStd::nullopt); - - // Given a mixture of test enumeration jobs that execute and return either successfully or with failure - for (size_t jobId = 0; jobId < m_testTargetJobArgs.size(); jobId++) - { - JobData jobData(m_testTargetPaths[jobId].second, AZStd::nullopt); - const AZStd::string args = (jobId % 2) - ? AZStd::string::format("%s %s", ValidProcessPath, ConstructTestProcessArgs(jobId, AZStd::chrono::milliseconds(0)).c_str()) - : m_testTargetJobArgs[jobId]; - m_jobInfos.emplace_back(JobInfo({jobId}, args, AZStd::move(jobData))); - } - - try - { - // When the test enumeration jobs are executed with different exception policies - const auto enumerationJobs = m_testEnumerator->Enumerate(m_jobInfos, CacheExceptionPolicy::Never, m_jobExceptionPolicy); - - // Expect this statement to be reachable only if no exception policy for jobs that return with error - EXPECT_FALSE(::IsFlagSet(m_jobExceptionPolicy, JobExceptionPolicy::OnExecutedWithFailure)); - - for (const auto& job : enumerationJobs) - { - const JobInfo::IdType jobId = job.GetJobInfo().GetId().m_value; - if (jobId % 2) - { - // Expect failed jobs to have job result ExecutedWithFailure and a non-zero return code - ValidateJobExecutedWithFailure(job); - } - else - { - // Expect the valid jobs job to successfully result in a test enumeration that matches the expected test enumeration data - ValidateJobExecutedSuccessfully(job); - ValidateTestTargetEnumeration(job.GetPayload().value(), m_expectedTestTargetEnumerations[jobId]); - } - } - } - catch ([[maybe_unused]] const TestImpact::TestJobException& e) - { - // Expect this statement to be reachable only if there is an exception policy for jobs that return with error - EXPECT_TRUE(::IsFlagSet(m_jobExceptionPolicy, JobExceptionPolicy::OnExecutedWithFailure)); - } - catch (...) - { - // Do not expect any other exceptions - FAIL(); - } - } - - TEST_P(TestEnumeratorFixtureWithConcurrencyAndCacheExceptionParams, EmptyCacheRead_NoCacheDataButEnumerationsMatchTestSuitesInTarget) - { - // Given a test enumerator with no client callback, enumeration timeout or enumerator timeout - m_testEnumerator = AZStd::make_unique( - AZStd::nullopt, - m_maxConcurrency, - AZStd::nullopt, - AZStd::nullopt); - - // Given an test enumeration job for each test target that reads from an enumeration caching - for (size_t jobId = 0; jobId < m_testTargetJobArgs.size(); jobId++) - { - JobData jobData(m_testTargetPaths[jobId].second, JobData::Cache{JobData::CachePolicy::Read, m_cacheFiles[jobId]}); - m_jobInfos.emplace_back(JobInfo({jobId}, m_testTargetJobArgs[jobId], AZStd::move(jobData))); - } - - try - { - // When the test enumeration jobs are executed with different exception policies - const auto enumerationJobs = m_testEnumerator->Enumerate(m_jobInfos, m_cacheExceptionPolicy, JobExceptionPolicy::Never); - - // Expect this statement to be reachable only if no exception policy for read attempts of non-existent caches - EXPECT_FALSE(::IsFlagSet(m_cacheExceptionPolicy, CacheExceptionPolicy::OnCacheNotExist)); - - // Expect each job to successfully result in a test enumeration that matches the expected test enumeration data for that test target - // even though the cache files could not be read - for (const auto& job : enumerationJobs) - { - const JobInfo::IdType jobId = job.GetJobInfo().GetId().m_value; - ValidateJobExecutedSuccessfully(job); - ValidateTestTargetEnumeration(job.GetPayload().value(), m_expectedTestTargetEnumerations[jobId]); - ValidateInvalidTestEnumerationCache(job.GetJobInfo().GetCache()->m_file); - } - } - catch ([[maybe_unused]] const TestImpact::TestEnumerationException& e) - { - // Expect this statement to be reachable only if there is an exception policy for read attempts of non-existent caches - EXPECT_TRUE(::IsFlagSet(m_cacheExceptionPolicy, CacheExceptionPolicy::OnCacheNotExist)); - } - catch (...) - { - // Do not expect any other exceptions - FAIL(); - } - } - - // Note: this test only cues up one test for enumeration but still runs the permutations for max concurrency so there is duplicated work - TEST_P( - TestEnumeratorFixtureWithConcurrencyAndCacheExceptionParams, - EmptyCacheDataRead_ExpectEnumerationsMatchTestSuitesInTargetOrTestEnumerationException) - { - // Given an enumeration cache for Test Target A with invalid JSON data - WriteTextToFile("", m_cacheFiles[TestTargetA]); - - // Given a test enumerator with no client callback, enumeration timeout or enumerator timeout - m_testEnumerator = AZStd::make_unique( - AZStd::nullopt, - m_maxConcurrency, - AZStd::nullopt, - AZStd::nullopt); - - // Given an test enumeration job that will attempt to read an invalid enumeration cache - JobData jobData(m_testTargetPaths[TestTargetA].second, JobData::Cache{JobData::CachePolicy::Read, m_cacheFiles[TestTargetA]}); - m_jobInfos.emplace_back(JobInfo({TestTargetA}, m_testTargetJobArgs[TestTargetA], AZStd::move(jobData))); - - try - { - // When the test enumeration jobs are executed with different exception policies - const auto enumerationJobs = m_testEnumerator->Enumerate(m_jobInfos, m_cacheExceptionPolicy, JobExceptionPolicy::Never); - - // Expect this statement to be reachable only if no exception policy for cache reads that fail - EXPECT_FALSE(::IsFlagSet(m_cacheExceptionPolicy, CacheExceptionPolicy::OnCacheReadFailure)); - - for (const auto& job : enumerationJobs) - { - const JobInfo::IdType jobId = job.GetJobInfo().GetId().m_value; - - // Expect the valid jobs job to successfully result in a test enumeration that matches the expected test enumeration data - ValidateJobExecutedSuccessfully(job); - ValidateTestTargetEnumeration(job.GetPayload().value(), m_expectedTestTargetEnumerations[jobId]); - } - } - catch ([[maybe_unused]] const TestImpact::TestEnumerationException& e) - { - // Expect this statement to be reachable only if there is an exception policy for cache reads that fail - EXPECT_TRUE(::IsFlagSet(m_cacheExceptionPolicy, CacheExceptionPolicy::OnCacheReadFailure)); - } - catch (...) - { - // Do not expect any other exceptions - FAIL(); - } - } - - TEST_P( - TestEnumeratorFixtureWithConcurrencyAndCacheExceptionParams, - InvalidCacheWrite_ExpectEnumerationsMatchTestSuitesInTargetOrTestEnumerationException) - { - // Given a test enumerator with no client callback,, enumeration timeout or enumerator timeout - m_testEnumerator = AZStd::make_unique( - AZStd::nullopt, - m_maxConcurrency, - AZStd::nullopt, - AZStd::nullopt); - - // Given an test enumeration job that will attempt to write to an invalid enumeration cache - JobData jobData(m_testTargetPaths[TestTargetA].second, JobData::Cache{JobData::CachePolicy::Write, InvalidProcessPath}); - m_jobInfos.emplace_back(JobInfo({TestTargetA}, m_testTargetJobArgs[TestTargetA], AZStd::move(jobData))); - - try - { - // When the test enumeration job is executed - const auto enumerationJobs = m_testEnumerator->Enumerate(m_jobInfos, m_cacheExceptionPolicy, JobExceptionPolicy::Never); - - // Expect this statement to be reachable only if no exception policy for cache writes that fail - EXPECT_FALSE(::IsFlagSet(m_cacheExceptionPolicy, CacheExceptionPolicy::OnCacheWriteFailure)); - - for (const auto& job : enumerationJobs) - { - const JobInfo::IdType jobId = job.GetJobInfo().GetId().m_value; - - // Expect the valid jobs job to successfully result in a test enumeration that matches the expected test enumeration data - ValidateJobExecutedSuccessfully(job); - ValidateTestTargetEnumeration(job.GetPayload().value(), m_expectedTestTargetEnumerations[jobId]); - } - } - catch ([[maybe_unused]] const TestImpact::TestEnumerationException& e) - { - // Expect this statement to be reachable only if there is an exception policy for cache writes that fail - EXPECT_TRUE(::IsFlagSet(m_cacheExceptionPolicy, CacheExceptionPolicy::OnCacheWriteFailure)); - } - catch (...) - { - // Do not expect any other exceptions - FAIL(); - } - } - - TEST_P(TestEnumeratorFixtureWithConcurrencyParams, ValidAndInvalidCacheRead_CachedEnumerationsMatchTestSuitesInTarget) - { - // Given the cache file written for only test target B - WriteTextToFile(TestImpact::SerializeTestEnumeration(m_expectedTestTargetEnumerations[TestTargetB]), m_cacheFiles[TestTargetB]); - - // Given a test enumerator with no client callback, enumeration timeout or enumerator timeout - m_testEnumerator = AZStd::make_unique( - AZStd::nullopt, - m_maxConcurrency, - AZStd::nullopt, - AZStd::nullopt); - - // Given test enumeration jobs test target A and D with no enumeration caching - m_jobInfos.emplace_back( - JobInfo({TestTargetA}, m_testTargetJobArgs[TestTargetA], JobData{m_testTargetPaths[TestTargetA].second, AZStd::nullopt})); - m_jobInfos.emplace_back( - JobInfo({TestTargetD}, m_testTargetJobArgs[TestTargetD], JobData{m_testTargetPaths[TestTargetD].second, AZStd::nullopt})); - - // Given test target B with enumeration cache reading and a valid cache file - m_jobInfos.emplace_back(JobInfo( - {TestTargetB}, m_testTargetJobArgs[TestTargetB], - JobData{m_testTargetPaths[TestTargetB].second, JobInfo::Cache{JobData::CachePolicy::Read, m_cacheFiles[TestTargetB]}})); - - // Given test target C with enumeration cache reading and an invalid cache file - m_jobInfos.emplace_back(JobInfo( - {TestTargetC}, m_testTargetJobArgs[TestTargetC], - JobData{m_testTargetPaths[TestTargetC].second, JobInfo::Cache{JobData::CachePolicy::Read, "nothing"}})); - - // When the test enumeration jobs are executed - const auto enumerationJobs = m_testEnumerator->Enumerate(m_jobInfos, CacheExceptionPolicy::Never, JobExceptionPolicy::Never); - - // Expect each job to successfully result in a test enumeration that matches the expected test enumeration data for that test target - for (const auto& job : enumerationJobs) - { - const JobInfo::IdType jobId = job.GetJobInfo().GetId().m_value; - - switch (jobId) - { - case TestTargetA: // No cache read - case TestTargetC: // Cache read, but invalid cache, so re-enumerate anyway - case TestTargetD: // No cache read - { - ValidateJobExecutedSuccessfully(job); - break; - } - case TestTargetB: // Cache read, successful cache read, so job not executed - { - ValidateJobSuccessfulCacheRead(job); - break; - } - default: - { - FAIL(); - } - } - - // Regardless of cache policy and cache failures all targets should still produce the expected test enumerations - ValidateTestTargetEnumeration(job.GetPayload().value(), m_expectedTestTargetEnumerations[jobId]); - } - } - - TEST_F(TestEnumeratorFixture, InvalidCacheDataRead_TestEnumerationException) - { - // Given an enumeration cache for Test Target A with invalid JSON data - WriteTextToFile("There is no valid cache data here", m_cacheFiles[TestTargetA]); - - // Given a test enumerator with no client callback, concurrency, enumeration timeout or enumerator timeout - m_testEnumerator = AZStd::make_unique( - AZStd::nullopt, - OneConcurrentProcess, - AZStd::nullopt, - AZStd::nullopt); - - // Given an test enumeration job that will attempt to read an invalid enumeration cache - JobData jobData(m_testTargetPaths[TestTargetA].second, JobData::Cache{JobData::CachePolicy::Read, m_cacheFiles[TestTargetA]}); - m_jobInfos.emplace_back(JobInfo({TestTargetA}, m_testTargetJobArgs[TestTargetA], AZStd::move(jobData))); - - try - { - // When the test enumeration jobs are executed with different exception policies - const auto enumerationJobs = m_testEnumerator->Enumerate(m_jobInfos, CacheExceptionPolicy::Never, JobExceptionPolicy::Never); - } - catch ([[maybe_unused]] const TestImpact::TestEnumerationException& e) - { - // Expect this statement to be reachable only if there is an exception policy for jobs that return with error - SUCCEED(); - } - catch (...) - { - // Do not expect any other exceptions - FAIL(); - } - } - - TEST_P(TestEnumeratorFixtureWithConcurrencyParams, ValidCacheWrite_CachedEnumerationsMatchTestSuitesInTarget) - { - // Given a test enumerator with no client callback, enumeration timeout or enumerator timeout - m_testEnumerator = AZStd::make_unique( - AZStd::nullopt, - m_maxConcurrency, - AZStd::nullopt, - AZStd::nullopt); - - // Given an test enumeration job for each test target with write enumeration caching - for (size_t jobId = 0; jobId < m_testTargetJobArgs.size(); jobId++) - { - JobData jobData(m_testTargetPaths[jobId].second, JobData::Cache{JobData::CachePolicy::Write, m_cacheFiles[jobId]}); - m_jobInfos.emplace_back(JobInfo({jobId}, m_testTargetJobArgs[jobId], AZStd::move(jobData))); - } - - // When the test enumeration jobs are executed - const auto enumerationJobs = m_testEnumerator->Enumerate(m_jobInfos, CacheExceptionPolicy::Never, JobExceptionPolicy::Never); - - // Expect each job to successfully result in a test enumeration and cache that matches the expected test enumeration data for that - // test target - for (const auto& job : enumerationJobs) - { - const JobInfo::IdType jobId = job.GetJobInfo().GetId().m_value; - ValidateJobExecutedSuccessfully(job); - ValidateTestTargetEnumeration(job.GetPayload().value(), m_expectedTestTargetEnumerations[jobId]); - ValidateTestEnumerationCache(job.GetJobInfo().GetCache()->m_file, m_expectedTestTargetEnumerations[jobId]); - } - } - - TEST_F(TestEnumeratorFixture, EmptyArtifact_ExpectTestEnumerationException) - { - // Given a test enumerator with no client callback, concurrency, enumeration timeout or enumerator timeout - m_testEnumerator = AZStd::make_unique( - AZStd::nullopt, - OneConcurrentProcess, - AZStd::nullopt, - AZStd::nullopt); - - // Given an test enumeration job that will return successfully but with an empty artifact string - m_jobInfos.emplace_back(JobInfo({0}, m_testTargetJobArgs[0], JobData("", AZStd::nullopt))); - - try - { - // When the test enumeration job is executed - const auto enumerationJobs = m_testEnumerator->Enumerate(m_jobInfos, CacheExceptionPolicy::Never, JobExceptionPolicy::Never); - - // Do not expect this statement to be reachable - FAIL(); - } - catch ([[maybe_unused]] const TestImpact::TestEnumerationException& e) - { - // Expect an enumeration exception - SUCCEED(); - } - catch (...) - { - // Do not expect any other exceptions - FAIL(); - } - } - - TEST_F(TestEnumeratorFixture, InvalidArtifact_ExpectTestEnumerationException) - { - // Given a test enumerator with no client callback, concurrency, enumeration timeout or enumerator timeout - m_testEnumerator = AZStd::make_unique( - AZStd::nullopt, - OneConcurrentProcess, - AZStd::nullopt, - AZStd::nullopt); - - // Given an test enumeration job that will return successfully but not produce an artifact - m_jobInfos.emplace_back(JobInfo( - {0}, AZStd::string::format("%s %s", ValidProcessPath, ConstructTestProcessArgs(0, AZStd::chrono::milliseconds(0)).c_str()), - JobData("", AZStd::nullopt))); - - try - { - // When the test enumeration job is executed - const auto enumerationJobs = m_testEnumerator->Enumerate(m_jobInfos, CacheExceptionPolicy::Never, JobExceptionPolicy::Never); - - // Do not expect this statement to be reachable - FAIL(); - } - catch ([[maybe_unused]] const TestImpact::TestEnumerationException& e) - { - // Expect an enumeration exception - SUCCEED(); - } - catch (...) - { - // Do not expect any other exceptions - FAIL(); - } - } - - TEST_P(TestEnumeratorFixtureWithConcurrencyParams, EnumerateTestTargets_EnumerationsMatchTestSuitesInTarget) - { - // Given a test enumerator with no client callback, enumeration timeout or enumerator timeout - m_testEnumerator = AZStd::make_unique( - AZStd::nullopt, - m_maxConcurrency, - AZStd::nullopt, - AZStd::nullopt); - - // Given an test enumeration job for each test target with no enumeration caching - for (size_t jobId = 0; jobId < m_testTargetJobArgs.size(); jobId++) - { - JobData jobData(m_testTargetPaths[jobId].second, AZStd::nullopt); - m_jobInfos.emplace_back(JobInfo({jobId}, m_testTargetJobArgs[jobId], AZStd::move(jobData))); - } - - // When the test enumeration jobs are executed - const auto enumerationJobs = m_testEnumerator->Enumerate(m_jobInfos, CacheExceptionPolicy::Never, JobExceptionPolicy::Never); - - // Expect each job to successfully result in a test enumeration that matches the expected test enumeration data for that test target - for (const auto& job : enumerationJobs) - { - const JobInfo::IdType jobId = job.GetJobInfo().GetId().m_value; - ValidateJobExecutedSuccessfully(job); - ValidateTestTargetEnumeration(job.GetPayload().value(), m_expectedTestTargetEnumerations[jobId]); - } - } - - TEST_P(TestEnumeratorFixtureWithConcurrencyParams, EnumerateTestTargetsWithArbitraryJobIds_EnumerationsMatchTestSuitesInTarget) - { - // Given a set of arbitrary job ids to be used for the test target jobs - enum - { - ArbitraryA = 36, - ArbitraryB = 890, - ArbitraryC = 19, - ArbitraryD = 1 - }; - - const AZStd::unordered_map sequentialToArbitrary = - { - {TestTargetA, ArbitraryA}, - {TestTargetB, ArbitraryB}, - {TestTargetC, ArbitraryC}, - {TestTargetD, ArbitraryD}, - }; - - const AZStd::unordered_map arbitraryToSequential = - { - {ArbitraryA, TestTargetA}, - {ArbitraryB, TestTargetB}, - {ArbitraryC, TestTargetC}, - {ArbitraryD, TestTargetD}, - }; - - // Given a test enumerator with no client callback, enumeration timeout or enumerator timeout - m_testEnumerator = AZStd::make_unique( - AZStd::nullopt, - m_maxConcurrency, - AZStd::nullopt, - AZStd::nullopt); - - // Given an test enumeration job for each test target with no enumeration caching - for (size_t jobId = 0; jobId < m_testTargetJobArgs.size(); jobId++) - { - JobData jobData(m_testTargetPaths[jobId].second, AZStd::nullopt); - m_jobInfos.emplace_back(JobInfo({sequentialToArbitrary.at(jobId)}, m_testTargetJobArgs[jobId], AZStd::move(jobData))); - } - - // When the test enumeration jobs are executed - const auto enumerationJobs = m_testEnumerator->Enumerate(m_jobInfos, CacheExceptionPolicy::Never, JobExceptionPolicy::Never); - - // Expect each job to successfully result in a test enumeration that matches the expected test enumeration data for that test target - for (const auto& job : enumerationJobs) - { - const JobInfo::IdType jobId = arbitraryToSequential.at(job.GetJobInfo().GetId().m_value); - ValidateJobExecutedSuccessfully(job); - ValidateTestTargetEnumeration(job.GetPayload().value(), m_expectedTestTargetEnumerations[jobId]); - } - } - - TEST_P(TestEnumeratorFixtureWithConcurrencyParams, EnumerateTestTargetsWithCallback_EnumerationsMatchTestSuitesInTarget) - { - // Given a client callback function that tracks the number of successful enumerations - size_t numSuccesses = 0; - const auto jobCallback = [&numSuccesses]([[maybe_unused]] const TestImpact::TestEnumerator::JobInfo& jobInfo, const TestImpact::JobMeta& meta) - { - if (meta.m_result == TestImpact::JobResult::ExecutedWithSuccess) - { - numSuccesses++; - } - }; - - // Given a test enumerator with no enumeration timeout or enumerator timeout - m_testEnumerator = AZStd::make_unique( - jobCallback, - m_maxConcurrency, - AZStd::nullopt, - AZStd::nullopt); - - // Given an test enumeration job for each test target with no enumeration caching - for (size_t jobId = 0; jobId < m_testTargetJobArgs.size(); jobId++) - { - JobData jobData(m_testTargetPaths[jobId].second, AZStd::nullopt); - m_jobInfos.emplace_back(JobInfo({jobId}, m_testTargetJobArgs[jobId], AZStd::move(jobData))); - } - - // When the test enumeration jobs are executed - const auto enumerationJobs = m_testEnumerator->Enumerate(m_jobInfos, CacheExceptionPolicy::Never, JobExceptionPolicy::Never); - - // Expect the number of successful enumerations tracked in the callback to match the number of test targets enumerated - EXPECT_EQ(numSuccesses, enumerationJobs.size()); - - // Expect each job to successfully result in a test enumeration that matches the expected test enumeration data for that test target - for (const auto& job : enumerationJobs) - { - const JobInfo::IdType jobId = job.GetJobInfo().GetId().m_value; - ValidateJobExecutedSuccessfully(job); - ValidateTestTargetEnumeration(job.GetPayload().value(), m_expectedTestTargetEnumerations[jobId]); - } - } - - TEST_P(TestEnumeratorFixtureWithConcurrencyParams, JobRunnerTimeout_InFlightJobsTimeoutAndQueuedJobsUnlaunched) - { - // Given a test enumerator with no client callback or enumerator timeout and 500ms enumeration timeout - m_testEnumerator = AZStd::make_unique( - AZStd::nullopt, - m_maxConcurrency, - AZStd::chrono::milliseconds(500), - AZStd::nullopt); - - // Given an test enumeration job for each test target with no enumeration caching where half will sleep indefinitely - for (size_t jobId = 0; jobId < m_testTargetJobArgs.size(); jobId++) - { - JobData jobData(m_testTargetPaths[jobId].second, AZStd::nullopt); - const AZStd::string args = (jobId % 2) - ? AZStd::string::format("%s %s", ValidProcessPath, ConstructTestProcessArgs(jobId, LongSleep).c_str()) - : m_testTargetJobArgs[jobId]; - m_jobInfos.emplace_back(JobInfo({jobId}, args, AZStd::move(jobData))); - } - - // When the test enumeration jobs are executed - const auto enumerationJobs = m_testEnumerator->Enumerate(m_jobInfos, CacheExceptionPolicy::Never, JobExceptionPolicy::Never); - - // Expect half the jobs to successfully result in a test enumeration that matches the expected test enumeration data for that test - // target with the other half having timed out - for (const auto& job : enumerationJobs) - { - const JobInfo::IdType jobId = job.GetJobInfo().GetId().m_value; - if (jobId % 2) - { - ValidateJobTimeout(job); - } - else - { - ValidateJobExecutedSuccessfully(job); - ValidateTestTargetEnumeration(job.GetPayload().value(), m_expectedTestTargetEnumerations[jobId]); - } - } - } - - TEST_F(TestEnumeratorFixture, JobTimeout_InFlightJobTimeoutAndQueuedJobsUnlaunched) - { - // Given a test enumerator with no client callback or enumeration timeout and a 5 second enumerator timeout - m_testEnumerator = AZStd::make_unique( - AZStd::nullopt, - FourConcurrentProcesses, - AZStd::nullopt, - AZStd::chrono::milliseconds(5000)); - - // Given an test enumeration job for each test target with no enumeration caching where half will sleep indefinitely - for (size_t jobId = 0; jobId < m_testTargetJobArgs.size(); jobId++) - { - JobData jobData(m_testTargetPaths[jobId].second, AZStd::nullopt); - const AZStd::string args = (jobId % 2) - ? AZStd::string::format("%s %s", ValidProcessPath, ConstructTestProcessArgs(jobId, LongSleep).c_str()) - : m_testTargetJobArgs[jobId]; - m_jobInfos.emplace_back(JobInfo({jobId}, args, AZStd::move(jobData))); - } - - // When the test enumeration jobs are executed - const auto enumerationJobs = m_testEnumerator->Enumerate(m_jobInfos, CacheExceptionPolicy::Never, JobExceptionPolicy::Never); - - // Expect half the jobs to successfully result in a test enumeration that matches the expected test enumeration data for that test - // target with the other half having timed out - for (const auto& job : enumerationJobs) - { - const JobInfo::IdType jobId = job.GetJobInfo().GetId().m_value; - if (jobId % 2) - { - ValidateJobTimeout(job); - } - else - { - ValidateJobExecutedSuccessfully(job); - ValidateTestTargetEnumeration(job.GetPayload().value(), m_expectedTestTargetEnumerations[jobId]); - } - } - } - - INSTANTIATE_TEST_CASE_P( - , - TestEnumeratorFixtureWithConcurrencyAndJobExceptionParams, - ::testing::Combine( - ::testing::ValuesIn(MaxConcurrentEnumerations), - ::testing::ValuesIn(JobExceptionPolicies)) - ); - - INSTANTIATE_TEST_CASE_P( - , - TestEnumeratorFixtureWithConcurrencyAndCacheExceptionParams, - ::testing::Combine( - ::testing::ValuesIn(MaxConcurrentEnumerations), - ::testing::ValuesIn(CacheExceptionPolicies)) - ); - - INSTANTIATE_TEST_CASE_P( - , - TestEnumeratorFixtureWithConcurrencyParams, - ::testing::ValuesIn(MaxConcurrentEnumerations) - ); -} // namespace UnitTest diff --git a/Code/Tools/TestImpactFramework/Runtime/Code/Tests/Test/TestImpactTestEumerationSerializerTest.cpp b/Code/Tools/TestImpactFramework/Runtime/Code/Tests/Test/TestImpactTestEumerationSerializerTest.cpp deleted file mode 100644 index aa94657378..0000000000 --- a/Code/Tools/TestImpactFramework/Runtime/Code/Tests/Test/TestImpactTestEumerationSerializerTest.cpp +++ /dev/null @@ -1,99 +0,0 @@ -/* - * All or portions of this file Copyright (c) Amazon.com, Inc. or its affiliates or - * its licensors. - * - * For complete copyright and license terms please see the LICENSE at the root of this - * distribution (the "License"). All use of this software is governed by the License, - * or, if provided, by the license below or the license accompanying this file. Do not - * remove or modify any license notices. This file is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * - */ - -#include - -#include -#include - -#include -#include - -namespace UnitTest -{ - const TestImpact::TestEnumeration EmptyTestEnum(AZStd::vector{}); - - TEST(TestEnumerationSerializer, EmptyTestEnumerationSuites_ExpectTemptyTestEnumeration) - { - // Given an empty set of test enumeration suites - // When the test enumeration is serialized and deserialized back again - const AZStd::string serializedString = TestImpact::SerializeTestEnumeration(EmptyTestEnum); - const TestImpact::TestEnumeration actualEnumeration = TestImpact::DeserializeTestEnumeration(serializedString); - - // Expect the actual test enumeration to match the expected test enumeration - EXPECT_TRUE(actualEnumeration == EmptyTestEnum); - } - - TEST(TestEnumerationSerializer, SerializeAndDeserializeSuitesForTestTargetA_ActualSuiteDataMatchesExpectedSuiteData) - { - // Given the test of test enumeration suites for Test target A - const TestImpact::TestEnumeration expectedEnumeration = TestImpact::TestEnumeration(GetTestTargetATestEnumerationSuites()); - - // When the test enumeration is serialized and deserialized back again - const AZStd::string serializedString = TestImpact::SerializeTestEnumeration(expectedEnumeration); - const TestImpact::TestEnumeration actualEnumeration = TestImpact::DeserializeTestEnumeration(serializedString); - - // Expect the actual test enumeration to match the expected test enumeration - EXPECT_TRUE(actualEnumeration == expectedEnumeration); - - // Do not expect the actual test enumeration to match the empty test enumeration - EXPECT_FALSE(actualEnumeration == EmptyTestEnum); - } - - TEST(TestEnumerationSerializer, SerializeAndDeserializeSuitesForTestTargetB_ActualSuiteDataMatchesExpectedSuiteData) - { - // Given the test of test enumeration suites for Test target B - const TestImpact::TestEnumeration expectedEnumeration = TestImpact::TestEnumeration(GetTestTargetBTestEnumerationSuites()); - - // When the test enumeration is serialized and deserialized back again - const AZStd::string serializedString = TestImpact::SerializeTestEnumeration(expectedEnumeration); - const TestImpact::TestEnumeration actualEnumeration = TestImpact::DeserializeTestEnumeration(serializedString); - - // Expect the actual test enumeration to match the expected test enumeration - EXPECT_TRUE(actualEnumeration == expectedEnumeration); - - // Do not expect the actual test enumeration to match the empty test enumeration - EXPECT_FALSE(actualEnumeration == EmptyTestEnum); - } - - TEST(TestEnumerationSerializer, SerializeAndDeserializeSuitesForTestTargetC_ActualSuiteDataMatchesExpectedSuiteData) - { - // Given the test of test enumeration suites for Test target B - const TestImpact::TestEnumeration expectedEnumeration = TestImpact::TestEnumeration(GetTestTargetCTestEnumerationSuites()); - - // When the test enumeration is serialized and deserialized back again - const AZStd::string serializedString = TestImpact::SerializeTestEnumeration(expectedEnumeration); - const TestImpact::TestEnumeration actualEnumeration = TestImpact::DeserializeTestEnumeration(serializedString); - - // Expect the actual test enumeration to match the expected test enumeration - EXPECT_TRUE(actualEnumeration == expectedEnumeration); - - // Do not expect the actual test enumeration to match the empty test enumeration - EXPECT_FALSE(actualEnumeration == EmptyTestEnum); - } - - TEST(TestEnumerationSerializer, SerializeAndDeserializeSuitesForTestTargetD_ActualSuiteDataMatchesExpectedSuiteData) - { - // Given the test of test enumeration suites for Test target B - const TestImpact::TestEnumeration expectedEnumeration = TestImpact::TestEnumeration(GetTestTargetDTestEnumerationSuites()); - - // When the test enumeration is serialized and deserialized back again - const AZStd::string serializedString = TestImpact::SerializeTestEnumeration(expectedEnumeration); - const TestImpact::TestEnumeration actualEnumeration = TestImpact::DeserializeTestEnumeration(serializedString); - - // Expect the actual test enumeration to match the expected test enumeration - EXPECT_TRUE(actualEnumeration == expectedEnumeration); - - // Do not expect the actual test enumeration to match the empty test enumeration - EXPECT_FALSE(actualEnumeration == EmptyTestEnum); - } -} // namespace UnitTest diff --git a/Code/Tools/TestImpactFramework/Runtime/Code/Tests/Test/TestImpactTestRunSerializerTest.cpp b/Code/Tools/TestImpactFramework/Runtime/Code/Tests/Test/TestImpactTestRunSerializerTest.cpp deleted file mode 100644 index 1fc7c62b3d..0000000000 --- a/Code/Tools/TestImpactFramework/Runtime/Code/Tests/Test/TestImpactTestRunSerializerTest.cpp +++ /dev/null @@ -1,99 +0,0 @@ -/* - * All or portions of this file Copyright (c) Amazon.com, Inc. or its affiliates or - * its licensors. - * - * For complete copyright and license terms please see the LICENSE at the root of this - * distribution (the "License"). All use of this software is governed by the License, - * or, if provided, by the license below or the license accompanying this file. Do not - * remove or modify any license notices. This file is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * - */ - -#include - -#include -#include - -#include -#include - -namespace UnitTest -{ - const TestImpact::TestRun EmptyTestRun(AZStd::vector{}, AZStd::chrono::milliseconds{0}); - - TEST(TestRunSerializer, EmptyTestRunSuites_ExpectTemptyTestRun) - { - // Given an empty set of test run suites - // When the test run is serialized and deserialized back again - const auto serializedString = TestImpact::SerializeTestRun(EmptyTestRun); - const auto actualRun = TestImpact::DeserializeTestRun(serializedString); - - // Expect the actual test run to match the expected test run - EXPECT_TRUE(actualRun == EmptyTestRun); - } - - TEST(TestRunSerializer, SerializeAndDeserializeSuitesForTestTargetA_ActualSuiteDataMatchesExpectedSuiteData) - { - // Given the test of test run suites for Test target A - const auto expectedRun = TestImpact::TestRun(GetTestTargetATestRunSuites(), AZStd::chrono::milliseconds{500}); - - // When the test run is serialized and deserialized back again - const auto serializedString = TestImpact::SerializeTestRun(expectedRun); - const auto actualRun = TestImpact::DeserializeTestRun(serializedString); - - // Expect the actual test run to match the expected test run - EXPECT_TRUE(actualRun == expectedRun); - - // Do not expect the actual test run to match the empty test run - EXPECT_FALSE(actualRun == EmptyTestRun); - } - - TEST(TestRunSerializer, SerializeAndDeserializeSuitesForTestTargetB_ActualSuiteDataMatchesExpectedSuiteData) - { - // Given the test of test run suites for Test target B - const auto expectedRun = TestImpact::TestRun(GetTestTargetBTestRunSuites(), AZStd::chrono::milliseconds{500}); - - // When the test run is serialized and deserialized back again - const auto serializedString = TestImpact::SerializeTestRun(expectedRun); - const auto actualRun = TestImpact::DeserializeTestRun(serializedString); - - // Expect the actual test run to match the expected test run - EXPECT_TRUE(actualRun == expectedRun); - - // Do not expect the actual test run to match the empty test run - EXPECT_FALSE(actualRun == EmptyTestRun); - } - - TEST(TestRunSerializer, SerializeAndDeserializeSuitesForTestTargetC_ActualSuiteDataMatchesExpectedSuiteData) - { - // Given the test of test run suites for Test target B - const auto expectedRun = TestImpact::TestRun(GetTestTargetCTestRunSuites(), AZStd::chrono::milliseconds{500}); - - // When the test run is serialized and deserialized back again - const auto serializedString = TestImpact::SerializeTestRun(expectedRun); - const auto actualRun = TestImpact::DeserializeTestRun(serializedString); - - // Expect the actual test run to match the expected test run - EXPECT_TRUE(actualRun == expectedRun); - - // Do not expect the actual test run to match the empty test run - EXPECT_FALSE(actualRun == EmptyTestRun); - } - - TEST(TestRunSerializer, SerializeAndDeserializeSuitesForTestTargetD_ActualSuiteDataMatchesExpectedSuiteData) - { - // Given the test of test run suites for Test target B - const auto expectedRun = TestImpact::TestRun(GetTestTargetDTestRunSuites(), AZStd::chrono::milliseconds{500}); - - // When the test run is serialized and deserialized back again - const auto serializedString = TestImpact::SerializeTestRun(expectedRun); - const auto actualRun = TestImpact::DeserializeTestRun(serializedString); - - // Expect the actual test run to match the expected test run - EXPECT_TRUE(actualRun == expectedRun); - - // Do not expect the actual test run to match the empty test run - EXPECT_FALSE(actualRun == EmptyTestRun); - } -} // namespace UnitTest diff --git a/Code/Tools/TestImpactFramework/Runtime/Code/Tests/Test/TestImpactTestRunnerTest.cpp b/Code/Tools/TestImpactFramework/Runtime/Code/Tests/Test/TestImpactTestRunnerTest.cpp deleted file mode 100644 index 08a0306e7f..0000000000 --- a/Code/Tools/TestImpactFramework/Runtime/Code/Tests/Test/TestImpactTestRunnerTest.cpp +++ /dev/null @@ -1,516 +0,0 @@ -/* - * All or portions of this file Copyright (c) Amazon.com, Inc. or its affiliates or - * its licensors. - * - * For complete copyright and license terms please see the LICENSE at the root of this - * distribution (the "License"). All use of this software is governed by the License, - * or, if provided, by the license below or the license accompanying this file. Do not - * remove or modify any license notices. This file is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * - */ - -#include -#include - -#include -#include -#include - -#include -#include -#include -#include -#include -#include -#include -#include - -namespace UnitTest -{ - using JobExceptionPolicy = TestImpact::TestRunner::JobExceptionPolicy; - - AZStd::string GetRunCommandForTarget(AZStd::pair testTarget) - { - return AZStd::string::format( - "%s %s AzRunUnitTests --gtest_output=xml:%s", LY_TEST_IMPACT_AZ_TESTRUNNER_BIN, testTarget.first.c_str(), - testTarget.second.c_str()); - } - - class TestRunnerFixture - : public AllocatorsTestFixture - { - public: - void SetUp() override; - - protected: - using JobInfo = TestImpact::TestRunner::JobInfo; - using JobData = TestImpact::TestRunner::JobData; - - AZStd::vector m_jobInfos; - AZStd::unique_ptr m_testRunner; - AZStd::vector m_testTargetJobArgs; - AZStd::vector> m_testTargetPaths; - AZStd::vector m_expectedTestTargetRuns; - AZStd::vector m_expectedTestTargetResult; - size_t m_maxConcurrency = 0; - }; - - void TestRunnerFixture::SetUp() - { - UnitTest::AllocatorsTestFixture::SetUp(); - - DeleteFiles(LY_TEST_IMPACT_TEST_TARGET_RESULTS_DIR, "*.xml"); - - // first: path to test target bin - // second: path to test target gtest results file in XML format - const AZStd::string runPath = AZStd::string(LY_TEST_IMPACT_TEST_TARGET_RESULTS_DIR) + "/%s.Run.xml"; - m_testTargetPaths.emplace_back( - LY_TEST_IMPACT_TEST_TARGET_A_BIN, AZStd::string::format(runPath.c_str(), LY_TEST_IMPACT_TEST_TARGET_A_BASE_NAME)); - m_testTargetPaths.emplace_back( - LY_TEST_IMPACT_TEST_TARGET_B_BIN, AZStd::string::format(runPath.c_str(), LY_TEST_IMPACT_TEST_TARGET_B_BASE_NAME)); - m_testTargetPaths.emplace_back( - LY_TEST_IMPACT_TEST_TARGET_C_BIN, AZStd::string::format(runPath.c_str(), LY_TEST_IMPACT_TEST_TARGET_C_BASE_NAME)); - m_testTargetPaths.emplace_back( - LY_TEST_IMPACT_TEST_TARGET_D_BIN, AZStd::string::format(runPath.c_str(), LY_TEST_IMPACT_TEST_TARGET_D_BASE_NAME)); - - m_expectedTestTargetRuns.emplace_back(GetTestTargetATestRunSuites(), AZStd::chrono::milliseconds{500}); - m_expectedTestTargetRuns.emplace_back(GetTestTargetBTestRunSuites(), AZStd::chrono::milliseconds{500}); - m_expectedTestTargetRuns.emplace_back(GetTestTargetCTestRunSuites(), AZStd::chrono::milliseconds{500}); - m_expectedTestTargetRuns.emplace_back(GetTestTargetDTestRunSuites(), AZStd::chrono::milliseconds{500}); - - m_expectedTestTargetResult.emplace_back(TestImpact::TestRunResult::Failed); - m_expectedTestTargetResult.emplace_back(TestImpact::TestRunResult::Passed); - m_expectedTestTargetResult.emplace_back(TestImpact::TestRunResult::Passed); - m_expectedTestTargetResult.emplace_back(TestImpact::TestRunResult::Passed); - - for (const auto& testTarget : m_testTargetPaths) - { - m_testTargetJobArgs.emplace_back(GetRunCommandForTarget(testTarget)); - } - } - - // Fixture parameterized for different max number of concurrent jobs - class TestRunnerFixtureWithConcurrencyParams - : public TestRunnerFixture - , public ::testing::WithParamInterface - { - public: - void SetUp() override - { - TestRunnerFixture::SetUp(); - m_maxConcurrency = GetParam(); - } - }; - - using ConcurrencyAndJobExceptionPermutation = AZStd::tuple - < - size_t, // Max number of concurrent processes - JobExceptionPolicy // Test job exception policy - >; - - // Fixture parameterized for different max number of concurrent jobs and different job exception policies - class TestRunnerFixtureWithConcurrencyAndJobExceptionParams - : public TestRunnerFixture - , public ::testing::WithParamInterface - { - public: - void SetUp() override - { - TestRunnerFixture::SetUp(); - const auto& [maxConcurrency, jobExceptionPolicy] = GetParam(); - m_maxConcurrency = maxConcurrency; - m_jobExceptionPolicy = jobExceptionPolicy; - } - - protected: - JobExceptionPolicy m_jobExceptionPolicy = JobExceptionPolicy::Never; - }; - - class TestRunnerFixtureWithConcurrencyAndFailedToLaunchExceptionParams - : public TestRunnerFixtureWithConcurrencyAndJobExceptionParams - { - }; - - class TestRunnerFixtureWithConcurrencyAndExecutedWithFailureExceptionParams - : public TestRunnerFixtureWithConcurrencyAndJobExceptionParams - { - }; - - namespace - { - AZStd::array MaxConcurrentRuns = {1, 2, 3, 4}; - - AZStd::array FailedToLaunchExceptionPolicies = { - JobExceptionPolicy::Never, JobExceptionPolicy::OnFailedToExecute}; - - AZStd::array ExecutedWithFailureExceptionPolicies = { - JobExceptionPolicy::Never, JobExceptionPolicy::OnExecutedWithFailure}; - } // namespace - - TEST_P( - TestRunnerFixtureWithConcurrencyAndFailedToLaunchExceptionParams, - InvalidCommandArgument_ExpectJobResulFailedToExecuteeOrTestJobException) - { - // Given a test runner with no client callback or run timeout or runner timeout - m_testRunner = AZStd::make_unique(AZStd::nullopt, m_maxConcurrency, AZStd::nullopt, AZStd::nullopt); - - // Given a mixture of test run jobs with valid and invalid command arguments - for (size_t jobId = 0; jobId < m_testTargetJobArgs.size(); jobId++) - { - const AZStd::string args = (jobId % 2) ? InvalidProcessPath : m_testTargetJobArgs[jobId]; - JobData jobData(m_testTargetPaths[jobId].second); - m_jobInfos.emplace_back(JobInfo({jobId}, args, AZStd::move(jobData))); - } - - try - { - // When the test run jobs are executed with different exception policies - const auto runnerJobs = m_testRunner->RunTests(m_jobInfos, m_jobExceptionPolicy); - - // Expect this statement to be reachable only if no exception policy for launch failures - EXPECT_FALSE(::IsFlagSet(m_jobExceptionPolicy, JobExceptionPolicy::OnFailedToExecute)); - - for (const auto& job : runnerJobs) - { - const JobInfo::IdType jobId = job.GetJobInfo().GetId().m_value; - if (jobId % 2) - { - // Expect invalid jobs have a job result of FailedToExecute - ValidateJobFailedToExecute(job); - } - else - { - // Expect the valid jobs to successfully result in a test run that matches the expected test run data - ValidateTestRunCompleted(job, m_expectedTestTargetResult[jobId]); - ValidateTestTargetRun(job.GetPayload().value(), m_expectedTestTargetRuns[jobId]); - } - } - } - catch ([[maybe_unused]] const TestImpact::TestJobException& e) - { - // Expect this statement to be reachable only if there is an exception policy for launch failures - EXPECT_TRUE(::IsFlagSet(m_jobExceptionPolicy, JobExceptionPolicy::OnFailedToExecute)); - } - catch (...) - { - // Do not expect any other exceptions - FAIL(); - } - } - - TEST_P( - TestRunnerFixtureWithConcurrencyAndExecutedWithFailureExceptionParams, - ErroneousReturnCode_ExpectJobResultExecutedWithFailureOrTestJobException) - { - // Given a test runner with no client callback or run timeout or runner timeout - m_testRunner = AZStd::make_unique(AZStd::nullopt, m_maxConcurrency, AZStd::nullopt, AZStd::nullopt); - - // Given a mixture of test run jobs that execute and return either successfully or with failure - for (size_t jobId = 0; jobId < m_testTargetJobArgs.size(); jobId++) - { - JobData jobData(m_testTargetPaths[jobId].second); - m_jobInfos.emplace_back(JobInfo({jobId}, m_testTargetJobArgs[jobId], AZStd::move(jobData))); - } - - try - { - // When the test run jobs are executed with different exception policies - const auto runnerJobs = m_testRunner->RunTests(m_jobInfos, m_jobExceptionPolicy); - - // Expect this statement to be reachable only if no exception policy for jobs that return with error - EXPECT_FALSE(::IsFlagSet(m_jobExceptionPolicy, JobExceptionPolicy::OnExecutedWithFailure)); - - for (const auto& job : runnerJobs) - { - const JobInfo::IdType jobId = job.GetJobInfo().GetId().m_value; - - // Expect the valid jobs to successfully result in a test run that matches the expected test run data - ValidateTestRunCompleted(job, m_expectedTestTargetResult[jobId]); - ValidateTestTargetRun(job.GetPayload().value(), m_expectedTestTargetRuns[jobId]); - } - } - catch ([[maybe_unused]] const TestImpact::TestJobException& e) - { - // Expect this statement to be reachable only if there is an exception policy for jobs that return with error - EXPECT_TRUE(::IsFlagSet(m_jobExceptionPolicy, JobExceptionPolicy::OnExecutedWithFailure)); - } - catch ([[maybe_unused]] const TestImpact::Exception& e) - { - FAIL(); - } - catch (...) - { - // Do not expect any other exceptions - FAIL(); - } - } - - TEST_F(TestRunnerFixture, EmptyArtifact_ExpectTestRunnerException) - { - // Given a test runner with no client callback, concurrency, run timeout or runner timeout - m_testRunner = AZStd::make_unique(AZStd::nullopt, OneConcurrentProcess, AZStd::nullopt, AZStd::nullopt); - - // Given an test runner job that will return successfully but with an empty artifact string - m_jobInfos.emplace_back(JobInfo({0}, m_testTargetJobArgs[0], JobData(""))); - - try - { - // When the test runner job is executed - const auto runnerJobs = m_testRunner->RunTests(m_jobInfos, JobExceptionPolicy::Never); - - // Do not expect this statement to be reachable - FAIL(); - } - catch ([[maybe_unused]] const TestImpact::TestRunException& e) - { - // Expect an runner exception - SUCCEED(); - } - catch (...) - { - // Do not expect any other exceptions - FAIL(); - } - } - - TEST_F(TestRunnerFixture, InvalidRunArtifact_ExpectArtifactException) - { - // Given a test run artifact with invalid contents - WriteTextToFile("There is nothing valid here", m_testTargetPaths[TestTargetA].second); - - // Given a job command that will write the test run artifact to a different location that what we will read from - auto invalidRunArtifact = m_testTargetPaths[TestTargetA]; - invalidRunArtifact.second /= ".xml"; - const AZStd::string args = GetRunCommandForTarget(invalidRunArtifact); - - // Given a test runner with no client callback, concurrency, run timeout or runner timeout - m_testRunner = AZStd::make_unique(AZStd::nullopt, OneConcurrentProcess, AZStd::nullopt, AZStd::nullopt); - - // Given an test runner job that will return successfully but not produce an artifact - JobData jobData(m_testTargetPaths[TestTargetA].second); - m_jobInfos.emplace_back(JobInfo({TestTargetA}, args, AZStd::move(jobData))); - - try - { - // When the test runner job is executed - const auto runnerJobs = m_testRunner->RunTests(m_jobInfos, JobExceptionPolicy::Never); - - // Do not expect this statement to be reachable - FAIL(); - } - catch ([[maybe_unused]] const TestImpact::ArtifactException& e) - { - // Expect an runner exception - SUCCEED(); - } - catch (const TestImpact::Exception& e) - { - std::cout << e.what(); - FAIL(); - } - catch (...) - { - // Do not expect any other exceptions - FAIL(); - } - } - - TEST_P(TestRunnerFixtureWithConcurrencyParams, RunTestTargets_RunsMatchTestSuitesInTarget) - { - // Given a test runner with no client callback, runner timeout or runner timeout - m_testRunner = AZStd::make_unique(AZStd::nullopt, m_maxConcurrency, AZStd::nullopt, AZStd::nullopt); - - // Given an test runner job for each test target with no runner caching - for (size_t jobId = 0; jobId < m_testTargetJobArgs.size(); jobId++) - { - JobData jobData(m_testTargetPaths[jobId].second); - m_jobInfos.emplace_back(JobInfo({jobId}, m_testTargetJobArgs[jobId], AZStd::move(jobData))); - } - - // When the test runner jobs are executed - const auto runnerJobs = m_testRunner->RunTests(m_jobInfos, JobExceptionPolicy::Never); - - // Expect each job to successfully result in a test runner that matches the expected test runner data for that test target - for (const auto& job : runnerJobs) - { - const JobInfo::IdType jobId = job.GetJobInfo().GetId().m_value; - ValidateTestRunCompleted(job, m_expectedTestTargetResult[jobId]); - ValidateTestTargetRun(job.GetPayload().value(), m_expectedTestTargetRuns[jobId]); - } - } - - TEST_P(TestRunnerFixtureWithConcurrencyParams, RunTestTargetsWithArbitraryJobIds_RunsMatchTestSuitesInTarget) - { - // Given a set of arbitrary job ids to be used for the test target jobs - enum - { - ArbitraryA = 36, - ArbitraryB = 890, - ArbitraryC = 19, - ArbitraryD = 1 - }; - - const AZStd::unordered_map sequentialToArbitrary = - { - {TestTargetA, ArbitraryA}, - {TestTargetB, ArbitraryB}, - {TestTargetC, ArbitraryC}, - {TestTargetD, ArbitraryD}, - }; - - const AZStd::unordered_map arbitraryToSequential = - { - {ArbitraryA, TestTargetA}, - {ArbitraryB, TestTargetB}, - {ArbitraryC, TestTargetC}, - {ArbitraryD, TestTargetD}, - }; - - // Given a test runner with no client callback, run timeout or runner timeout - m_testRunner = AZStd::make_unique(AZStd::nullopt, m_maxConcurrency, AZStd::nullopt, AZStd::nullopt); - - // Given an test run job for each test target - for (size_t jobId = 0; jobId < m_testTargetJobArgs.size(); jobId++) - { - JobData jobData(m_testTargetPaths[jobId].second); - m_jobInfos.emplace_back(JobInfo({sequentialToArbitrary.at(jobId)}, m_testTargetJobArgs[jobId], AZStd::move(jobData))); - } - - // When the test run jobs are executed - const auto runnerJobs = m_testRunner->RunTests(m_jobInfos, JobExceptionPolicy::Never); - - // Expect each job to successfully result in a test run that matches the expected test run data for that test target - for (const auto& job : runnerJobs) - { - const JobInfo::IdType jobId = arbitraryToSequential.at(job.GetJobInfo().GetId().m_value); - ValidateTestRunCompleted(job, m_expectedTestTargetResult[jobId]); - ValidateTestTargetRun(job.GetPayload().value(), m_expectedTestTargetRuns[jobId]); - } - } - - TEST_P(TestRunnerFixtureWithConcurrencyParams, RunTestTargetsWithCallback_RunsMatchTestSuitesInTarget) - { - // Given a client callback function that tracks the number of successful runs - size_t numSuccesses = 0; - const auto jobCallback = - [&numSuccesses]([[maybe_unused]] const TestImpact::TestRunner::JobInfo& jobInfo, const TestImpact::JobMeta& meta) { - if (meta.m_result == TestImpact::JobResult::ExecutedWithSuccess) - { - numSuccesses++; - } - }; - - // Given a test runner with no run timeout or runner timeout - m_testRunner = AZStd::make_unique(jobCallback, m_maxConcurrency, AZStd::nullopt, AZStd::nullopt); - - // Given an test run job for each test target - for (size_t jobId = 0; jobId < m_testTargetJobArgs.size(); jobId++) - { - JobData jobData(m_testTargetPaths[jobId].second); - m_jobInfos.emplace_back(JobInfo({jobId}, m_testTargetJobArgs[jobId], AZStd::move(jobData))); - } - - // When the test run jobs are executed - const auto runnerJobs = m_testRunner->RunTests(m_jobInfos, JobExceptionPolicy::Never); - - // Expect the number of successful runs tracked in the callback to match the number of test targets run with no failures - EXPECT_EQ( - numSuccesses, - AZStd::count_if(m_expectedTestTargetResult.begin(), m_expectedTestTargetResult.end(), [](TestImpact::TestRunResult result) { - return result == TestImpact::TestRunResult::Passed; - })); - - // Expect each job to successfully result in a test run that matches the expected test run data for that test target - for (const auto& job : runnerJobs) - { - const JobInfo::IdType jobId = job.GetJobInfo().GetId().m_value; - ValidateTestRunCompleted(job, m_expectedTestTargetResult[jobId]); - ValidateTestTargetRun(job.GetPayload().value(), m_expectedTestTargetRuns[jobId]); - } - } - - TEST_P(TestRunnerFixtureWithConcurrencyParams, JobRunnerTimeout_InFlightJobsTimeoutAndQueuedJobsUnlaunched) - { - // Given a test runner with no client callback or runner timeout and 500ms run timeout - m_testRunner = - AZStd::make_unique(AZStd::nullopt, m_maxConcurrency, AZStd::chrono::milliseconds(500), AZStd::nullopt); - - // Given an test run job for each test target where half will sleep indefinitely - for (size_t jobId = 0; jobId < m_testTargetJobArgs.size(); jobId++) - { - JobData jobData(m_testTargetPaths[jobId].second); - const AZStd::string args = (jobId % 2) - ? AZStd::string::format("%s %s", ValidProcessPath, ConstructTestProcessArgs(jobId, LongSleep).c_str()) - : m_testTargetJobArgs[jobId]; - m_jobInfos.emplace_back(JobInfo({jobId}, args, AZStd::move(jobData))); - } - - // When the test run jobs are executed - const auto runnerJobs = m_testRunner->RunTests(m_jobInfos, JobExceptionPolicy::Never); - - // Expect half the jobs to successfully result in a test run that matches the expected test run data for that test target - // with the other half having timed out - for (const auto& job : runnerJobs) - { - const JobInfo::IdType jobId = job.GetJobInfo().GetId().m_value; - if (jobId % 2) - { - ValidateJobTimeout(job); - } - else - { - ValidateTestRunCompleted(job, m_expectedTestTargetResult[jobId]); - ValidateTestTargetRun(job.GetPayload().value(), m_expectedTestTargetRuns[jobId]); - } - } - } - - TEST_F(TestRunnerFixture, JobTimeout_InFlightJobTimeoutAndQueuedJobsUnlaunched) - { - // Given a test runner with no client callback or run timeout and a 5 second runner timeout - m_testRunner = AZStd::make_unique( - AZStd::nullopt, FourConcurrentProcesses, AZStd::nullopt, AZStd::chrono::milliseconds(5000)); - - // Given an test run job for each test target where half will sleep indefinitely - for (size_t jobId = 0; jobId < m_testTargetJobArgs.size(); jobId++) - { - JobData jobData(m_testTargetPaths[jobId].second); - const AZStd::string args = (jobId % 2) - ? AZStd::string::format("%s %s", ValidProcessPath, ConstructTestProcessArgs(jobId, LongSleep).c_str()) - : m_testTargetJobArgs[jobId]; - m_jobInfos.emplace_back(JobInfo({jobId}, args, AZStd::move(jobData))); - } - - // When the test run jobs are executed - const auto runnerJobs = m_testRunner->RunTests(m_jobInfos, JobExceptionPolicy::Never); - - // Expect half the jobs to successfully result in a test run that matches the expected test run data for that test target - // with the other half having timed out - for (const auto& job : runnerJobs) - { - const JobInfo::IdType jobId = job.GetJobInfo().GetId().m_value; - if (jobId % 2) - { - ValidateJobTimeout(job); - } - else - { - ValidateTestRunCompleted(job, m_expectedTestTargetResult[jobId]); - ValidateTestTargetRun(job.GetPayload().value(), m_expectedTestTargetRuns[jobId]); - } - } - } - - INSTANTIATE_TEST_CASE_P( - , - TestRunnerFixtureWithConcurrencyAndFailedToLaunchExceptionParams, - ::testing::Combine(::testing::ValuesIn(MaxConcurrentRuns), ::testing::ValuesIn(FailedToLaunchExceptionPolicies))); - - INSTANTIATE_TEST_CASE_P( - , - TestRunnerFixtureWithConcurrencyAndExecutedWithFailureExceptionParams, - ::testing::Combine(::testing::ValuesIn(MaxConcurrentRuns), ::testing::ValuesIn(ExecutedWithFailureExceptionPolicies))); - - INSTANTIATE_TEST_CASE_P(, TestRunnerFixtureWithConcurrencyParams, ::testing::ValuesIn(MaxConcurrentRuns)); -} // namespace UnitTest diff --git a/Code/Tools/TestImpactFramework/Runtime/Code/Tests/TestImpactFrameworkPathTest.cpp b/Code/Tools/TestImpactFramework/Runtime/Code/Tests/TestImpactFrameworkPathTest.cpp deleted file mode 100644 index 2982cb0551..0000000000 --- a/Code/Tools/TestImpactFramework/Runtime/Code/Tests/TestImpactFrameworkPathTest.cpp +++ /dev/null @@ -1,137 +0,0 @@ -/* - * All or portions of this file Copyright (c) Amazon.com, Inc. or its affiliates or - * its licensors. - * - * For complete copyright and license terms please see the LICENSE at the root of this - * distribution (the "License"). All use of this software is governed by the License, - * or, if provided, by the license below or the license accompanying this file. Do not - * remove or modify any license notices. This file is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * - */ - -#include -#include -#include - -namespace UnitTest -{ - class FrameworkPathTestFixture - : public AllocatorsTestFixture - { - public: - void SetUp() override - { - UnitTest::AllocatorsTestFixture::SetUp(); - - m_parentPathAbs = AZStd::string(AZStd::string("parent") + AZ_TRAIT_OS_PATH_SEPARATOR + "path"); - m_childPathRel = AZStd::string(AZStd::string("child") + AZ_TRAIT_OS_PATH_SEPARATOR + "path"); - m_childPathAbs = AZStd::string(m_parentPathAbs + AZ_TRAIT_OS_PATH_SEPARATOR + m_childPathRel); - - m_pathComponentA = AZStd::string("DirA"); - m_pathComponentB = AZStd::string("DirB"); - m_pathComponentC = AZStd::string("DirC"); - - m_posixPath = m_pathComponentA + AZ::IO::PosixPathSeparator + m_pathComponentB + AZ::IO::PosixPathSeparator + m_pathComponentC; - - m_windowsPath = - m_pathComponentA + AZ::IO::WindowsPathSeparator + m_pathComponentB + AZ::IO::WindowsPathSeparator + m_pathComponentC; - - m_mixedPath = - m_pathComponentA + AZ::IO::WindowsPathSeparator + m_pathComponentB + AZ::IO::PosixPathSeparator + m_pathComponentC; - - m_referredPath = - m_pathComponentA + AZ_TRAIT_OS_PATH_SEPARATOR + m_pathComponentB + AZ_TRAIT_OS_PATH_SEPARATOR + m_pathComponentC; - } - - void TearDown() override - { - UnitTest::AllocatorsTestFixture::TearDown(); - } - - AZStd::string m_parentPathAbs; - AZStd::string m_childPathRel; - AZStd::string m_childPathAbs; - - AZStd::string m_pathComponentA; - AZStd::string m_pathComponentB; - AZStd::string m_pathComponentC; - - AZ::IO::Path m_posixPath; - AZ::IO::Path m_windowsPath; - AZ::IO::Path m_mixedPath; - AZ::IO::Path m_referredPath; - }; - - TEST_F(FrameworkPathTestFixture, DefaultConstructor_HasEmptyAbsAndRelPaths) - { - // Given an empty framework path - TestImpact::FrameworkPath path; - - // Expect the absolute path to be empty - EXPECT_TRUE(path.Absolute().empty()); - - // Expect the relative path to be empty - EXPECT_TRUE(path.Relative().empty()); - } - - TEST_F(FrameworkPathTestFixture, OrphanConstructor_HasAbsAndEmptyRelPaths) - { - // Given an orhpan framework path - TestImpact::FrameworkPath path(m_parentPathAbs); - - // Expect the absolute path to be equal to the specified path - EXPECT_EQ(path.Absolute().String(), m_parentPathAbs); - - // Expect the relative path to be current directory symbol - EXPECT_STREQ(path.Relative().c_str(), "."); - } - - TEST_F(FrameworkPathTestFixture, ParentConstructor_HasAbsAndRelPaths) - { - // Given a child framework path - TestImpact::FrameworkPath path(m_childPathAbs, TestImpact::FrameworkPath(m_parentPathAbs)); - - // Expect the absolute path to be equal to the concatenation of the parent and child path - EXPECT_EQ(path.Absolute().String(), m_childPathAbs); - - // Expect the relative path to equal to the specified path - EXPECT_EQ(path.Relative().String(), m_childPathRel); - } - - TEST_F(FrameworkPathTestFixture, PosixSeperators_HasUniformPreferredSeperators) - { - // Given an orhpan framework path with Posix separators - TestImpact::FrameworkPath path(m_posixPath); - - // Expect the absolute path to be equal to the specified path with preferred separators - EXPECT_EQ(path.Absolute(), m_referredPath); - - // Expect the relative path to be current directory symbol - EXPECT_STREQ(path.Relative().c_str(), "."); - } - - TEST_F(FrameworkPathTestFixture, WindowsSeperators_HasUniformPreferredSeperators) - { - // Given an orhpan framework path with Windows separators - TestImpact::FrameworkPath path(m_windowsPath); - - // Expect the absolute path to be equal to the specified path with preferred separators - EXPECT_EQ(path.Absolute(), m_referredPath); - - // Expect the relative path to be current directory symbol - EXPECT_STREQ(path.Relative().c_str(), "."); - } - - TEST_F(FrameworkPathTestFixture, MixedSeperators_HasUniformPreferredSeperators) - { - // Given an orhpan framework path with mixed separators - TestImpact::FrameworkPath path(m_windowsPath); - - // Expect the absolute path to be equal to the specified path with preferred separators - EXPECT_EQ(path.Absolute(), m_referredPath); - - // Expect the relative path to be current directory symbol - EXPECT_STREQ(path.Relative().c_str(), "."); - } -} // namespace UnitTest diff --git a/Code/Tools/TestImpactFramework/Runtime/Code/Tests/TestImpactTestMain.cpp b/Code/Tools/TestImpactFramework/Runtime/Code/Tests/TestImpactTestMain.cpp deleted file mode 100644 index 632e84ad9f..0000000000 --- a/Code/Tools/TestImpactFramework/Runtime/Code/Tests/TestImpactTestMain.cpp +++ /dev/null @@ -1,33 +0,0 @@ -/* - * All or portions of this file Copyright (c) Amazon.com, Inc. or its affiliates or - * its licensors. - * - * For complete copyright and license terms please see the LICENSE at the root of this - * distribution (the "License"). All use of this software is governed by the License, - * or, if provided, by the license below or the license accompanying this file. Do not - * remove or modify any license notices. This file is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * - */ - -#include -#include -#include - -class TestImpactTestEnvironment : public AZ::Test::ITestEnvironment -{ -protected: - void SetupEnvironment() override - { - AZ::AllocatorInstance::Create(); - AZ::AllocatorInstance::Create(); - } - - void TeardownEnvironment() override - { - AZ::AllocatorInstance::Destroy(); - AZ::AllocatorInstance::Destroy(); - } -}; - -AZ_UNIT_TEST_HOOK(new TestImpactTestEnvironment); diff --git a/cmake/LYTestWrappers.cmake b/cmake/LYTestWrappers.cmake index 2a65929cf6..40669f7426 100644 --- a/cmake/LYTestWrappers.cmake +++ b/cmake/LYTestWrappers.cmake @@ -151,12 +151,6 @@ function(ly_add_test) set(LY_ADDED_TEST_NAME ${qualified_test_run_name_with_suite}::TEST_RUN) set(LY_ADDED_TEST_NAME ${LY_ADDED_TEST_NAME} PARENT_SCOPE) - # Store the test so we can walk through all of them in LYTestImpactFramework.cmake - set_property(GLOBAL APPEND PROPERTY LY_ALL_TESTS ${LY_ADDED_TEST_NAME}) - set_property(GLOBAL APPEND PROPERTY LY_ALL_TESTS_${LY_ADDED_TEST_NAME}_TEST_NAME ${ly_add_test_NAME}) - set_property(GLOBAL APPEND PROPERTY LY_ALL_TESTS_${LY_ADDED_TEST_NAME}_TEST_SUITE ${ly_add_test_TEST_SUITE}) - set_property(GLOBAL APPEND PROPERTY LY_ALL_TESTS_${LY_ADDED_TEST_NAME}_TEST_LIBRARY ${ly_add_test_TEST_LIBRARY}) - set(final_labels SUITE_${ly_add_test_TEST_SUITE}) if (ly_add_test_TEST_REQUIRES) @@ -247,6 +241,12 @@ function(ly_add_test) endif() + # Store the test so we can walk through all of them in LYTestImpactFramework.cmake + set_property(GLOBAL APPEND PROPERTY LY_ALL_TESTS ${ly_add_test_NAME}) + set_property(GLOBAL APPEND PROPERTY LY_ALL_TESTS_${ly_add_test_NAME}_TEST_SUITE ${ly_add_test_TEST_SUITE}) + set_property(GLOBAL APPEND PROPERTY LY_ALL_TESTS_${ly_add_test_NAME}_TEST_LIBRARY ${ly_add_test_TEST_LIBRARY}) + set_property(GLOBAL APPEND PROPERTY LY_ALL_TESTS_${ly_add_test_NAME}_TEST_TIMEOUT ${ly_add_test_TIMEOUT}) + endfunction() #! ly_add_pytest: registers target PyTest-based test with CTest @@ -298,8 +298,8 @@ function(ly_add_pytest) ${ly_add_pytest_UNPARSED_ARGUMENTS} ) + set_property(GLOBAL APPEND PROPERTY LY_ALL_TESTS_${ly_add_pytest_NAME}_SCRIPT_PATH ${ly_add_pytest_PATH}) set_tests_properties(${LY_ADDED_TEST_NAME} PROPERTIES RUN_SERIAL "${ly_add_pytest_TEST_SERIAL}") - set_property(GLOBAL APPEND PROPERTY LY_ALL_TESTS_${LY_ADDED_TEST_NAME}_SCRIPT_PATH ${ly_add_pytest_PATH}) endfunction() #! ly_add_editor_python_test: registers target Editor Python Bindings test with CTest @@ -363,8 +363,8 @@ function(ly_add_editor_python_test) COMPONENT ${ly_add_editor_python_test_COMPONENT} ) + set_property(GLOBAL APPEND PROPERTY LY_ALL_TESTS_${ly_add_editor_python_test_NAME}_SCRIPT_PATH ${ly_add_editor_python_test_PATH}) set_tests_properties(${LY_ADDED_TEST_NAME} PROPERTIES RUN_SERIAL "${ly_add_editor_python_test_TEST_SERIAL}") - set_property(GLOBAL APPEND PROPERTY LY_ALL_TESTS_${LY_ADDED_TEST_NAME}_SCRIPT_PATH ${ly_add_editor_python_test_PATH}) endfunction() #! ly_add_googletest: Adds a new RUN_TEST using for the specified target using the supplied command or fallback to running @@ -424,8 +424,14 @@ function(ly_add_googletest) set(full_test_command $ $ AzRunUnitTests) # Add AzTestRunner as a build dependency ly_add_dependencies(${build_target} AZ::AzTestRunner) + # Ideally, we would populate the full command procedurally but the generator expressions won't be expanded by the time we need this data + set_property(GLOBAL APPEND PROPERTY LY_ALL_TESTS_${ly_add_googletest_NAME}_TEST_COMMAND "AzRunUnitTests") else() set(full_test_command ${ly_add_googletest_TEST_COMMAND}) + # Remove the generator expressions so we are left with the argument(s) required to run unit tests for executable targets + string(REPLACE ";" "" stripped_test_command ${full_test_command}) + string(GENEX_STRIP ${stripped_test_command} stripped_test_command) + set_property(GLOBAL APPEND PROPERTY LY_ALL_TESTS_${ly_add_googletest_NAME}_TEST_COMMAND ${stripped_test_command}) endif() string(REPLACE "::" "_" report_directory "${GTEST_XML_OUTPUT_DIR}/${ly_add_googletest_NAME}.xml") @@ -507,8 +513,14 @@ function(ly_add_googlebenchmark) # If command is not supplied attempts, uses the AzTestRunner to run googlebenchmarks on the supplied TARGET set(full_test_command $ $ AzRunBenchmarks ${output_format_args}) + # Ideally, we would populate the full command procedurally but the generator expressions won't be expanded by the time we need this data + set_property(GLOBAL APPEND PROPERTY LY_ALL_TESTS_${ly_add_googlebenchmark_NAME}_TEST_COMMAND "AzRunUnitTests") else() set(full_test_command ${ly_add_googlebenchmark_TEST_COMMAND}) + # Remove the generator expressions so we are left with the argument(s) required to run unit tests for executable targets + string(REPLACE ";" "" stripped_test_command ${full_test_command}) + string(GENEX_STRIP ${stripped_test_command} stripped_test_command) + set_property(GLOBAL APPEND PROPERTY LY_ALL_TESTS_${ly_add_googletest_NAME}_TEST_COMMAND ${stripped_test_command}) endif() ly_add_test( @@ -524,6 +536,5 @@ function(ly_add_googlebenchmark) AZ::AzTestRunner COMPONENT ${ly_add_googlebenchmark_COMPONENT} ) - endfunction() diff --git a/cmake/TestImpactFramework/ConsoleFrontendConfig.in b/cmake/TestImpactFramework/ConsoleFrontendConfig.in index 5f3908671e..f6af3d801d 100644 --- a/cmake/TestImpactFramework/ConsoleFrontendConfig.in +++ b/cmake/TestImpactFramework/ConsoleFrontendConfig.in @@ -1,111 +1,188 @@ -[path.configuration] -repo_dir = "${repo_dir}" -working_dir = "${working_dir}" -bin_dir = "${runtime_bin_dir}" -tests_dir = "${tests_dir}" -temp_dir = "${temp_dir}" -target_mappings_dir = "${source_target_mapping_dir}" -test_type_dir = "${test_type_dir}" -dependencies_dir = "${target_dependency_dir}" - -[sourcetree.configuration.filters.autogen] -# E.g. matches input /Foo/{Bar}.FooBar.xml with output /Baz/{Bar}.BazBar.cpp -input_output_pairer = "(.*)\\..*" -[sourcetree.configuration.filters.autogen.input] -exclude_filter = [".jinja"] -[sourcetree.configuration.filters.source] -exclude_filter = [".cmake"] -[sourcetree.configuration.testtype.enumerated] -file = "All.tests" -# The table to read from the test enumeration file that contains the test targets -target_table = "google.test" -[sourcetree.configuration.dependency] -# E.g. matches WhiteBox.Editor.Static\n(Gem::WhiteBox.Editor.Static) or WhiteBox.Editor.Static -target_dependency_file_matcher = "target\\.(.*)\\.(dependers)?" -# E.g. matches target.WhiteBox.Editor.Static (for dependency) target.WhiteBox.Editor.Static.dependers (for dependers) -target_vertex_matcher = "(?:(.*)\\n|(.*)" - -[spartia.configuration] -test_impact_Data_file = "TestImpactData.spartia" -test_run_coverage_file = "{test_dir}\\{test_target}.coverage.xml" -test_run_results_file = "{test_dir}\\{test_target}.results.xml" -test_enumeration_file = "{temp_dir}\\{test_target}.enum" -test_shard_selection_file = "{temp_dir}\\{test_target}.filter.{shard_id}" -exclude_filter = [ -{ target = "AssetBundler.Tests", tests = ["*"] }, -{ target = "AssetProcessor.Tests", tests = ["*"] }, -{ target = "CryRenderD3D11.Tests", tests = ["*"] }, -{ target = "CryRenderD3D12.Tests", tests = ["*"] }, -{ target = "LyzardApplicationDescriptors.Tests", tests = ["*"] }, -{ target = "EMotionFX.Editor.Tests", tests = ["UIFixture.*", "SimulatedObjectModelTestsFixture.*", "TestParametersFixture.*", "CanSeeJointsFixture.*", "LODSkinnedMeshFixtureTests/LODSkinnedMeshFixture.CheckLODLevels/*"] }, -{ target = "EMotionFX.Tests", tests = ["UIFixture.*", "SimulatedObjectModelTestsFixture.*", "TestParametersFixture.*", "CanSeeJointsFixture.*"] }, -{ target = "AzCore.Tests", tests = ["AllocatorsTestFixtureLeakDetectionDeathTest_SKIPCODECOVERAGE.AllocatorLeak"] }, -] -[spartia.configuration.shard] -# Long tests that will be sharded -include_filter = [ -{ target = "AzCore.Tests", policy = "fixture_contiguous" }, -{ target = "AzToolsFramework.Tests", policy = "fixture_contiguous" }, -{ target = "Framework.Tests", policy = "test_interleaved" }, -{ target = "LmbrCentral.Editor.Tests", policy = "test_interleaved" }, -{ target = "EditorLib.Tests", policy = "test_interleaved" }, -{ target = "PhysX.Tests", policy = "test_interleaved" }, -{ target = "ImageProcessing.Tests", policy = "test_interleaved" }, -{ target = "Atom_RPI.Tests", policy = "test_interleaved" }, -{ target = "Atom_RHI.Tests", policy = "test_interleaved" }, -{ target = "AzManipulatorFramework.Tests", policy = "test_interleaved" }, -{ target = "WhiteBox.Editor.Tests", policy = "test_interleaved" }, -{ target = "AzManipulatorTestFramework.Tests", policy = "test_interleaved" }, -{ target = "AtomCore.Tests", policy = "test_interleaved" }, -{ target = "ImageProcessingAtom.Editor.Tests", policy = "test_interleaved" }, -{ target = "EditorPythonBindings.Tests", policy = "test_interleaved" }, -{ target = "Atom_Utils.Tests", policy = "test_interleaved" }, -{ target = "AudioEngineWwise.Editor.Tests", policy = "test_interleaved" }, -{ target = "Multiplayer.Tests", policy = "test_interleaved" }, -{ target = "LmbrCentral.Tests", policy = "test_interleaved" }, -{ target = "LyMetricsShared.Tests", policy = "fixture_contiguous" }, -{ target = "PhysX.Editor.Tests", policy = "test_interleaved" }, -{ target = "ComponentEntityEditorPlugin.Tests", policy = "test_interleaved" }, -{ target = "DeltaCataloger.Tests", policy = "test_interleaved" }, -{ target = "GradientSignal.Tests", policy = "test_interleaved" }, -{ target = "LyShine.Tests", policy = "test_interleaved" }, -{ target = "EMotionFX.Editor.Tests", policy = "test_interleaved" }, -{ target = "EMotionFX.Tests", policy = "test_interleaved" }, -{ target = "CrySystem.Tests", policy = "test_interleaved" }, -] -[spartia.configuration.instrumentation] -abs_bin = "${instrumentation_bin}" -[spartia.configuration.instrumentation.errors] -# AzCppCoverage error codes -incorrect_args = -1618178468 -[spartia.configuration.instrumentation.test_coverage] -args = "--export_type cobertura:\"{test_run_coverage_file}\"" -[spartia.configuration.instrumentation.test_selection] -args = "--gtest_filter={test_selection}" -[spartia.configuration.instrumentation.test_enumeration] -args = "--gtest_list_tests" -[spartia.configuration.instrumentation.test_results] -args = "--gtest_output=xml:\"{test_run_results_file}\"" -[spartia.configuration.instrumentation.test_results.errors] -test_success = 0 -test_failures = 1 -[spartia.configuration.instrumentation.binary_type.dynlib] -abs_bin = "{bin_dir}\\AzTestRunner.exe" -args = "\"{bin_dir}\\{test_target}.dll\" AzRunUnitTests" -[spartia.configuration.instrumentation.binary_type.dynlib.test_enumeration] -args = "--stdout_to_file \"{test_enumeration_file}\" {test_enumeration}" -[spartia.configuration.instrumentation.binary_type.dynlib.test_shard_selection] -args = "--args_from_file \"{test_shard_selection_file}\"" -[spartia.configuration.instrumentation.binary_type.dynlib.errors] -# AzTestRunner error codes -failed_to_find_target_bin = 102 -incorrect_args = 101 -known_errors = [ 103, 104] -[spartia.configuration.instrumentation.binary_type.executable] -abs_bin = "{bin_dir}\\{test_target}.exe" -[spartia.configuration.instrumentation.binary_type.executable.test_enumeration] -args = "--stdout_to_file \"{test_enumeration_file}\" {test_enumeration}" -[spartia.configuration.instrumentation.binary_type.executable.test_shard_selection] -args = "--args_from_file \"{test_shard_selection_file}\"" -[spartia.configuration.test_run.seed] -instrumentation_args = "--modules \"{bin_dir}\" --excluded_modules \"{binary_type.dynlib.abs_bin}\" --sources \"{repo_dir}\" --no_breakpoints {test_coverage} -- " \ No newline at end of file +{ + "meta": { + "platform": "${platform}", + "timestamp": "${timestamp}" + }, + "repo": { + "root": "${repo_dir}" + }, + "workspace": { + "temp": { + "root": "${temp_dir}", + "relative_paths": { + "artifact_dir": "RuntimeArtifact" + } + }, + "persistent": { + "root": "${persistent_dir}", + "relative_paths": { + "test_impact_data_file": "TestImpactData.spartia", + "enumeration_cache_dir": "EnumerationCache" + } + } + }, + "artifacts": { + "static": { + "build_target_descriptor": { + "dir": "${source_target_mapping_dir}", + "target_sources": { + "static": { + "include_filters": [ + ".h", ".hpp", ".hxx", ".inl", ".c", ".cpp", ".cxx" + ] + }, + "autogen": { + "input_output_pairer": "(.*)\\..*", + "input": { + "include_filters": [ + ".xml" + ] + } + } + } + }, + "dependency_graph_data": { + "dir": "${target_dependency_dir}", + "matchers": { + "target_dependency_file": "target\\.(.*)\\.(dependers)?", + "target_vertex": "(?:(.*)\\n|(.*)" + } + }, + "test_target_meta": { + "file": "${test_target_type_file}" + } + } + }, + "test_engine": { + "test_runner": { + "bin": "${test_runner_bin}" + }, + "instrumentation": { + "bin": "${instrumentation_bin}" + } + }, + "target": { + "dir": "${bin_dir}", + "exclude": [ + + ], + "shard": [ + { + "policy": "fixture_contiguous", + "target": "AzCore.Tests" + }, + { + "policy": "fixture_contiguous", + "target": "AzToolsFramework.Tests" + }, + { + "policy": "test_interleaved", + "target": "Framework.Tests" + }, + { + "policy": "test_interleaved", + "target": "LmbrCentral.Editor.Tests" + }, + { + "policy": "test_interleaved", + "target": "EditorLib.Tests" + }, + { + "policy": "test_interleaved", + "target": "PhysX.Tests" + }, + { + "policy": "test_interleaved", + "target": "ImageProcessing.Tests" + }, + { + "policy": "test_interleaved", + "target": "Atom_RPI.Tests" + }, + { + "policy": "test_interleaved", + "target": "Atom_RHI.Tests" + }, + { + "policy": "test_interleaved", + "target": "AzManipulatorFramework.Tests" + }, + { + "policy": "test_interleaved", + "target": "WhiteBox.Editor.Tests" + }, + { + "policy": "test_interleaved", + "target": "ImageProcessing.Tests" + }, + { + "policy": "test_interleaved", + "target": "AzManipulatorTestFramework.Tests" + }, + { + "policy": "test_interleaved", + "target": "AtomCore.Tests" + }, + { + "policy": "test_interleaved", + "target": "ImageProcessingAtom.Editor.Tests" + }, + { + "policy": "test_interleaved", + "target": "EditorPythonBindings.Tests" + }, + { + "policy": "test_interleaved", + "target": "Atom_Utils.Tests" + }, + { + "policy": "test_interleaved", + "target": "AudioEngineWwise.Editor.Tests" + }, + { + "policy": "test_interleaved", + "target": "Multiplayer.Tests" + }, + { + "policy": "test_interleaved", + "target": "LmbrCentral.Tests" + }, + { + "policy": "fixture_contiguous", + "target": "LyMetricsShared.Tests" + }, + { + "policy": "test_interleaved", + "target": "PhysX.Editor.Tests" + }, + { + "policy": "test_interleaved", + "target": "ComponentEntityEditorPlugin.Tests" + }, + { + "policy": "test_interleaved", + "target": "DeltaCataloger.Tests" + }, + { + "policy": "test_interleaved", + "target": "GradientSignal.Tests" + }, + { + "policy": "test_interleaved", + "target": "LyShine.Tests" + }, + { + "policy": "test_interleaved", + "target": "EMotionFX.Editor.Tests" + }, + { + "policy": "test_interleaved", + "target": "EMotionFX.Tests" + }, + { + "policy": "test_interleaved", + "target": "CrySystem.Tests" + } + ] + } +} diff --git a/cmake/TestImpactFramework/LYTestImpactFramework.cmake b/cmake/TestImpactFramework/LYTestImpactFramework.cmake index c08c64db80..c85b5ff1e9 100644 --- a/cmake/TestImpactFramework/LYTestImpactFramework.cmake +++ b/cmake/TestImpactFramework/LYTestImpactFramework.cmake @@ -15,6 +15,9 @@ option(LY_TEST_IMPACT_ACTIVE "Enable test impact framework" OFF) # Path to test instrumentation binary option(LY_TEST_IMPACT_INSTRUMENTATION_BIN "Path to test impact framework instrumentation binary" OFF) +# Name of test impact framework console static library target +set(LY_TEST_IMPACT_CONSOLE_STATIC_TARGET "TestImpact.Frontend.Console.Static") + # Name of test impact framework console target set(LY_TEST_IMPACT_CONSOLE_TARGET "TestImpact.Frontend.Console") @@ -33,11 +36,8 @@ set(LY_TEST_IMPACT_SOURCE_TARGET_MAPPING_DIR "${LY_TEST_IMPACT_ARTIFACT_DIR}/Map # Directory for build target dependency/depender graphs set(LY_TEST_IMPACT_TARGET_DEPENDENCY_DIR "${LY_TEST_IMPACT_ARTIFACT_DIR}/Dependency") -# Directory for test type enumeration files -set(LY_TEST_IMPACT_TEST_TYPE_DIR "${LY_TEST_IMPACT_ARTIFACT_DIR}/TestType") - # Master test enumeration file for all test types -set(LY_TEST_IMPACT_TEST_TYPE_FILE "${LY_TEST_IMPACT_TEST_TYPE_DIR}/All.tests") +set(LY_TEST_IMPACT_TEST_TYPE_FILE "${LY_TEST_IMPACT_ARTIFACT_DIR}/TestType/All.tests") #! ly_test_impact_rebase_file_to_repo_root: rebases the relative and/or absolute path to be relative to repo root directory and places the resulting path in quotes. # @@ -93,14 +93,13 @@ function(ly_test_impact_get_test_launch_method TARGET_NAME LAUNCH_METHOD) endif() endfunction() -#! ly_test_impact_extract_google_test: explodes a composite google test string into namespace, test and suite components. +#! ly_test_impact_extract_google_test_name: extracts the google test name from the composite 'namespace::test_name' string # # \arg:COMPOSITE_TEST test in the form 'namespace::test' -# \arg:TEST_NAMESPACE namespace for the test # \arg:TEST_NAME name of test function(ly_test_impact_extract_google_test COMPOSITE_TEST TEST_NAMESPACE TEST_NAME) get_property(test_components GLOBAL PROPERTY LY_ALL_TESTS_${COMPOSITE_TEST}_TEST_NAME) - # Namespace and test are mandetiry + # Namespace and test are mandatory string(REPLACE "::" ";" test_components ${test_components}) list(LENGTH test_components num_test_components) if(num_test_components LESS 2) @@ -113,37 +112,72 @@ function(ly_test_impact_extract_google_test COMPOSITE_TEST TEST_NAMESPACE TEST_N set(${TEST_NAME} ${test_name} PARENT_SCOPE) endfunction() -#! ly_test_impact_extract_python_test: explodes a composite python test string into filename, namespace, test and suite components. +#! ly_test_impact_extract_python_test_name: extracts the python test name from the composite 'namespace::test_name' string # # \arg:COMPOSITE_TEST test in form 'namespace::test' or 'test' -# \arg:TEST_NAMESPACE namespace for the test (optional) # \arg:TEST_NAME name of test -# \arg:TEST_FILE the Python script path for this test -function(ly_test_impact_extract_python_test COMPOSITE_TEST TEST_NAMESPACE TEST_NAME TEST_FILE) +function(ly_test_impact_extract_python_test COMPOSITE_TEST TEST_NAME) get_property(test_components GLOBAL PROPERTY LY_ALL_TESTS_${COMPOSITE_TEST}_TEST_NAME) - get_property(test_file GLOBAL PROPERTY LY_ALL_TESTS_${COMPOSITE_TEST}_SCRIPT_PATH) # namespace is optional, in which case this component will be simply the test name string(REPLACE "::" ";" test_components ${test_components}) list(LENGTH test_components num_test_components) if(num_test_components GREATER 1) - list(GET test_components 0 test_namespace) list(GET test_components 1 test_name) else() - set(test_namespace "") + set(test_name ${test_components}) + endif() + + set(${TEST_NAME} ${test_name} PARENT_SCOPE) +endfunction() + +#! ly_test_impact_extract_google_test_params: extracts the google test name and command parameters. +# +# \arg:COMPOSITE_TEST test in the form 'namespace::test' +# \arg:TEST_NAME name of test +# \arg:TEST_COMMAND optional command arguments to run the test +function(ly_test_impact_extract_google_test_params COMPOSITE_TEST TEST_NAME TEST_COMMAND) + get_property(test_command GLOBAL PROPERTY LY_ALL_TESTS_${COMPOSITE_TEST}_TEST_COMMAND) + # Namespace and test are mandatory + string(REPLACE "::" ";" test_components ${COMPOSITE_TEST}) + list(LENGTH test_components num_test_components) + if(num_test_components LESS 2) + message(FATAL_ERROR "The test ${test_components} appears to have been specified without a namespace, i.e.:\ly_add_googletest/benchmark(NAME ${test_components})\nInstead of (perhaps):\ly_add_googletest/benchmark(NAME Gem::${test_components})\nPlease add the missing namespace before proceeding.") + endif() + + list(GET test_components 0 test_namespace) + list(GET test_components 1 test_name) + set(${TEST_NAMESPACE} ${test_namespace} PARENT_SCOPE) + set(${TEST_NAME} ${test_name} PARENT_SCOPE) + set(${TEST_COMMAND} ${test_command} PARENT_SCOPE) +endfunction() + +#! ly_test_impact_extract_python_test_params: extracts the python test name and relative script path parameters. +# +# \arg:COMPOSITE_TEST test in form 'namespace::test' or 'test' +# \arg:TEST_NAME name of test +# \arg:SCRIPT_PATH name of test +function(ly_test_impact_extract_python_test_params COMPOSITE_TEST TEST_NAME SCRIPT_PATH) + get_property(script_path GLOBAL PROPERTY LY_ALL_TESTS_${COMPOSITE_TEST}_SCRIPT_PATH) + + # namespace is optional, in which case this component will be simply the test name + string(REPLACE "::" ";" test_components ${COMPOSITE_TEST}) + list(LENGTH test_components num_test_components) + if(num_test_components GREATER 1) + list(GET test_components 1 test_name) + else() set(test_name ${test_components}) endif() # Get python script path relative to repo root ly_test_impact_rebase_file_to_repo_root( - ${test_file} - test_file + ${script_path} + script_path ${LY_ROOT_FOLDER} ) - set(${TEST_NAMESPACE} ${test_namespace} PARENT_SCOPE) set(${TEST_NAME} ${test_name} PARENT_SCOPE) - set(${TEST_FILE} ${test_file} PARENT_SCOPE) + set(${SCRIPT_PATH} ${script_path} PARENT_SCOPE) endfunction() #! ly_test_impact_write_test_enumeration_file: exports the master test lists to file. @@ -164,25 +198,26 @@ function(ly_test_impact_write_test_enumeration_file TEST_ENUMERATION_TEMPLATE_FI message(TRACE "Parsing ${test}") get_property(test_type GLOBAL PROPERTY LY_ALL_TESTS_${test}_TEST_LIBRARY) get_property(test_suite GLOBAL PROPERTY LY_ALL_TESTS_${test}_TEST_SUITE) + get_property(test_timeout GLOBAL PROPERTY LY_ALL_TESTS_${test}_TEST_TIMEOUT) if("${test_type}" STREQUAL "pytest") # Python tests - ly_test_impact_extract_python_test(${test} test_namespace test_name test_file) - list(APPEND python_tests " { \"name\": \"${test_name}\", \"namespace\": \"${test_namespace}\", \"suite\": \"${test_suite}\", \"path\": \"${test_file}\" }") + ly_test_impact_extract_python_test_params(${test} test_name script_path) + list(APPEND python_tests " { \"name\": \"${test_name}\", \"suite\": \"${test_suite}\", \"script\": \"${script_path}\", \"timeout\":${test_timeout} }") elseif("${test_type}" STREQUAL "pytest_editor") # Python editor tests - ly_test_impact_extract_python_test(${test} test_namespace test_name test_file) - list(APPEND python_editor_tests " { \"name\": \"${test_name}\", \"namespace\": \"${test_namespace}\", \"suite\": \"${test_suite}\", \"path\": \"${test_file}\" }") + ly_test_impact_extract_python_test_params(${test} test_name script_path) + list(APPEND python_editor_tests " { \"name\": \"${test_name}\", \"suite\": \"${test_suite}\", \"script\": \"${script_path}\", \"timeout\":${test_timeout} }") elseif("${test_type}" STREQUAL "googletest") # Google tests - ly_test_impact_extract_google_test(${test} test_namespace test_name) + ly_test_impact_extract_google_test_params(${test} test_name test_command) ly_test_impact_get_test_launch_method(${test_name} launch_method) - list(APPEND google_tests " { \"name\": \"${test_name}\", \"namespace\": \"${test_namespace}\", \"suite\": \"${test_suite}\", \"launch_method\": \"${launch_method}\" }") + list(APPEND google_tests " { \"name\": \"${test_name}\", \"suite\": \"${test_suite}\", \"command\": \"${test_command}\", \"timeout\":${test_timeout}, \"launch_method\": \"${launch_method}\" }") elseif("${test_type}" STREQUAL "googlebenchmark") # Google benchmarks - ly_test_impact_extract_google_test(${test} test_namespace test_name) - list(APPEND google_benchmarks " { \"name\": \"${test_name}\", \"namespace\": \"${test_namespace}\", \"suite\": \"${test_suite}\" }") + ly_test_impact_extract_google_test_params(${test} test_name test_command) + list(APPEND google_benchmarks " { \"name\": \"${test_name}\", \"suite\": \"${test_suite}\", \"command\": \"${test_command}\", \"timeout\":${test_timeout} }") else() - message("${test} is of unknown type (TEST_LIBRARY property is empty)") + message("${test_name} is of unknown type (TEST_LIBRARY property is empty)") list(APPEND unknown_tests " { \"name\": \"${test}\" }") endif() endforeach() @@ -242,12 +277,7 @@ function(ly_test_impact_export_source_target_mappings MAPPING_TEMPLATE_FILE) endif() # Static source file mappings - get_target_property(target_type ${target} TYPE) - if("${target_type}" STREQUAL "INTERFACE_LIBRARY") - get_target_property(static_sources ${target}_HEADERS SOURCES) - else() - get_target_property(static_sources ${target} SOURCES) - endif() + get_target_property(static_sources ${target} SOURCES) # Rebase static source files to repo root ly_test_impact_rebase_files_to_repo_root( @@ -271,75 +301,52 @@ endfunction() # # \arg:CONFIG_TEMPLATE_FILE path to the runtime configuration template file # \arg:PERSISTENT_DATA_DIR path to the test impact framework persistent data directory -# \arg:RUNTIME_BIN_DIR path to repo binary ourput directory -function(ly_test_impact_write_config_file CONFIG_TEMPLATE_FILE PERSISTENT_DATA_DIR RUNTIME_BIN_DIR) - set(repo_dir ${LY_ROOT_FOLDER}) +# \arg:BIN_DIR path to repo binary output directory +function(ly_test_impact_write_config_file CONFIG_TEMPLATE_FILE PERSISTENT_DATA_DIR BIN_DIR) + # Platform this config file is being generated for + set(platform ${PAL_PLATFORM_NAME}) - # SparTIA instrumentation binary + # Timestamp this config file was generated at + string(TIMESTAMP timestamp "%Y-%m-%d %H:%M:%S") + + # Instrumentation binary if(NOT LY_TEST_IMPACT_INSTRUMENTATION_BIN) message(FATAL_ERROR "No test impact framework instrumentation binary was specified, please provide the path with option LY_TEST_IMPACT_INSTRUMENTATION_BIN") endif() file(TO_CMAKE_PATH ${LY_TEST_IMPACT_INSTRUMENTATION_BIN} instrumentation_bin) - - # test impact framework working dir - ly_test_impact_rebase_file_to_repo_root( - ${LY_TEST_IMPACT_WORKING_DIR} - working_dir - ${LY_ROOT_FOLDER} - ) - - # test impact framework console binary dir - ly_test_impact_rebase_file_to_repo_root( - ${RUNTIME_BIN_DIR} - runtime_bin_dir - ${LY_ROOT_FOLDER} - ) - # Test dir - ly_test_impact_rebase_file_to_repo_root( - "${PERSISTENT_DATA_DIR}/Tests" - tests_dir - ${LY_ROOT_FOLDER} - ) + # Testrunner binary + set(test_runner_bin $) + + # Repository root + set(repo_dir ${LY_ROOT_FOLDER}) + + # Test impact framework output binary dir + set(bin_dir ${BIN_DIR}) # Temp dir - ly_test_impact_rebase_file_to_repo_root( - "${LY_TEST_IMPACT_TEMP_DIR}" - temp_dir - ${LY_ROOT_FOLDER} - ) - + set(temp_dir "${LY_TEST_IMPACT_TEMP_DIR}") + + # Persistent dir + set(persistent_dir "${PERSISTENT_DATA_DIR}") + # Source to target mappings dir - ly_test_impact_rebase_file_to_repo_root( - "${LY_TEST_IMPACT_SOURCE_TARGET_MAPPING_DIR}" - source_target_mapping_dir - ${LY_ROOT_FOLDER} - ) + set(source_target_mapping_dir "${LY_TEST_IMPACT_SOURCE_TARGET_MAPPING_DIR}") - # Test type artifact dir - ly_test_impact_rebase_file_to_repo_root( - "${LY_TEST_IMPACT_TEST_TYPE_DIR}" - test_type_dir - ${LY_ROOT_FOLDER} - ) + # Test type artifact file + set(test_target_type_file "${LY_TEST_IMPACT_TEST_TYPE_FILE}") # Build dependency artifact dir - ly_test_impact_rebase_file_to_repo_root( - "${LY_TEST_IMPACT_TARGET_DEPENDENCY_DIR}" - target_dependency_dir - ${LY_ROOT_FOLDER} - ) + set(target_dependency_dir "${LY_TEST_IMPACT_TARGET_DEPENDENCY_DIR}") # Substitute config file template with above vars file(READ "${CONFIG_TEMPLATE_FILE}" config_file) string(CONFIGURE ${config_file} config_file) # Write out entire config contents to a file in the build directory of the test impact framework console target - string(TIMESTAMP timestamp "%Y-%m-%d %H:%M:%S") - set(header "# Test Impact Framework configuration file for Lumberyard\n# Platform: ${CMAKE_SYSTEM_NAME}\n# Build: $\n# ${timestamp}") file(GENERATE - OUTPUT "${CMAKE_RUNTIME_OUTPUT_DIRECTORY}/$/$.$.cfg" - CONTENT "${header}\n\n${config_file}" + OUTPUT "${CMAKE_RUNTIME_OUTPUT_DIRECTORY}/$/$.$.json" + CONTENT ${config_file} ) endfunction() @@ -353,7 +360,7 @@ function(ly_test_impact_post_step) set(persistent_data_dir "${LY_ROOT_FOLDER}/Tests/test_impact_framework/${CMAKE_SYSTEM_NAME}/$") # Directory for binaries built for this profile - set(runtime_bin_dir "${CMAKE_RUNTIME_OUTPUT_DIRECTORY}/$") + set(bin_dir "${CMAKE_RUNTIME_OUTPUT_DIRECTORY}/$") # Erase any existing non-persistent data to avoid getting test impact framework out of sync with current repo state file(REMOVE_RECURSE "${LY_TEST_IMPACT_WORKING_DIR}") @@ -372,7 +379,7 @@ function(ly_test_impact_post_step) ly_test_impact_write_config_file( "cmake/TestImpactFramework/ConsoleFrontendConfig.in" ${persistent_data_dir} - ${runtime_bin_dir} + ${bin_dir} ) # Copy over the graphviz options file for the build dependency graphs @@ -380,6 +387,6 @@ function(ly_test_impact_post_step) file(COPY "cmake/TestImpactFramework/CMakeGraphVizOptions.cmake" DESTINATION ${CMAKE_BINARY_DIR}) # Set the above config file as the default config file to use for the test impact framework console target - target_compile_definitions(${LY_TEST_IMPACT_CONSOLE_TARGET} PRIVATE "LY_TEST_IMPACT_DEFAULT_CONFIG_FILE=\"$.$.cfg\"") + target_compile_definitions(${LY_TEST_IMPACT_CONSOLE_STATIC_TARGET} PUBLIC "LY_TEST_IMPACT_DEFAULT_CONFIG_FILE=\"${CMAKE_RUNTIME_OUTPUT_DIRECTORY}/$/$.$.json\"") message(DEBUG "Test impact framework post steps complete") -endfunction() \ No newline at end of file +endfunction()