birkeh/o3de
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Merge pull request #26 from aws-lumberyard-dev/TIF/Jenkins

Browse Source 
Tif/jenkins

@lumberyard-employee-dm and @amznestebanpapp if you have any comments i'll open up another PR 👍
This commit is contained in:
jonawals
2021-05-20 09:15:10 +01:00
committed by GitHub
parent 26a4101f32 c011c4c64c
commit f7cac706f7
20 changed files with 303 additions and 3668 deletions
Download Patch File Download Diff File Expand all files Collapse all files
Code/Tools/PythonBindingsExample/CMakeLists.txt
+1 -1
View File
@@ -114,6 +114,6 @@ if(PAL_TRAIT_BUILD_TESTS_SUPPORTED)
ly_add_googletest(
NAME AZ::PythonBindingsExample.Tests
TEST_COMMAND $<TARGET_FILE:AZ::PythonBindingsExample.Tests>
TEST_COMMAND $<TARGET_FILE:AZ::PythonBindingsExample.Tests> --unittest
)
endif()
Code/Tools/TestImpactFramework/Runtime/Code/Include/TestImpactFramework/TestImpactBitwise.h
-35
View File
@@ -1,35 +0,0 @@
/*
* All or portions of this file Copyright (c) Amazon.com, Inc. or its affiliates or
* its licensors.
*
* For complete copyright and license terms please see the LICENSE at the root of this
* distribution (the "License"). All use of this software is governed by the License,
* or, if provided, by the license below or the license accompanying this file. Do not
* remove or modify any license notices. This file is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
*
*/
#pragma once
namespace TestImpact
{
//! Convinience namespace for allowing bitwise operations on enum classes.
//! @note Any types declared in this namespace will have the bitwise operator overloads declared within.
namespace Bitwise
{
template<typename Flags>
Flags operator|(Flags lhs, Flags rhs)
{
return static_cast<Flags>(
static_cast<std::underlying_type<Flags>::type>(lhs) | static_cast<std::underlying_type<Flags>::type>(rhs));
}
template<typename Flags>
bool IsFlagSet(Flags flags, Flags flag)
{
return static_cast<bool>(
static_cast<std::underlying_type<Flags>::type>(flags) & static_cast<std::underlying_type<Flags>::type>(flag));
}
} // namespace Bitwise
} // namespace TestImpact
Code/Tools/TestImpactFramework/Runtime/Code/Include/TestImpactFramework/TestImpactCallback.h
-23
View File
@@ -1,23 +0,0 @@
/*
* All or portions of this file Copyright (c) Amazon.com, Inc. or its affiliates or
* its licensors.
*
* For complete copyright and license terms please see the LICENSE at the root of this
* distribution (the "License"). All use of this software is governed by the License,
* or, if provided, by the license below or the license accompanying this file. Do not
* remove or modify any license notices. This file is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
*
*/
#pragma once
namespace TestImpact
{
//! Generic callback result used by test impact systems.
enum class CallbackResult : bool
{
Continue,
Abort
};
} // namespace TestImpact
Code/Tools/TestImpactFramework/Runtime/Code/Include/TestImpactFramework/TestImpactFrameworkPath.h
-41
View File
@@ -1,41 +0,0 @@
/*
* All or portions of this file Copyright (c) Amazon.com, Inc. or its affiliates or
* its licensors.
*
* For complete copyright and license terms please see the LICENSE at the root of this
* distribution (the "License"). All use of this software is governed by the License,
* or, if provided, by the license below or the license accompanying this file. Do not
* remove or modify any license notices. This file is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
*
*/
#pragma once
#include <AzCore/IO/Path/Path.h>
namespace TestImpact
{
//! Wrapper for OS paths relative to a specified parent path.
//! @note Mimics path semantics only, makes no guarantees about validity.
class FrameworkPath
{
public:
FrameworkPath() = default;
//! Creates a path with no parent.
explicit FrameworkPath(const AZ::IO::Path& absolutePath);
//! Creates a path with an absolute path and path relative to the specified parent path.
explicit FrameworkPath(const AZ::IO::Path& absolutePath, const FrameworkPath& relativeTo);
//! Retrieves the absolute path.
const AZ::IO::Path& Absolute() const;
//! Retrieves the path relative to the specified parent path.
const AZ::IO::Path& Relative() const;
private:
//! The absolute path value.
AZ::IO::Path m_absolutePath;
//! The path value relative to the specified parent path.
AZ::IO::Path m_relativePath;
};
} // namespace TestImpact
Code/Tools/TestImpactFramework/Runtime/Code/Source/Artifact/Dynamic/TestImpactChangeList.h
-27
View File
@@ -1,27 +0,0 @@
/*
* All or portions of this file Copyright (c) Amazon.com, Inc. or its affiliates or
* its licensors.
*
* For complete copyright and license terms please see the LICENSE at the root of this
* distribution (the "License"). All use of this software is governed by the License,
* or, if provided, by the license below or the license accompanying this file. Do not
* remove or modify any license notices. This file is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
*
*/
#pragma once
#include <AzCore/std/containers/vector.h>
#include <AzCore/std/string/string.h>
namespace TestImpact
{
//! Artifact produced by the unified diff parsing process representing the file CRUD operations of a given diff.
struct ChangeList
{
AZStd::vector<AZStd::string> m_createdFiles;
AZStd::vector<AZStd::string> m_updatedFiles;
AZStd::vector<AZStd::string> m_deletedFiles;
};
} // namespace TestImpact
Code/Tools/TestImpactFramework/Runtime/Code/Source/Artifact/Factory/TestImpactChangeListFactory.cpp
-175
View File
@@ -1,175 +0,0 @@
/*
* All or portions of this file Copyright (c) Amazon.com, Inc. or its affiliates or
* its licensors.
*
* For complete copyright and license terms please see the LICENSE at the root of this
* distribution (the "License"). All use of this software is governed by the License,
* or, if provided, by the license below or the license accompanying this file. Do not
* remove or modify any license notices. This file is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
*
*/
#include <Artifact/Factory/TestImpactChangeListFactory.h>
#include <Artifact/TestImpactArtifactException.h>
#include <AzCore/std/optional.h>
namespace TestImpact
{
namespace Utils
{
template<typename C>
auto split(C&& str, const AZStd::string& delimiter)
{
AZStd::vector<C> strings;
for (auto p = str.data(), end = p + str.length(); p != end; p += ((p == end) ? 0 : delimiter.length()))
{
const auto pre = p;
p = AZStd::search(pre, end, delimiter.cbegin(), delimiter.cend());
if (p != pre)
{
strings.emplace_back(pre, p - pre);
}
}
return strings;
}
} // namespace Utils
namespace UnifiedDiff
{
class UnifiedDiffParser
{
public:
ChangeList Parse(const AZStd::string& unifiedDiff);
private:
AZStd::optional<AZStd::string_view> GetTargetFile(const AZStd::string_view& targetFile);
ChangeList GenerateChangelist(
const AZStd::vector<AZStd::optional<AZStd::string_view>>& src,
const AZStd::vector<AZStd::optional<AZStd::string_view>>& dst);
const AZStd::string m_srcFilePrefix = "--- ";
const AZStd::string m_dstFilePrefix = "+++ ";
const AZStd::string m_gitTargetPrefix = "b/";
const AZStd::string m_perforceTargetPrefix = "/b/";
const AZStd::string m_renameFromPrefix = "rename from ";
const AZStd::string m_renameToPrefix = "rename to ";
const AZStd::string m_nullFile = "/dev/null";
bool m_hasGitHeader = false;
};
AZStd::optional<AZStd::string_view> UnifiedDiffParser::GetTargetFile(const AZStd::string_view& targetFile)
{
size_t startIndex = 0;
if (targetFile.starts_with(m_renameFromPrefix))
{
startIndex = m_renameFromPrefix.length();
}
else if (targetFile.starts_with(m_renameToPrefix))
{
startIndex = m_renameToPrefix.length();
}
else if (targetFile.find(m_nullFile) != AZStd::string::npos)
{
return AZStd::nullopt;
}
else
{
startIndex = m_hasGitHeader ? m_gitTargetPrefix.size() + m_dstFilePrefix.size()
: m_perforceTargetPrefix.size() + m_dstFilePrefix.size();
}
const auto endIndex = targetFile.find('\t');
if (endIndex != AZStd::string::npos)
{
return targetFile.substr(startIndex, endIndex - startIndex);
}
return targetFile.substr(startIndex);
}
ChangeList UnifiedDiffParser::GenerateChangelist(
const AZStd::vector<AZStd::optional<AZStd::string_view>>& src, const AZStd::vector<AZStd::optional<AZStd::string_view>>& dst)
{
AZ_TestImpact_Eval(src.size() == dst.size(), ArtifactException, "Change list source and destination file count mismatch");
ChangeList changelist;
for (size_t i = 0; i < src.size(); i++)
{
if (!src[i].has_value())
{
changelist.m_createdFiles.emplace_back(dst[i].value());
}
else if (!dst[i].has_value())
{
changelist.m_deletedFiles.emplace_back(src[i].value());
}
else if (src[i] != dst[i])
{
changelist.m_deletedFiles.emplace_back(src[i].value());
changelist.m_createdFiles.emplace_back(dst[i].value());
}
else
{
changelist.m_updatedFiles.emplace_back(src[i].value());
}
}
return changelist;
}
ChangeList UnifiedDiffParser::Parse(const AZStd::string& unifiedDiff)
{
const AZStd::string GitHeader = "diff --git";
const auto lines = Utils::split<AZStd::string_view>(unifiedDiff, "\n");
AZStd::vector<AZStd::optional<AZStd::string_view>> src;
AZStd::vector<AZStd::optional<AZStd::string_view>> dst;
for (const auto& line : lines)
{
if (line.starts_with(GitHeader))
{
m_hasGitHeader = true;
}
else if (line.starts_with(m_srcFilePrefix))
{
src.emplace_back(GetTargetFile(line));
}
else if (line.starts_with(m_dstFilePrefix))
{
dst.emplace_back(GetTargetFile(line));
}
else if (line.starts_with(m_renameFromPrefix))
{
src.emplace_back(GetTargetFile(line));
}
else if (line.starts_with(m_renameToPrefix))
{
dst.emplace_back(GetTargetFile(line));
}
}
return GenerateChangelist(src, dst);
}
ChangeList ChangeListFactory(const AZStd::string& unifiedDiff)
{
AZ_TestImpact_Eval(!unifiedDiff.empty(), ArtifactException, "Unified diff is empty");
UnifiedDiffParser diff;
ChangeList changeList = diff.Parse(unifiedDiff);
AZ_TestImpact_Eval(
!changeList.m_createdFiles.empty() ||
!changeList.m_updatedFiles.empty() ||
!changeList.m_deletedFiles.empty(),
ArtifactException, "The unified diff contained no changes");
return changeList;
}
} // namespace UnifiedDiff
} // namespace TestImpact
Code/Tools/TestImpactFramework/Runtime/Code/Source/Artifact/Factory/TestImpactChangeListFactory.h
-29
View File
@@ -1,29 +0,0 @@
/*
* All or portions of this file Copyright (c) Amazon.com, Inc. or its affiliates or
* its licensors.
*
* For complete copyright and license terms please see the LICENSE at the root of this
* distribution (the "License"). All use of this software is governed by the License,
* or, if provided, by the license below or the license accompanying this file. Do not
* remove or modify any license notices. This file is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
*
*/
#pragma once
#include <Artifact/Dynamic/TestImpactChangeList.h>
#include <AzCore/std/containers/vector.h>
#include <AzCore/std/string/string.h>
namespace TestImpact
{
namespace UnifiedDiff
{
//! Constructs a change list artifact from the specified unified diff data.
//! @param unifiedDiffData The raw change list data in unified diff format.
//! @return The constructed change list artifact.
ChangeList ChangeListFactory(const AZStd::string& unifiedDiffData);
} // namespace UnifiedDiff
} // namespace TestImpact
Code/Tools/TestImpactFramework/Runtime/Code/Source/TestImpactFrameworkPath.cpp
-38
View File
@@ -1,38 +0,0 @@
/*
* All or portions of this file Copyright (c) Amazon.com, Inc. or its affiliates or
* its licensors.
*
* For complete copyright and license terms please see the LICENSE at the root of this
* distribution (the "License"). All use of this software is governed by the License,
* or, if provided, by the license below or the license accompanying this file. Do not
* remove or modify any license notices. This file is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
*
*/
#include <TestImpactFramework/TestImpactFrameworkPath.h>
namespace TestImpact
{
FrameworkPath::FrameworkPath(const AZ::IO::Path& absolutePath)
{
m_absolutePath = AZ::IO::Path(absolutePath).MakePreferred();
m_relativePath = m_absolutePath.LexicallyRelative(m_absolutePath);
}
FrameworkPath::FrameworkPath(const AZ::IO::Path& absolutePath, const FrameworkPath& relativeTo)
{
m_absolutePath = AZ::IO::Path(absolutePath).MakePreferred();
m_relativePath = m_absolutePath.LexicallyRelative(relativeTo.Absolute());
}
const AZ::IO::Path& FrameworkPath::Absolute() const
{
return m_absolutePath;
}
const AZ::IO::Path& FrameworkPath::Relative() const
{
return m_relativePath;
}
} // namespace TestImpact
Code/Tools/TestImpactFramework/Runtime/Code/Tests/Artifact/TestImpactChangeListFactoryTest.cpp
-288
View File
@@ -1,288 +0,0 @@
/*
* All or portions of this file Copyright (c) Amazon.com, Inc. or its affiliates or
* its licensors.
*
* For complete copyright and license terms please see the LICENSE at the root of this
* distribution (the "License"). All use of this software is governed by the License,
* or, if provided, by the license below or the license accompanying this file. Do not
* remove or modify any license notices. This file is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
*
*/
#include <Artifact/Factory/TestImpactChangeListFactory.h>
#include <Artifact/TestImpactArtifactException.h>
#include <AzCore/UnitTest/TestTypes.h>
#include <AzTest/AzTest.h>
namespace UnitTest
{
TEST(ChangeListFactoryTest, NoRawData_ExpectArtifactException)
{
// Given an empty unified diff string
const AZStd::string unifiedDiff;
try
{
// When attempting to construct the change list
const TestImpact::ChangeList changeList = TestImpact::UnifiedDiff::ChangeListFactory(unifiedDiff);
// Do not expect this statement to be reachable
FAIL();
}
catch ([[maybe_unused]] const TestImpact::ArtifactException& e)
{
// Expect an artifact exception
SUCCEED();
}
catch (...)
{
// Do not expect any other exceptions
FAIL();
}
}
TEST(ChangeListFactoryTest, NoChanges_ExpectArtifactException)
{
// Given a unified diff string with no changes
const AZStd::string unifiedDiff = "On this day in 1738 absolutely nothing happened";
try
{
// When attempting to construct the change list
const TestImpact::ChangeList changeList = TestImpact::UnifiedDiff::ChangeListFactory(unifiedDiff);
// Do not expect this statement to be reachable
FAIL();
}
catch ([[maybe_unused]] const TestImpact::ArtifactException& e)
{
// Expect an artifact exception
SUCCEED();
}
catch (...)
{
// Do not expect any other exceptions
FAIL();
}
}
TEST(ChangeListFactoryTest, CreateOnly_ExpectValidChangeListWithFileCreateOperations)
{
// Given a unified diff with only one file creation and no file updates or deletions
const AZStd::string unifiedDiff =
"From f642a2f698452fc18484758b0046132415f09467 Mon Sep 17 00:00:00 2001\n"
"From: user <user@website.com>\n"
"Date: Sat, 13 Mar 2021 22:58:07 +0000\n"
"Subject: Test\n"
"\n"
"---\n"
" New.txt | 1 +\n"
" create mode 100644 New.txt\n"
"diff --git a/New.txt b/New.txt\n"
"new file mode 100644\n"
"index 0000000..30d74d2\n"
"--- /dev/null\n"
"+++ b/New.txt\n"
"@@ -0,0 +1 @@\n"
"+test\n"
"\\ No newline at end of file\n"
"-- \n"
"2.30.0.windows.2\n"
"\n"
"\n";
// When attempting to construct the change list
const TestImpact::ChangeList changeList = TestImpact::UnifiedDiff::ChangeListFactory(unifiedDiff);
// Expect the change list to contain the 1 created file
EXPECT_EQ(changeList.m_createdFiles.size(), 1);
EXPECT_TRUE(
AZStd::find(changeList.m_createdFiles.begin(), changeList.m_createdFiles.end(), "New.txt") != changeList.m_createdFiles.end());
// Expect the change list to contain no updated files
EXPECT_TRUE(changeList.m_updatedFiles.empty());
// Expect the change list to contain no deleted files
EXPECT_TRUE(changeList.m_deletedFiles.empty());
}
TEST(ChangeListFactoryTest, UpdateOnly_ExpectValidChangeListWithFileUpdateOperations)
{
// Given a unified diff with only one file update and no file creations or deletions
const AZStd::string unifiedDiff =
"From f642a2f698452fc18484758b0046132415f09467 Mon Sep 17 00:00:00 2001\n"
"From: user <user@website.com>\n"
"Date: Sat, 13 Mar 2021 22:58:07 +0000\n"
"Subject: Test\n"
"\n"
"---\n"
" A.txt | 2 +-\n"
"diff --git a/A.txt b/A.txt\n"
"index 7c4a013..e132db2 100644\n"
"--- a/A.txt\n"
"+++ b/A.txt\n"
"@@ -1 +1 @@\n"
"-aaa\n"
"\\ No newline at end of file\n"
"+zzz\n"
"\\ No newline at end of file\n"
"-- \n"
"2.30.0.windows.2\n"
"\n"
"\n";
// When attempting to construct the change list
const TestImpact::ChangeList changeList = TestImpact::UnifiedDiff::ChangeListFactory(unifiedDiff);
// Expect the change list to contain no created files
EXPECT_TRUE(changeList.m_createdFiles.empty());
// Expect the change list to contain one updated file
EXPECT_EQ(changeList.m_updatedFiles.size(), 1);
EXPECT_TRUE(
AZStd::find(changeList.m_updatedFiles.begin(), changeList.m_updatedFiles.end(), "A.txt") != changeList.m_updatedFiles.end());
// Expect the change list to contain no deleted files
EXPECT_TRUE(changeList.m_deletedFiles.empty());
}
TEST(ChangeListFactoryTest, DeleteOnly_ExpectValidChangeListWithFileDeleteOperations)
{
// Given a unified diff with only one file deletion and no file creations or updates
const AZStd::string unifiedDiff =
"From f642a2f698452fc18484758b0046132415f09467 Mon Sep 17 00:00:00 2001\n"
"From: user <user@website.com>\n"
"Date: Sat, 13 Mar 2021 22:58:07 +0000\n"
"Subject: Test\n"
"\n"
"---\n"
" B.txt | 1 -\n"
" delete mode 100644 B.txt\n"
"diff --git a/B.txt b/B.txt\n"
"deleted file mode 100644\n"
"index 01f02e3..0000000\n"
"--- a/B.txt\n"
"+++ /dev/null\n"
"@@ -1 +0,0 @@\n"
"-bbb\n"
"\\ No newline at end of file\n"
"-- \n"
"2.30.0.windows.2\n"
"\n"
"\n";
// When attempting to construct the change list
const TestImpact::ChangeList changeList = TestImpact::UnifiedDiff::ChangeListFactory(unifiedDiff);
// Expect the change list to contain no created files
EXPECT_TRUE(changeList.m_createdFiles.empty());
// Expect the change list to contain no updated files
EXPECT_TRUE(changeList.m_updatedFiles.empty());
// Expect the change list to contain one deleted file
EXPECT_EQ(changeList.m_deletedFiles.size(), 1);
EXPECT_TRUE(
AZStd::find(changeList.m_deletedFiles.begin(), changeList.m_deletedFiles.end(), "B.txt") != changeList.m_deletedFiles.end());
}
TEST(ChangeListFactoryTest, ParseUnifiedDiffWithAllPossibleOperations_ExpectChangeListMatchingOperations)
{
// Given a unified diff with created files, updated files, deleted files, renamed files and moved files
const AZStd::string unifiedDiff =
"From f642a2f698452fc18484758b0046132415f09467 Mon Sep 17 00:00:00 2001\n"
"From: user <user@website.com>\n"
"Date: Sat, 13 Mar 2021 22:58:07 +0000\n"
"Subject: Test\n"
"\n"
"---\n"
" A.txt | 2 +-\n"
" B.txt | 1 -\n"
" D.txt => Foo/D.txt | 0\n"
" E.txt => Foo/Y.txt | 0\n"
" New.txt | 1 +\n"
" C.txt => X.txt | 0\n"
" 6 files changed, 2 insertions(+), 2 deletions(-)\n"
" delete mode 100644 B.txt\n"
" rename D.txt => Foo/D.txt (100%)\n"
" rename E.txt => Foo/Y.txt (100%)\n"
" create mode 100644 New.txt\n"
" rename C.txt => X.txt (100%)\n"
"\n"
"diff --git a/A.txt b/A.txt\n"
"index 7c4a013..e132db2 100644\n"
"--- a/A.txt\n"
"+++ b/A.txt\n"
"@@ -1 +1 @@\n"
"-aaa\n"
"\\ No newline at end of file\n"
"+zzz\n"
"\\ No newline at end of file\n"
"diff --git a/B.txt b/B.txt\n"
"deleted file mode 100644\n"
"index 01f02e3..0000000\n"
"--- a/B.txt\n"
"+++ /dev/null\n"
"@@ -1 +0,0 @@\n"
"-bbb\n"
"\\ No newline at end of file\n"
"diff --git a/D.txt b/Foo/D.txt\n"
"similarity index 100%\n"
"rename from D.txt\n"
"rename to Foo/D.txt\n"
"diff --git a/E.txt b/Foo/Y.txt\n"
"similarity index 100%\n"
"rename from E.txt\n"
"rename to Foo/Y.txt\n"
"diff --git a/New.txt b/New.txt\n"
"new file mode 100644\n"
"index 0000000..30d74d2\n"
"--- /dev/null\n"
"+++ b/New.txt\n"
"@@ -0,0 +1 @@\n"
"+test\n"
"\\ No newline at end of file\n"
"diff --git a/C.txt b/X.txt\n"
"similarity index 100%\n"
"rename from C.txt\n"
"rename to X.txt\n"
"-- \n"
"2.30.0.windows.2\n"
"\n"
"\n";
// When attempting to construct the change list
const TestImpact::ChangeList changeList = TestImpact::UnifiedDiff::ChangeListFactory(unifiedDiff);
// Expect the change list to contain the 4 created files
EXPECT_EQ(changeList.m_createdFiles.size(), 4);
EXPECT_TRUE(
AZStd::find(changeList.m_createdFiles.begin(), changeList.m_createdFiles.end(), "Foo/D.txt") !=
changeList.m_createdFiles.end());
EXPECT_TRUE(
AZStd::find(changeList.m_createdFiles.begin(), changeList.m_createdFiles.end(), "Foo/Y.txt") !=
changeList.m_createdFiles.end());
EXPECT_TRUE(
AZStd::find(changeList.m_createdFiles.begin(), changeList.m_createdFiles.end(), "X.txt") != changeList.m_createdFiles.end());
EXPECT_TRUE(
AZStd::find(changeList.m_createdFiles.begin(), changeList.m_createdFiles.end(), "New.txt") != changeList.m_createdFiles.end());
// Expect the change list to contain the 1 updated file
EXPECT_EQ(changeList.m_updatedFiles.size(), 1);
EXPECT_TRUE(
AZStd::find(changeList.m_updatedFiles.begin(), changeList.m_updatedFiles.end(), "A.txt") != changeList.m_updatedFiles.end());
// Expect the change list to contain the 4 deleted files
EXPECT_EQ(changeList.m_deletedFiles.size(), 4);
EXPECT_TRUE(
AZStd::find(changeList.m_deletedFiles.begin(), changeList.m_deletedFiles.end(), "B.txt") != changeList.m_deletedFiles.end());
EXPECT_TRUE(
AZStd::find(changeList.m_deletedFiles.begin(), changeList.m_deletedFiles.end(), "D.txt") != changeList.m_deletedFiles.end());
EXPECT_TRUE(
AZStd::find(changeList.m_deletedFiles.begin(), changeList.m_deletedFiles.end(), "E.txt") != changeList.m_deletedFiles.end());
EXPECT_TRUE(
AZStd::find(changeList.m_deletedFiles.begin(), changeList.m_deletedFiles.end(), "C.txt") != changeList.m_deletedFiles.end());
}
} // namespace UnitTest
Code/Tools/TestImpactFramework/Runtime/Code/Tests/Test/TestImpactInstrumentedTestRunnerTest.cpp
-822
View File
@@ -1,822 +0,0 @@
/*
* All or portions of this file Copyright (c) Amazon.com, Inc. or its affiliates or
* its licensors.
*
* For complete copyright and license terms please see the LICENSE at the root of this
* distribution (the "License"). All use of this software is governed by the License,
* or, if provided, by the license below or the license accompanying this file. Do not
* remove or modify any license notices. This file is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
*
*/
#include <TestImpactTestJobRunnerCommon.h>
#include <TestImpactTestUtils.h>
#include <Artifact/TestImpactArtifactException.h>
#include <Test/Run/TestImpactInstrumentedTestRunner.h>
#include <Test/Run/TestImpactTestRunException.h>
#include <AzCore/IO/Path/Path.h>
#include <AzCore/UnitTest/TestTypes.h>
#include <AzCore/std/containers/array.h>
#include <AzCore/std/containers/vector.h>
#include <AzCore/std/optional.h>
#include <AzCore/std/smart_ptr/unique_ptr.h>
#include <AzCore/std/string/regex.h>
#include <AzCore/std/string/string.h>
#include <AzTest/AzTest.h>
namespace UnitTest
{
using JobExceptionPolicy = TestImpact::InstrumentedTestRunner::JobExceptionPolicy;
using CoverageExceptionPolicy = TestImpact::InstrumentedTestRunner::CoverageExceptionPolicy;
struct TargetPaths
{
AZ::IO::Path m_targetBinary;
AZ::IO::Path m_testRunArtifact;
AZ::IO::Path m_testCoverageArtifact;
};
// Indices for looking up job command arguments for the different coverage levels
enum CoverageLevel : uint8_t
{
LineLevel = 0,
SourceLevel
};
// Get the job command for an instrumented test run
AZStd::string GetRunCommandForTarget(const TargetPaths& testTarget, CoverageLevel coverageLevel, const char* sourcesFilter)
{
AZStd::string args = AZStd::string::format(
"%s " // 1. Instrumented test runner
"--coverage_level %s " // 2. Coverage level
"--export_type cobertura:\"%s\" " // 3. Test coverage artifact path
"--modules \"%s\" " // 4. Modules path
"--excluded_modules \"%s\" " // 5. Exclude modules
"--sources \"%s\" -- " // 6. Sources path
"\"%s\" " // 7. Test runner binary
"\"%s\" " // 8. Test target bin
"AzRunUnitTests "
"--gtest_output=xml:%s", // 9. Test run result artifact
LY_TEST_IMPACT_INSTRUMENTATION_BIN, // 1.
(coverageLevel == CoverageLevel::LineLevel ? "line" : "source"), // 2.
testTarget.m_testCoverageArtifact.c_str(), // 3.
LY_TEST_IMPACT_MODULES_DIR, // 4.
LY_TEST_IMPACT_AZ_TESTRUNNER_BIN, // 5.
sourcesFilter, // 6.
LY_TEST_IMPACT_AZ_TESTRUNNER_BIN, // 7.
testTarget.m_targetBinary.c_str(), // 8.
testTarget.m_testRunArtifact.c_str() // 9.
);
// OpenCppCoverage doesn't support forward slash directory separators so replace all with escaped backslashes
return AZStd::regex_replace(args, AZStd::regex("/"), "\\");
}
// Get the job command for an instrumented test run with valid source filters to produce coverage artifact
AZStd::string GetRunCommandForTargetWithSources(const TargetPaths& testTarget, CoverageLevel coverageLevel)
{
return GetRunCommandForTarget(testTarget, coverageLevel, LY_TEST_IMPACT_COVERAGE_SOURCES_DIR);
}
// Get the job command for an instrumented test run without valid source filters to produce empty coverage artifact
AZStd::string GetRunCommandForTargetWithoutSources(const TargetPaths& testTarget, CoverageLevel coverageLevel)
{
return GetRunCommandForTarget(testTarget, coverageLevel, "C:\\No\\Sources\\Here\\At\\All\\Ever\\Ever\\Ever");
}
class InstrumentedTestRunnerFixture
: public AllocatorsTestFixture
{
public:
void SetUp() override;
void TearDown() override;
protected:
using JobInfo = TestImpact::InstrumentedTestRunner::JobInfo;
using JobData = TestImpact::InstrumentedTestRunner::JobData;
AZStd::vector<JobInfo> m_jobInfos;
AZStd::unique_ptr<TestImpact::InstrumentedTestRunner> m_testRunner;
AZStd::vector<AZStd::array<AZStd::string, 2>> m_testTargetJobArgs;
AZStd::vector<TargetPaths> m_testTargetPaths;
AZStd::vector<TestImpact::TestRun> m_expectedTestTargetRuns;
AZStd::vector<AZStd::array<TestImpact::TestCoverage, 2>> m_expectedTestTargetCoverages;
AZStd::vector<TestImpact::TestRunResult> m_expectedTestTargetResult;
size_t m_maxConcurrency = 0;
CoverageLevel m_coverageLevel = CoverageLevel::LineLevel;
inline static AZ::u32 s_uniqueTestCaseId = 0; // Unique id for each test case to be used as the suffix for the written files
};
void InstrumentedTestRunnerFixture::SetUp()
{
AllocatorsTestFixture::SetUp();
// Suffix each artifact file with the unique test case id to ensure that there are no possible race conditions between cleaning
// up the artifact files after each test case and those artifact files being written and read again in future test cases
const AZStd::string fileExtension = AZStd::string::format(".%u.xml", s_uniqueTestCaseId++);
const AZStd::string runPath = AZStd::string(LY_TEST_IMPACT_TEST_TARGET_RESULTS_DIR) + "/%s.Run" + fileExtension;
const AZStd::string coveragePath = AZStd::string(LY_TEST_IMPACT_TEST_TARGET_COVERAGE_DIR) + "/%s.Coverage" + fileExtension;
// TestTargetA
m_testTargetPaths.emplace_back(TargetPaths{
LY_TEST_IMPACT_TEST_TARGET_A_BIN, AZStd::string::format(runPath.c_str(), LY_TEST_IMPACT_TEST_TARGET_A_BASE_NAME),
AZStd::string::format(coveragePath.c_str(), LY_TEST_IMPACT_TEST_TARGET_A_BASE_NAME)});
// TestTargetB
m_testTargetPaths.emplace_back(TargetPaths{
LY_TEST_IMPACT_TEST_TARGET_B_BIN, AZStd::string::format(runPath.c_str(), LY_TEST_IMPACT_TEST_TARGET_B_BASE_NAME),
AZStd::string::format(coveragePath.c_str(), LY_TEST_IMPACT_TEST_TARGET_B_BASE_NAME)});
// TestTargetC
m_testTargetPaths.emplace_back(TargetPaths{
LY_TEST_IMPACT_TEST_TARGET_C_BIN, AZStd::string::format(runPath.c_str(), LY_TEST_IMPACT_TEST_TARGET_C_BASE_NAME),
AZStd::string::format(coveragePath.c_str(), LY_TEST_IMPACT_TEST_TARGET_C_BASE_NAME)});
// TestTargetD
m_testTargetPaths.emplace_back(TargetPaths{
LY_TEST_IMPACT_TEST_TARGET_D_BIN, AZStd::string::format(runPath.c_str(), LY_TEST_IMPACT_TEST_TARGET_D_BASE_NAME),
AZStd::string::format(coveragePath.c_str(), LY_TEST_IMPACT_TEST_TARGET_D_BASE_NAME)});
m_expectedTestTargetRuns.emplace_back(GetTestTargetATestRunSuites(), AZStd::chrono::milliseconds{500}); // TestTargetA
m_expectedTestTargetRuns.emplace_back(GetTestTargetBTestRunSuites(), AZStd::chrono::milliseconds{500}); // TestTargetB
m_expectedTestTargetRuns.emplace_back(GetTestTargetCTestRunSuites(), AZStd::chrono::milliseconds{500}); // TestTargetC
m_expectedTestTargetRuns.emplace_back(GetTestTargetDTestRunSuites(), AZStd::chrono::milliseconds{500}); // TestTargetD
// TestTargetA
m_expectedTestTargetCoverages.emplace_back(AZStd::array<TestImpact::TestCoverage, 2>{
TestImpact::TestCoverage(GetTestTargetALineModuleCoverages()),
TestImpact::TestCoverage(GetTestTargetASourceModuleCoverages())});
// TestTargetB
m_expectedTestTargetCoverages.emplace_back(AZStd::array<TestImpact::TestCoverage, 2>{
TestImpact::TestCoverage(GetTestTargetBLineModuleCoverages()),
TestImpact::TestCoverage(GetTestTargetBSourceModuleCoverages())});
// TestTargetC
m_expectedTestTargetCoverages.emplace_back(AZStd::array<TestImpact::TestCoverage, 2>{
TestImpact::TestCoverage(GetTestTargetCLineModuleCoverages()),
TestImpact::TestCoverage(GetTestTargetCSourceModuleCoverages())});
// TestTargetD
m_expectedTestTargetCoverages.emplace_back(AZStd::array<TestImpact::TestCoverage, 2>{
TestImpact::TestCoverage(GetTestTargetDLineModuleCoverages()),
TestImpact::TestCoverage(GetTestTargetDSourceModuleCoverages())});
m_expectedTestTargetResult.emplace_back(TestImpact::TestRunResult::Failed); // TestTargetA
m_expectedTestTargetResult.emplace_back(TestImpact::TestRunResult::Passed); // TestTargetB
m_expectedTestTargetResult.emplace_back(TestImpact::TestRunResult::Passed); // TestTargetC
m_expectedTestTargetResult.emplace_back(TestImpact::TestRunResult::Passed); // TestTargetD
// Generate the job command arguments for both line level and source level coverage permutations
for (const auto& testTarget : m_testTargetPaths)
{
m_testTargetJobArgs.emplace_back(AZStd::array<AZStd::string, 2>{
GetRunCommandForTargetWithSources(testTarget, CoverageLevel::LineLevel),
GetRunCommandForTargetWithSources(testTarget, CoverageLevel::SourceLevel)});
}
}
void InstrumentedTestRunnerFixture::TearDown()
{
DeleteFiles(LY_TEST_IMPACT_TEST_TARGET_COVERAGE_DIR, "*.xml");
DeleteFiles(LY_TEST_IMPACT_TEST_TARGET_RESULTS_DIR, "*.xml");
AllocatorsTestFixture::TearDown();
}
using ConcurrencyAndCoveragePermutation = AZStd::tuple
<
size_t, // Max number of concurrent processes
CoverageLevel // Coverage level
>;
// Fixture parameterized for different max number of concurrent jobs and coverage levels
class InstrumentedTestRunnerFixtureWithConcurrencyAndCoverageParams
: public InstrumentedTestRunnerFixture
, public ::testing::WithParamInterface<ConcurrencyAndCoveragePermutation>
{
public:
void SetUp() override
{
InstrumentedTestRunnerFixture::SetUp();
auto [maxConcurrency, coverageLevel] = GetParam();
m_maxConcurrency = maxConcurrency;
m_coverageLevel = coverageLevel;
}
};
using ConcurrencyAndJobExceptionPermutation = AZStd::tuple
<
size_t, // Max number of concurrent processes
CoverageLevel, // Coverage level
JobExceptionPolicy // Test job exception policy
>;
// Fixture parameterized for different max number of concurrent jobs, coverage levels and different job exception policies
class InstrumentedTestRunnerFixtureWithConcurrencyAndCoverageAndJobExceptionParams
: public InstrumentedTestRunnerFixture
, public ::testing::WithParamInterface<ConcurrencyAndJobExceptionPermutation>
{
public:
void SetUp() override
{
InstrumentedTestRunnerFixture::SetUp();
const auto& [maxConcurrency, coverageLevel, jobExceptionPolicy] = GetParam();
m_maxConcurrency = maxConcurrency;
m_coverageLevel = coverageLevel;
m_jobExceptionPolicy = jobExceptionPolicy;
}
protected:
JobExceptionPolicy m_jobExceptionPolicy = JobExceptionPolicy::Never;
};
class InstrumentedTestRunnerFixtureWithConcurrencyAndCoverageAndFailedToLaunchExceptionParams
: public InstrumentedTestRunnerFixtureWithConcurrencyAndCoverageAndJobExceptionParams
{
};
class InstrumentedTestRunnerFixtureWithConcurrencyAndCoverageAndExecutedWithFailureExceptionParams
: public InstrumentedTestRunnerFixtureWithConcurrencyAndCoverageAndJobExceptionParams
{
};
using ConcurrencyAndCoverageExceptionPermutation = AZStd::tuple
<
size_t, // Max number of concurrent processes
CoverageExceptionPolicy // Test coverage exception policy
>;
// Fixture parameterized for different max number of concurrent jobs and different coverage exception policies
class InstrumentedTestRunnerFixtureWithConcurrencyAndCoverageAndCoverageExceptionParams
: public InstrumentedTestRunnerFixture
, public ::testing::WithParamInterface<ConcurrencyAndCoverageExceptionPermutation>
{
public:
void SetUp() override
{
InstrumentedTestRunnerFixture::SetUp();
const auto& [maxConcurrency, coverageExceptionPolicy] = GetParam();
m_maxConcurrency = maxConcurrency;
m_coverageExceptionPolicy = coverageExceptionPolicy;
}
protected:
CoverageExceptionPolicy m_coverageExceptionPolicy = CoverageExceptionPolicy::Never;
};
namespace
{
AZStd::array<size_t, 4> MaxConcurrentRuns = {1, 2, 3, 4};
AZStd::array<CoverageLevel, 2> CoverageLevels = {CoverageLevel::LineLevel, CoverageLevel::SourceLevel};
AZStd::array<JobExceptionPolicy, 2> FailedToLaunchExceptionPolicies = {
JobExceptionPolicy::Never, JobExceptionPolicy::OnFailedToExecute};
AZStd::array<JobExceptionPolicy, 2> ExecutedWithFailureExceptionPolicies = {
JobExceptionPolicy::Never, JobExceptionPolicy::OnExecutedWithFailure};
} // namespace
// Validates that the specified test coverage matches the expected output
void ValidateTestTargetCoverage(const TestImpact::TestCoverage& actualResult, const TestImpact::TestCoverage& expectedResult)
{
EXPECT_TRUE(actualResult == expectedResult);
}
// Validates that the specified test coverage is empty
void ValidateEmptyTestTargetCoverage(const TestImpact::TestCoverage& actualResult)
{
EXPECT_TRUE(actualResult.GetSourcesCovered().empty());
EXPECT_TRUE(actualResult.GetModuleCoverages().empty());
EXPECT_EQ(actualResult.GetNumSourcesCovered(), 0);
EXPECT_EQ(actualResult.GetNumModulesCovered(), 0);
}
TEST_P(
InstrumentedTestRunnerFixtureWithConcurrencyAndCoverageAndCoverageExceptionParams,
EmptyTestCoverages_ExpectEmptyTestCoveragesOrTestRunException)
{
// Given a test runner with no client callback or run timeout or runner timeout
m_testRunner =
AZStd::make_unique<TestImpact::InstrumentedTestRunner>(AZStd::nullopt, m_maxConcurrency, AZStd::nullopt, AZStd::nullopt);
// Given a mixture of instrumented test run jobs with and without coverage sources
for (size_t jobId = 0; jobId < m_testTargetJobArgs.size(); jobId++)
{
const AZStd::string args = (jobId % 2) ? GetRunCommandForTargetWithoutSources(m_testTargetPaths[jobId], m_coverageLevel)
: m_testTargetJobArgs[jobId][m_coverageLevel];
JobData jobData(m_testTargetPaths[jobId].m_testRunArtifact, m_testTargetPaths[jobId].m_testCoverageArtifact);
m_jobInfos.emplace_back(JobInfo({jobId}, args, AZStd::move(jobData)));
}
try
{
// When the instrumented test run jobs are executed with different exception policies
const auto runnerJobs = m_testRunner->RunInstrumentedTests(m_jobInfos, m_coverageExceptionPolicy, JobExceptionPolicy::Never);
// Expect this statement to be reachable only if no exception policy for empty coverages
EXPECT_FALSE(::IsFlagSet(m_coverageExceptionPolicy, CoverageExceptionPolicy::OnEmptyCoverage));
for (const auto& job : runnerJobs)
{
const JobInfo::IdType jobId = job.GetJobInfo().GetId().m_value;
ValidateTestRunCompleted(job, m_expectedTestTargetResult[jobId]);
ValidateTestTargetRun(job.GetPayload().value().first, m_expectedTestTargetRuns[jobId]);
if (jobId % 2)
{
// Expect jobs have a empty test coverages
ValidateEmptyTestTargetCoverage(job.GetPayload().value().second);
}
else
{
// Expect the jobs to successfully result in a test run and coverage that matches the expected test run data
ValidateTestTargetCoverage(job.GetPayload().value().second, m_expectedTestTargetCoverages[jobId][m_coverageLevel]);
}
}
}
catch ([[maybe_unused]] const TestImpact::TestRunException& e)
{
// Expect this statement to be reachable only if there is an exception policy for empty coverages
EXPECT_TRUE(::IsFlagSet(m_coverageExceptionPolicy, CoverageExceptionPolicy::OnEmptyCoverage));
}
catch (...)
{
// Do not expect any other exceptions
FAIL();
}
}
TEST_P(
InstrumentedTestRunnerFixtureWithConcurrencyAndCoverageAndFailedToLaunchExceptionParams,
InvalidCommandArgument_ExpectJobResulFailedToExecuteeOrTestJobException)
{
// Given a test runner with no client callback or run timeout or runner timeout
m_testRunner =
AZStd::make_unique<TestImpact::InstrumentedTestRunner>(AZStd::nullopt, m_maxConcurrency, AZStd::nullopt, AZStd::nullopt);
// Given a mixture of instrumented test run jobs with valid and invalid command arguments
for (size_t jobId = 0; jobId < m_testTargetJobArgs.size(); jobId++)
{
const AZStd::string args = (jobId % 2) ? InvalidProcessPath : m_testTargetJobArgs[jobId][m_coverageLevel];
JobData jobData(m_testTargetPaths[jobId].m_testRunArtifact, m_testTargetPaths[jobId].m_testCoverageArtifact);
m_jobInfos.emplace_back(JobInfo({jobId}, args, AZStd::move(jobData)));
}
try
{
// When the instrumented test run jobs are executed with different exception policies
const auto runnerJobs = m_testRunner->RunInstrumentedTests(m_jobInfos, CoverageExceptionPolicy::Never, m_jobExceptionPolicy);
// Expect this statement to be reachable only if no exception policy for launch failures
EXPECT_FALSE(::IsFlagSet(m_jobExceptionPolicy, JobExceptionPolicy::OnFailedToExecute));
for (const auto& job : runnerJobs)
{
const JobInfo::IdType jobId = job.GetJobInfo().GetId().m_value;
if (jobId % 2)
{
// Expect invalid jobs have a job result of FailedToExecute
ValidateJobFailedToExecute(job);
}
else
{
// Expect the valid jobs to successfully result in a test run that matches the expected test run data
ValidateTestRunCompleted(job, m_expectedTestTargetResult[jobId]);
ValidateTestTargetRun(job.GetPayload().value().first, m_expectedTestTargetRuns[jobId]);
ValidateTestTargetCoverage(job.GetPayload().value().second, m_expectedTestTargetCoverages[jobId][m_coverageLevel]);
}
}
}
catch ([[maybe_unused]] const TestImpact::TestJobException& e)
{
// Expect this statement to be reachable only if there is an exception policy for launch failures
EXPECT_TRUE(::IsFlagSet(m_jobExceptionPolicy, JobExceptionPolicy::OnFailedToExecute));
}
catch (...)
{
// Do not expect any other exceptions
FAIL();
}
}
TEST_P(
InstrumentedTestRunnerFixtureWithConcurrencyAndCoverageAndExecutedWithFailureExceptionParams,
ErroneousReturnCode_ExpectJobResultExecutedWithFailureOrTestJobException)
{
// Given a test runner with no client callback or run timeout or runner timeout
m_testRunner =
AZStd::make_unique<TestImpact::InstrumentedTestRunner>(AZStd::nullopt, m_maxConcurrency, AZStd::nullopt, AZStd::nullopt);
// Given a mixture of instrumented test run jobs that execute and return either successfully or with failure
for (size_t jobId = 0; jobId < m_testTargetJobArgs.size(); jobId++)
{
JobData jobData(m_testTargetPaths[jobId].m_testRunArtifact, m_testTargetPaths[jobId].m_testCoverageArtifact);
m_jobInfos.emplace_back(JobInfo({jobId}, m_testTargetJobArgs[jobId][m_coverageLevel], AZStd::move(jobData)));
}
try
{
// When the instrumented test run jobs are executed with different exception policies
const auto runnerJobs = m_testRunner->RunInstrumentedTests(m_jobInfos, CoverageExceptionPolicy::Never, m_jobExceptionPolicy);
// Expect this statement to be reachable only if no exception policy for jobs that return with error
EXPECT_FALSE(::IsFlagSet(m_jobExceptionPolicy, JobExceptionPolicy::OnExecutedWithFailure));
for (const auto& job : runnerJobs)
{
const JobInfo::IdType jobId = job.GetJobInfo().GetId().m_value;
// Expect the valid jobs to successfully result in a test run that matches the expected test run data
ValidateTestRunCompleted(job, m_expectedTestTargetResult[jobId]);
ValidateTestTargetRun(job.GetPayload().value().first, m_expectedTestTargetRuns[jobId]);
ValidateTestTargetCoverage(job.GetPayload().value().second, m_expectedTestTargetCoverages[jobId][m_coverageLevel]);
}
}
catch ([[maybe_unused]] const TestImpact::TestJobException& e)
{
// Expect this statement to be reachable only if there is an exception policy for jobs that return with error
EXPECT_TRUE(::IsFlagSet(m_jobExceptionPolicy, JobExceptionPolicy::OnExecutedWithFailure));
}
catch ([[maybe_unused]] const TestImpact::Exception& e)
{
FAIL();
}
catch (...)
{
// Do not expect any other exceptions
FAIL();
}
}
TEST_F(InstrumentedTestRunnerFixture, EmptyRunRawData_ExpectTestRunnerException)
{
// Given a test runner with no client callback, concurrency, run timeout or runner timeout
m_testRunner =
AZStd::make_unique<TestImpact::InstrumentedTestRunner>(AZStd::nullopt, OneConcurrentProcess, AZStd::nullopt, AZStd::nullopt);
// Given an test runner job that will return successfully but with an empty artifact string
JobData jobData("", m_testTargetPaths[TestTargetA].m_testCoverageArtifact);
m_jobInfos.emplace_back(JobInfo({TestTargetA}, m_testTargetJobArgs[TestTargetA][LineLevel], AZStd::move(jobData)));
try
{
// When the test runner job is executed
const auto runnerJobs =
m_testRunner->RunInstrumentedTests(m_jobInfos, CoverageExceptionPolicy::Never, JobExceptionPolicy::Never);
// Do not expect this statement to be reachable
FAIL();
}
catch ([[maybe_unused]] const TestImpact::TestRunException& e)
{
// Expect an runner exception
SUCCEED();
}
catch (...)
{
// Do not expect any other exceptions
FAIL();
}
}
TEST_F(InstrumentedTestRunnerFixture, EmptyCoverageRawData_ExpectTestRunnerException)
{
// Given a test runner with no client callback, concurrency, run timeout or runner timeout
m_testRunner =
AZStd::make_unique<TestImpact::InstrumentedTestRunner>(AZStd::nullopt, OneConcurrentProcess, AZStd::nullopt, AZStd::nullopt);
// Given an test runner job that will return successfully but with an empty artifact string
JobData jobData(m_testTargetPaths[TestTargetA].m_testRunArtifact, "");
m_jobInfos.emplace_back(JobInfo({TestTargetA}, m_testTargetJobArgs[TestTargetA][LineLevel], AZStd::move(jobData)));
try
{
// When the test runner job is executed
const auto runnerJobs =
m_testRunner->RunInstrumentedTests(m_jobInfos, CoverageExceptionPolicy::Never, JobExceptionPolicy::Never);
// Do not expect this statement to be reachable
FAIL();
}
catch ([[maybe_unused]] const TestImpact::TestRunException& e)
{
// Expect an runner exception
SUCCEED();
}
catch (...)
{
// Do not expect any other exceptions
FAIL();
}
}
TEST_F(InstrumentedTestRunnerFixture, InvalidRunArtifact_ExpectArtifactException)
{
// Given a run artifact with invalid contents
WriteTextToFile("There is nothing valid here", m_testTargetPaths[TestTargetA].m_testRunArtifact);
// Given a job command that will write the run artifact to a different location that what we will read from
TargetPaths invalidRunArtifact = m_testTargetPaths[TestTargetA];
invalidRunArtifact.m_testRunArtifact /= ".xml";
const AZStd::string args = GetRunCommandForTargetWithSources(invalidRunArtifact, LineLevel);
// Given a test runner with no client callback, concurrency, run timeout or runner timeout
m_testRunner =
AZStd::make_unique<TestImpact::InstrumentedTestRunner>(AZStd::nullopt, OneConcurrentProcess, AZStd::nullopt, AZStd::nullopt);
// Given an test runner job that will return successfully but not produce a run artifact
JobData jobData(m_testTargetPaths[TestTargetA].m_testRunArtifact, m_testTargetPaths[TestTargetA].m_testCoverageArtifact);
m_jobInfos.emplace_back(JobInfo({TestTargetA}, args, AZStd::move(jobData)));
try
{
// When the test runner job is executed
const auto runnerJobs =
m_testRunner->RunInstrumentedTests(m_jobInfos, CoverageExceptionPolicy::Never, JobExceptionPolicy::Never);
// Do not expect this statement to be reachable
FAIL();
}
catch ([[maybe_unused]] const TestImpact::ArtifactException& e)
{
// Expect an runner exception
SUCCEED();
}
catch (...)
{
// Do not expect any other exceptions
FAIL();
}
}
TEST_F(InstrumentedTestRunnerFixture, InvalidCoverageArtifact_ExpectArtifactException)
{
// Given a coverage artifact with invalid contents
WriteTextToFile("There is nothing valid here", m_testTargetPaths[TestTargetA].m_testCoverageArtifact);
// Given a job command that will write the coverage artifact to a different location that what we will read from
TargetPaths invalidCoverageArtifact = m_testTargetPaths[TestTargetA];
invalidCoverageArtifact.m_testCoverageArtifact /= ".xml";
const AZStd::string args = GetRunCommandForTargetWithSources(invalidCoverageArtifact, LineLevel);
// Given a test runner with no client callback, concurrency, run timeout or runner timeout
m_testRunner =
AZStd::make_unique<TestImpact::InstrumentedTestRunner>(AZStd::nullopt, OneConcurrentProcess, AZStd::nullopt, AZStd::nullopt);
// Given an test runner job that will return successfully but not produce a run artifact
JobData jobData(m_testTargetPaths[TestTargetA].m_testRunArtifact, m_testTargetPaths[TestTargetA].m_testCoverageArtifact);
m_jobInfos.emplace_back(JobInfo({TestTargetA}, args, AZStd::move(jobData)));
try
{
// When the test runner job is executed
const auto runnerJobs =
m_testRunner->RunInstrumentedTests(m_jobInfos, CoverageExceptionPolicy::Never, JobExceptionPolicy::Never);
// Do not expect this statement to be reachable
FAIL();
}
catch ([[maybe_unused]] const TestImpact::ArtifactException& e)
{
// Expect an runner exception
SUCCEED();
}
catch (...)
{
// Do not expect any other exceptions
FAIL();
}
}
TEST_P(InstrumentedTestRunnerFixtureWithConcurrencyAndCoverageParams, RunTestTargets_RunsAndCoverageMatchTestSuitesInTarget)
{
// Given a test runner with no client callback, runner timeout or runner timeout
m_testRunner =
AZStd::make_unique<TestImpact::InstrumentedTestRunner>(AZStd::nullopt, m_maxConcurrency, AZStd::nullopt, AZStd::nullopt);
// Given an test runner job for each test target with no runner caching
for (size_t jobId = 0; jobId < m_testTargetJobArgs.size(); jobId++)
{
JobData jobData(m_testTargetPaths[jobId].m_testRunArtifact, m_testTargetPaths[jobId].m_testCoverageArtifact);
m_jobInfos.emplace_back(JobInfo({jobId}, m_testTargetJobArgs[jobId][m_coverageLevel], AZStd::move(jobData)));
}
// When the test runner jobs are executed
const auto runnerJobs = m_testRunner->RunInstrumentedTests(m_jobInfos, CoverageExceptionPolicy::Never, JobExceptionPolicy::Never);
// Expect each job to successfully result in a test runner that matches the expected test runner data for that test target
for (const auto& job : runnerJobs)
{
const JobInfo::IdType jobId = job.GetJobInfo().GetId().m_value;
ValidateTestRunCompleted(job, m_expectedTestTargetResult[jobId]);
ValidateTestTargetRun(job.GetPayload().value().first, m_expectedTestTargetRuns[jobId]);
ValidateTestTargetCoverage(job.GetPayload().value().second, m_expectedTestTargetCoverages[jobId][m_coverageLevel]);
}
}
TEST_P(
InstrumentedTestRunnerFixtureWithConcurrencyAndCoverageParams,
RunTestTargetsWithArbitraryJobIds_RunsAndCoverageMatchTestSuitesInTarget)
{
// Given a set of arbitrary job ids to be used for the test target jobs
enum
{
ArbitraryA = 36,
ArbitraryB = 890,
ArbitraryC = 19,
ArbitraryD = 1
};
const AZStd::unordered_map<JobInfo::IdType, JobInfo::IdType> sequentialToArbitrary =
{
{TestTargetA, ArbitraryA},
{TestTargetB, ArbitraryB},
{TestTargetC, ArbitraryC},
{TestTargetD, ArbitraryD},
};
const AZStd::unordered_map<JobInfo::IdType, JobInfo::IdType> arbitraryToSequential =
{
{ArbitraryA, TestTargetA},
{ArbitraryB, TestTargetB},
{ArbitraryC, TestTargetC},
{ArbitraryD, TestTargetD},
};
// Given a test runner with no client callback, run timeout or runner timeout
m_testRunner =
AZStd::make_unique<TestImpact::InstrumentedTestRunner>(AZStd::nullopt, m_maxConcurrency, AZStd::nullopt, AZStd::nullopt);
// Given an test run job for each test target
for (size_t jobId = 0; jobId < m_testTargetJobArgs.size(); jobId++)
{
JobData jobData(m_testTargetPaths[jobId].m_testRunArtifact, m_testTargetPaths[jobId].m_testCoverageArtifact);
m_jobInfos.emplace_back(
JobInfo({sequentialToArbitrary.at(jobId)}, m_testTargetJobArgs[jobId][m_coverageLevel], AZStd::move(jobData)));
}
// When the instrumented test run jobs are executed
const auto runnerJobs = m_testRunner->RunInstrumentedTests(m_jobInfos, CoverageExceptionPolicy::Never, JobExceptionPolicy::Never);
// Expect each job to successfully result in a test run that matches the expected test run data for that test target
for (const auto& job : runnerJobs)
{
const JobInfo::IdType jobId = arbitraryToSequential.at(job.GetJobInfo().GetId().m_value);
ValidateTestRunCompleted(job, m_expectedTestTargetResult[jobId]);
ValidateTestTargetRun(job.GetPayload().value().first, m_expectedTestTargetRuns[jobId]);
ValidateTestTargetCoverage(job.GetPayload().value().second, m_expectedTestTargetCoverages[jobId][m_coverageLevel]);
}
}
TEST_P(InstrumentedTestRunnerFixtureWithConcurrencyAndCoverageParams, RunTestTargetsWithCallback_RunsAndCoverageMatchTestSuitesInTarget)
{
// Given a client callback function that tracks the number of successful runs
size_t numSuccesses = 0;
const auto jobCallback =
[&numSuccesses]([[maybe_unused]] const TestImpact::InstrumentedTestRunner::JobInfo& jobInfo, const TestImpact::JobMeta& meta)
{
if (meta.m_result == TestImpact::JobResult::ExecutedWithSuccess)
{
numSuccesses++;
}
};
// Given a test runner with no run timeout or runner timeout
m_testRunner =
AZStd::make_unique<TestImpact::InstrumentedTestRunner>(jobCallback, m_maxConcurrency, AZStd::nullopt, AZStd::nullopt);
// Given an test run job for each test target
for (size_t jobId = 0; jobId < m_testTargetJobArgs.size(); jobId++)
{
JobData jobData(m_testTargetPaths[jobId].m_testRunArtifact, m_testTargetPaths[jobId].m_testCoverageArtifact);
m_jobInfos.emplace_back(JobInfo({jobId}, m_testTargetJobArgs[jobId][m_coverageLevel], AZStd::move(jobData)));
}
// When the instrumented test run jobs are executed
const auto runnerJobs = m_testRunner->RunInstrumentedTests(m_jobInfos, CoverageExceptionPolicy::Never, JobExceptionPolicy::Never);
// Expect the number of successful runs tracked in the callback to match the number of test targets run with no failures
EXPECT_EQ(
numSuccesses,
AZStd::count_if(m_expectedTestTargetResult.begin(), m_expectedTestTargetResult.end(), [](TestImpact::TestRunResult result) {
return result == TestImpact::TestRunResult::Passed;
}));
// Expect each job to successfully result in a test run that matches the expected test run data for that test target
for (const auto& job : runnerJobs)
{
const JobInfo::IdType jobId = job.GetJobInfo().GetId().m_value;
ValidateTestRunCompleted(job, m_expectedTestTargetResult[jobId]);
ValidateTestTargetRun(job.GetPayload().value().first, m_expectedTestTargetRuns[jobId]);
ValidateTestTargetCoverage(job.GetPayload().value().second, m_expectedTestTargetCoverages[jobId][m_coverageLevel]);
}
}
TEST_P(InstrumentedTestRunnerFixtureWithConcurrencyAndCoverageParams, JobRunnerTimeout_InFlightJobsTimeoutAndQueuedJobsUnlaunched)
{
// Given a test runner with no client callback or runner timeout and 2 second run timeout
m_testRunner = AZStd::make_unique<TestImpact::InstrumentedTestRunner>(
AZStd::nullopt, m_maxConcurrency, AZStd::chrono::seconds(2), AZStd::nullopt);
// Given an test run job for each test target where half will sleep indefinitely
for (size_t jobId = 0; jobId < m_testTargetJobArgs.size(); jobId++)
{
JobData jobData(m_testTargetPaths[jobId].m_testRunArtifact, m_testTargetPaths[jobId].m_testCoverageArtifact);
const AZStd::string args = (jobId % 2)
? AZStd::string::format("%s %s", ValidProcessPath, ConstructTestProcessArgs(jobId, LongSleep).c_str())
: m_testTargetJobArgs[jobId][m_coverageLevel];
m_jobInfos.emplace_back(JobInfo({jobId}, args, AZStd::move(jobData)));
}
// When the instrumented test run jobs are executed
const auto runnerJobs = m_testRunner->RunInstrumentedTests(m_jobInfos, CoverageExceptionPolicy::Never, JobExceptionPolicy::Never);
// Expect half the jobs to successfully result in a test run that matches the expected test run data for that test target
// with the other half having timed out
for (const auto& job : runnerJobs)
{
const JobInfo::IdType jobId = job.GetJobInfo().GetId().m_value;
if (jobId % 2)
{
ValidateJobTimeout(job);
}
else
{
ValidateTestRunCompleted(job, m_expectedTestTargetResult[jobId]);
ValidateTestTargetRun(job.GetPayload().value().first, m_expectedTestTargetRuns[jobId]);
ValidateTestTargetCoverage(job.GetPayload().value().second, m_expectedTestTargetCoverages[jobId][m_coverageLevel]);
}
}
}
TEST_F(InstrumentedTestRunnerFixture, JobTimeout_InFlightJobTimeoutAndQueuedJobsUnlaunched)
{
// Given a test runner with no client callback or run timeout and a 5 second runner timeout
m_testRunner = AZStd::make_unique<TestImpact::InstrumentedTestRunner>(
AZStd::nullopt, FourConcurrentProcesses, AZStd::nullopt, AZStd::chrono::seconds(5));
// Given an test run job for each test target where half will sleep indefinitely
for (size_t jobId = 0; jobId < m_testTargetJobArgs.size(); jobId++)
{
JobData jobData(m_testTargetPaths[jobId].m_testRunArtifact, m_testTargetPaths[jobId].m_testCoverageArtifact);
const AZStd::string args = (jobId % 2)
? AZStd::string::format("%s %s", ValidProcessPath, ConstructTestProcessArgs(jobId, LongSleep).c_str())
: m_testTargetJobArgs[jobId][m_coverageLevel];
m_jobInfos.emplace_back(JobInfo({jobId}, args, AZStd::move(jobData)));
}
// When the instrumented test run jobs are executed
const auto runnerJobs = m_testRunner->RunInstrumentedTests(m_jobInfos, CoverageExceptionPolicy::Never, JobExceptionPolicy::Never);
// Expect half the jobs to successfully result in a test run that matches the expected test run data for that test target
// with the other half having timed out
for (const auto& job : runnerJobs)
{
const JobInfo::IdType jobId = job.GetJobInfo().GetId().m_value;
if (jobId % 2)
{
ValidateJobTimeout(job);
}
else
{
ValidateTestRunCompleted(job, m_expectedTestTargetResult[jobId]);
ValidateTestTargetRun(job.GetPayload().value().first, m_expectedTestTargetRuns[jobId]);
ValidateTestTargetCoverage(job.GetPayload().value().second, m_expectedTestTargetCoverages[jobId][m_coverageLevel]);
}
}
}
INSTANTIATE_TEST_CASE_P(
,
InstrumentedTestRunnerFixtureWithConcurrencyAndCoverageAndCoverageExceptionParams,
::testing::Combine(
::testing::ValuesIn(MaxConcurrentRuns),
::testing::Values(CoverageExceptionPolicy::Never, CoverageExceptionPolicy::OnEmptyCoverage)));
INSTANTIATE_TEST_CASE_P(
,
InstrumentedTestRunnerFixtureWithConcurrencyAndCoverageAndFailedToLaunchExceptionParams,
::testing::Combine(
::testing::ValuesIn(MaxConcurrentRuns), ::testing::ValuesIn(CoverageLevels),
::testing::ValuesIn(FailedToLaunchExceptionPolicies)));
INSTANTIATE_TEST_CASE_P(
,
InstrumentedTestRunnerFixtureWithConcurrencyAndCoverageAndExecutedWithFailureExceptionParams,
::testing::Combine(
::testing::ValuesIn(MaxConcurrentRuns), ::testing::ValuesIn(CoverageLevels),
::testing::ValuesIn(ExecutedWithFailureExceptionPolicies)));
INSTANTIATE_TEST_CASE_P(
,
InstrumentedTestRunnerFixtureWithConcurrencyAndCoverageParams,
::testing::Combine(::testing::ValuesIn(MaxConcurrentRuns), ::testing::ValuesIn(CoverageLevels)));
} // namespace UnitTest
Code/Tools/TestImpactFramework/Runtime/Code/Tests/Test/TestImpactTestCoverageTest.cpp
-208
View File
@@ -1,208 +0,0 @@
/*
* All or portions of this file Copyright (c) Amazon.com, Inc. or its affiliates or
* its licensors.
*
* For complete copyright and license terms please see the LICENSE at the root of this
* distribution (the "License"). All use of this software is governed by the License,
* or, if provided, by the license below or the license accompanying this file. Do not
* remove or modify any license notices. This file is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
*
*/
#include <Test/Run/TestImpactTestCoverage.h>
#include <AzCore/UnitTest/TestTypes.h>
#include <AzTest/AzTest.h>
namespace UnitTest
{
namespace
{
AZStd::string GenerateSourcePath(AZ::u32 index)
{
return AZStd::string::format("SourceFile%u", index);
}
AZStd::string GenerateModulePath(AZ::u32 index)
{
return AZStd::string::format("Module%u", index);
}
AZStd::vector<TestImpact::LineCoverage> GenerateLineCoverages(AZ::u32 numLines)
{
AZStd::vector<TestImpact::LineCoverage> lineCoverages;
for (size_t i = 0; i < numLines; i++)
{
// Fudge some superficially different but trivially checkable line coverage data
lineCoverages.emplace_back(TestImpact::LineCoverage{i, i * 2});
}
return lineCoverages;
}
TestImpact::SourceCoverage GenerateSourceCoverage(AZ::u32 index, TestImpact::CoverageLevel coverageLevel)
{
TestImpact::SourceCoverage sourceCoverage;
sourceCoverage.m_path = GenerateSourcePath(index);
if (coverageLevel == TestImpact::CoverageLevel::Line)
{
sourceCoverage.m_coverage = GenerateLineCoverages(index + 1);
}
return sourceCoverage;
}
AZStd::vector<TestImpact::SourceCoverage> GenerateSourceCoverages(AZ::u32 numSources, TestImpact::CoverageLevel coverageLevel)
{
AZStd::vector<TestImpact::SourceCoverage> sourceCoverages;
for (AZ::u32 i = 0; i < numSources; i++)
{
sourceCoverages.emplace_back(GenerateSourceCoverage(i, coverageLevel));
}
return sourceCoverages;
}
TestImpact::ModuleCoverage GenerateModuleCoverage(AZ::u32 index, AZ::u32 numSources, TestImpact::CoverageLevel coverageLevel)
{
TestImpact::ModuleCoverage moduleCoverage;
moduleCoverage.m_path = GenerateModulePath(index);
moduleCoverage.m_sources = GenerateSourceCoverages(numSources, coverageLevel);
return moduleCoverage;
}
AZStd::vector<TestImpact::ModuleCoverage> GenerateModuleCoverages(AZ::u32 numModules, TestImpact::CoverageLevel coverageLevel)
{
AZStd::vector<TestImpact::ModuleCoverage> moduleCoverages;
for (AZ::u32 i = 0; i < numModules; i++)
{
// Fudge some superficially different but trivially deducible module coverage data
moduleCoverages.emplace_back(GenerateModuleCoverage(i, i + 1, coverageLevel));
}
return moduleCoverages;
}
} // namespace
using CoveragePermutation = AZStd::tuple
<
unsigned, // Number of modules covered
TestImpact::CoverageLevel // Test coverage level
>;
// Fixture parameterized for different max number of concurrent jobs
class TestCoverageFixtureWithCoverageParams
: public AllocatorsTestFixture
, public ::testing::WithParamInterface<CoveragePermutation>
{
public:
void SetUp() override;
protected:
void ValidateTestCoverage(const TestImpact::TestCoverage& testCoverage);
size_t m_numModulesCovered;
TestImpact::CoverageLevel m_coverageLevel;
};
void TestCoverageFixtureWithCoverageParams::SetUp()
{
AllocatorsTestFixture::SetUp();
const auto& [numModulesCovered, coverageLevel] = GetParam();
m_numModulesCovered = numModulesCovered;
m_coverageLevel = coverageLevel;
}
void TestCoverageFixtureWithCoverageParams::ValidateTestCoverage(const TestImpact::TestCoverage& testCoverage)
{
// Expect the coverage level to match that which was used to generate the module coverages generated
EXPECT_EQ(testCoverage.GetCoverageLevel(), m_coverageLevel);
// Expect the number of modules covered to match the number of module coverages generated
EXPECT_EQ(testCoverage.GetNumModulesCovered(), m_numModulesCovered);
// Expect the number of unique sources covered to match the number of modules coverages generated
EXPECT_EQ(testCoverage.GetNumSourcesCovered(), m_numModulesCovered);
// Expect the unique sources covered to match the procedurally generated source paths
for (size_t sourceIndex = 0; sourceIndex < testCoverage.GetNumSourcesCovered(); sourceIndex++)
{
EXPECT_EQ(testCoverage.GetSourcesCovered()[sourceIndex], GenerateSourcePath(sourceIndex));
}
// Expect each module covered to match that of the corresponding procedurally generated modules
for (size_t moduleIndex = 0; moduleIndex < testCoverage.GetNumModulesCovered(); moduleIndex++)
{
const TestImpact::ModuleCoverage& moduleCoverage = testCoverage.GetModuleCoverages()[moduleIndex];
// Expect the module path to match that of the corresponding procedurally generated module
EXPECT_EQ(moduleCoverage.m_path, GenerateModulePath(moduleIndex));
// Expect the module's number of sources to match that of the corresponding procedurally generated module
EXPECT_EQ(moduleCoverage.m_sources.size(), moduleIndex + 1);
for (size_t sourceIndex = 0; sourceIndex < moduleCoverage.m_sources.size(); sourceIndex++)
{
const TestImpact::SourceCoverage& sourceCoverage = moduleCoverage.m_sources[sourceIndex];
// Expect the source path to match the procedurally generated source path
EXPECT_EQ(sourceCoverage.m_path, GenerateSourcePath(sourceIndex));
if (m_coverageLevel == TestImpact::CoverageLevel::Line)
{
// Expect there to actually be line coverage data if this coverage was procedurally generated with line data
EXPECT_FALSE(sourceCoverage.m_coverage.empty());
const AZStd::vector<TestImpact::LineCoverage>& lineCoverages = sourceCoverage.m_coverage;
// Expect the source's number of lines to match that of the corresponding procedurally generated source
EXPECT_EQ(lineCoverages.size(), sourceIndex + 1);
for (size_t lineIndex = 0; lineIndex < lineCoverages.size(); lineIndex++)
{
// The expected line number and hit count are deduced as follows:
// Line number: line index
// Hit count: 2x line index
EXPECT_EQ(lineCoverages[lineIndex].m_lineNumber, lineIndex);
EXPECT_EQ(lineCoverages[lineIndex].m_hitCount, lineIndex * 2);
}
}
else
{
// Do not expect there to actually be line coverage data if this coverage was not procedurally generated with line data
EXPECT_TRUE(sourceCoverage.m_coverage.empty());
}
}
}
}
TEST(TestCoverage, EmptyCoverage_ExpectTestRunException)
{
// When constructing a test coverage from the empty module coverages
TestImpact::TestCoverage testCoverage(AZStd::vector<TestImpact::ModuleCoverage>{});
// Expect the test coverage fields to be empty
EXPECT_EQ(testCoverage.GetNumModulesCovered(), 0);
EXPECT_EQ(testCoverage.GetNumSourcesCovered(), 0);
EXPECT_TRUE(testCoverage.GetModuleCoverages().empty());
EXPECT_TRUE(testCoverage.GetSourcesCovered().empty());
}
TEST_P(TestCoverageFixtureWithCoverageParams, AllCoveragePermutations_ExpectTestCoverageMetaDatasToMatchPermutations)
{
// Given a procedurally generated test coverage
const TestImpact::TestCoverage testCoverage(GenerateModuleCoverages(m_numModulesCovered, m_coverageLevel));
// Expect the test coverage data and meta-data to match that of the rules used to procedurally generate the coverage data
ValidateTestCoverage(testCoverage);
}
INSTANTIATE_TEST_CASE_P(
,
TestCoverageFixtureWithCoverageParams,
::testing::Combine(
::testing::Range(1u, 11u),
::testing::Values(TestImpact::CoverageLevel::Line, TestImpact::CoverageLevel::Source))
);
} // namespace UnitTest
Code/Tools/TestImpactFramework/Runtime/Code/Tests/Test/TestImpactTestEnumeratorTest.cpp
-890
View File
@@ -1,890 +0,0 @@
/*
* All or portions of this file Copyright (c) Amazon.com, Inc. or its affiliates or
* its licensors.
*
* For complete copyright and license terms please see the LICENSE at the root of this
* distribution (the "License"). All use of this software is governed by the License,
* or, if provided, by the license below or the license accompanying this file. Do not
* remove or modify any license notices. This file is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
*
*/
#include <TestImpactTestJobRunnerCommon.h>
#include <TestImpactTestUtils.h>
#include <Test/Enumeration/TestImpactTestEnumerationException.h>
#include <Test/Enumeration/TestImpactTestEnumerationSerializer.h>
#include <Test/Enumeration/TestImpactTestEnumerator.h>
#include <AzCore/IO/Path/Path.h>
#include <AzCore/UnitTest/TestTypes.h>
#include <AzCore/std/containers/array.h>
#include <AzCore/std/containers/vector.h>
#include <AzCore/std/optional.h>
#include <AzCore/std/smart_ptr/unique_ptr.h>
#include <AzCore/std/string/string.h>
#include <AzTest/AzTest.h>
namespace UnitTest
{
using JobExceptionPolicy = TestImpact::TestEnumerator::JobExceptionPolicy;
using CacheExceptionPolicy = TestImpact::TestEnumerator::CacheExceptionPolicy;
// Generates the command to run the given test target through AzTestRunner and get gtest to output the enumeration file
AZStd::string GetEnumerateCommandForTarget(AZStd::pair<AZ::IO::Path, AZ::IO::Path> testTarget)
{
return AZStd::string::format(
"%s %s AzRunUnitTests --gtest_list_tests --gtest_output=xml:%s",
LY_TEST_IMPACT_AZ_TESTRUNNER_BIN,
testTarget.first.c_str(), // Path to test target bin
testTarget.second.c_str()); // Path to test target gtest enumeration file
}
class TestEnumeratorFixture
: public AllocatorsTestFixture
{
public:
void SetUp() override;
protected:
using JobInfo = TestImpact::TestEnumerator::JobInfo;
using JobData = TestImpact::TestEnumerator::JobData;
AZStd::vector<JobInfo> m_jobInfos;
AZStd::unique_ptr<TestImpact::TestEnumerator> m_testEnumerator;
AZStd::vector<AZStd::string> m_testTargetJobArgs;
AZStd::vector<AZStd::pair<AZ::IO::Path, AZ::IO::Path>> m_testTargetPaths;
AZStd::vector<TestImpact::TestEnumeration> m_expectedTestTargetEnumerations;
AZStd::vector<AZStd::string> m_cacheFiles;
size_t m_maxConcurrency = 0;
};
void TestEnumeratorFixture::SetUp()
{
UnitTest::AllocatorsTestFixture::SetUp();
DeleteFiles(LY_TEST_IMPACT_TEST_TARGET_ENUMERATION_DIR, "*.cache");
DeleteFiles(LY_TEST_IMPACT_TEST_TARGET_ENUMERATION_DIR, "*.xml");
// first: path to test target bin
// second: path to test target gtest enumeration file in XML format
const AZStd::string enumPath = AZStd::string(LY_TEST_IMPACT_TEST_TARGET_ENUMERATION_DIR) + "/%s.Enumeration.xml";
m_testTargetPaths.emplace_back(
LY_TEST_IMPACT_TEST_TARGET_A_BIN, AZStd::string::format(enumPath.c_str(), LY_TEST_IMPACT_TEST_TARGET_A_BASE_NAME));
m_testTargetPaths.emplace_back(
LY_TEST_IMPACT_TEST_TARGET_B_BIN, AZStd::string::format(enumPath.c_str(), LY_TEST_IMPACT_TEST_TARGET_B_BASE_NAME));
m_testTargetPaths.emplace_back(
LY_TEST_IMPACT_TEST_TARGET_C_BIN, AZStd::string::format(enumPath.c_str(), LY_TEST_IMPACT_TEST_TARGET_C_BASE_NAME));
m_testTargetPaths.emplace_back(
LY_TEST_IMPACT_TEST_TARGET_D_BIN, AZStd::string::format(enumPath.c_str(), LY_TEST_IMPACT_TEST_TARGET_D_BASE_NAME));
m_expectedTestTargetEnumerations.emplace_back(GetTestTargetATestEnumerationSuites());
m_expectedTestTargetEnumerations.emplace_back(GetTestTargetBTestEnumerationSuites());
m_expectedTestTargetEnumerations.emplace_back(GetTestTargetCTestEnumerationSuites());
m_expectedTestTargetEnumerations.emplace_back(GetTestTargetDTestEnumerationSuites());
// Path to enumeration file in TIAF internal JSON format
m_cacheFiles.emplace_back(
AZStd::string::format("%s/%s.cache", LY_TEST_IMPACT_TEST_TARGET_ENUMERATION_DIR, LY_TEST_IMPACT_TEST_TARGET_A_BASE_NAME));
m_cacheFiles.emplace_back(
AZStd::string::format("%s/%s.cache", LY_TEST_IMPACT_TEST_TARGET_ENUMERATION_DIR, LY_TEST_IMPACT_TEST_TARGET_B_BASE_NAME));
m_cacheFiles.emplace_back(
AZStd::string::format("%s/%s.cache", LY_TEST_IMPACT_TEST_TARGET_ENUMERATION_DIR, LY_TEST_IMPACT_TEST_TARGET_C_BASE_NAME));
m_cacheFiles.emplace_back(
AZStd::string::format("%s/%s.cache", LY_TEST_IMPACT_TEST_TARGET_ENUMERATION_DIR, LY_TEST_IMPACT_TEST_TARGET_D_BASE_NAME));
for (const auto& testTarget : m_testTargetPaths)
{
m_testTargetJobArgs.emplace_back(GetEnumerateCommandForTarget(testTarget));
}
}
// Fixture parameterized for different max number of concurrent jobs
class TestEnumeratorFixtureWithConcurrencyParams
: public TestEnumeratorFixture
, public ::testing::WithParamInterface<size_t>
{
public:
void SetUp() override
{
TestEnumeratorFixture::SetUp();
m_maxConcurrency = GetParam();
}
};
using ConcurrencyAndJobExceptionPermutation = AZStd::tuple
<
size_t, // Max number of concurrent processes
JobExceptionPolicy // Test job exception policy
>;
// Fixture parameterized for different max number of concurrent jobs and different job exception policies
class TestEnumeratorFixtureWithConcurrencyAndJobExceptionParams
: public TestEnumeratorFixture
, public ::testing::WithParamInterface<ConcurrencyAndJobExceptionPermutation>
{
public:
void SetUp() override
{
TestEnumeratorFixture::SetUp();
const auto& [maxConcurrency, jobExceptionPolicy] = GetParam();
m_maxConcurrency = maxConcurrency;
m_jobExceptionPolicy = jobExceptionPolicy;
}
protected:
JobExceptionPolicy m_jobExceptionPolicy = JobExceptionPolicy::Never;
};
using ConcurrencyAndCacheExceptionPermutation = AZStd::tuple
<
size_t, // Max number of concurrent processes
CacheExceptionPolicy // Test enumeration exception policy
>;
// Fixture parameterized for different max number of concurrent jobs and different enumeration exception policies
class TestEnumeratorFixtureWithConcurrencyAndCacheExceptionParams
: public TestEnumeratorFixture
, public ::testing::WithParamInterface<ConcurrencyAndCacheExceptionPermutation>
{
public:
void SetUp() override
{
TestEnumeratorFixture::SetUp();
const auto& [maxConcurrency, cacheExceptionPolicy] = GetParam();
m_maxConcurrency = maxConcurrency;
m_cacheExceptionPolicy = cacheExceptionPolicy;
}
protected:
CacheExceptionPolicy m_cacheExceptionPolicy = CacheExceptionPolicy::Never;
};
namespace
{
AZStd::array<size_t, 4> MaxConcurrentEnumerations = {{1, 2, 3, 4}};
AZStd::array<JobExceptionPolicy, 3> JobExceptionPolicies =
{
JobExceptionPolicy::Never,
JobExceptionPolicy::OnExecutedWithFailure,
JobExceptionPolicy::OnFailedToExecute
};
AZStd::array<CacheExceptionPolicy, 4> CacheExceptionPolicies =
{
CacheExceptionPolicy::Never,
CacheExceptionPolicy::OnCacheNotExist,
CacheExceptionPolicy::OnCacheReadFailure,
CacheExceptionPolicy::OnCacheWriteFailure
};
}
// Validates that the specified job successfully read from its test enumeration cache
void ValidateJobSuccessfulCacheRead(const TestImpact::TestEnumerator::Job& job)
{
EXPECT_EQ(job.GetResult(), TestImpact::JobResult::NotExecuted);
EXPECT_EQ(job.GetStartTime(), AZStd::chrono::high_resolution_clock::time_point());
EXPECT_EQ(job.GetEndTime(), AZStd::chrono::high_resolution_clock::time_point());
EXPECT_EQ(job.GetDuration(), AZStd::chrono::milliseconds(0));
EXPECT_FALSE(job.GetReturnCode().has_value());
EXPECT_TRUE(job.GetPayload().has_value());
}
// Validates that the specified test enumeration matched the expected output
void ValidateTestTargetEnumeration(const TestImpact::TestEnumeration& actualResult, const TestImpact::TestEnumeration& expectedResult)
{
EXPECT_TRUE(actualResult == expectedResult);
EXPECT_EQ(actualResult.GetNumTestSuites(), CalculateNumTestSuites(expectedResult.GetTestSuites()));
EXPECT_EQ(actualResult.GetNumTests(), CalculateNumTests(expectedResult.GetTestSuites()));
EXPECT_EQ(actualResult.GetNumEnabledTests(), CalculateNumEnabledTests(expectedResult.GetTestSuites()));
EXPECT_EQ(actualResult.GetNumDisabledTests(), CalculateNumDisabledTests(expectedResult.GetTestSuites()));
}
// Validates that the specified test enumeration cache matches the expected output
void ValidateTestEnumerationCache(const AZ::IO::Path& cacheFile, const TestImpact::TestEnumeration& expectedEnumeration)
{
// Cache file must exist
const auto fileSize = AZ::IO::SystemFile::Length(cacheFile.c_str());
EXPECT_GT(fileSize, 0);
// Read raw byte data from cache
AZStd::vector<char> buffer(fileSize + 1);
buffer[fileSize] = 0;
EXPECT_TRUE(AZ::IO::SystemFile::Read(cacheFile.c_str(), buffer.data()));
// Transform raw byte data to raw string data and attempt to construct the test enumeration
AZStd::string rawEnum(buffer.begin(), buffer.end());
TestImpact::TestEnumeration actualEnumeration = TestImpact::DeserializeTestEnumeration(rawEnum);
// Check that the constructed test enumeration matches the expected enumeration
ValidateTestTargetEnumeration(actualEnumeration, expectedEnumeration);
}
// Validates that the specified cache file does not exist
void ValidateInvalidTestEnumerationCache(const AZ::IO::Path& cacheFile)
{
EXPECT_FALSE(AZ::IO::SystemFile::Exists(cacheFile.c_str()));
}
TEST_P(
TestEnumeratorFixtureWithConcurrencyAndJobExceptionParams, InvalidCommandArgument_ExpectJobResulFailedToExecuteeOrTestJobException)
{
// Given a test enumerator with no client callback or enumeration timeout or enumerator timeout
m_testEnumerator = AZStd::make_unique<TestImpact::TestEnumerator>(
AZStd::nullopt,
m_maxConcurrency,
AZStd::nullopt,
AZStd::nullopt);
// Given a mixture of test enumeration jobs with valid and invalid command arguments
for (size_t jobId = 0; jobId < m_testTargetJobArgs.size(); jobId++)
{
const AZStd::string args = (jobId % 2) ? InvalidProcessPath : m_testTargetJobArgs[jobId];
JobData jobData(m_testTargetPaths[jobId].second, AZStd::nullopt);
m_jobInfos.emplace_back(JobInfo({jobId}, args, AZStd::move(jobData)));
}
try
{
// When the test enumeration jobs are executed with different exception policies
const auto enumerationJobs = m_testEnumerator->Enumerate(m_jobInfos, CacheExceptionPolicy::Never, m_jobExceptionPolicy);
// Expect this statement to be reachable only if no exception policy for launch failures
EXPECT_FALSE(::IsFlagSet(m_jobExceptionPolicy, JobExceptionPolicy::OnFailedToExecute));
for (const auto& job : enumerationJobs)
{
const JobInfo::IdType jobId = job.GetJobInfo().GetId().m_value;
if (jobId % 2)
{
// Expect invalid jobs have a job result of FailedToExecute
ValidateJobFailedToExecute(job);
}
else
{
// Expect the valid jobs job to successfully result in a test enumeration that matches the expected test enumeration data
ValidateJobExecutedSuccessfully(job);
ValidateTestTargetEnumeration(job.GetPayload().value(), m_expectedTestTargetEnumerations[jobId]);
}
}
}
catch ([[maybe_unused]] const TestImpact::TestJobException& e)
{
// Expect this statement to be reachable only if there is an exception policy for launch failures
EXPECT_TRUE(::IsFlagSet(m_jobExceptionPolicy, JobExceptionPolicy::OnFailedToExecute));
}
catch (...)
{
// Do not expect any other exceptions
FAIL();
}
}
TEST_P(
TestEnumeratorFixtureWithConcurrencyAndJobExceptionParams, ErroneousReturnCode_ExpectJobResultExecutedWithFailureOrTestJobException)
{
// Given a test enumerator with no client callback or enumeration timeout or enumerator timeout
m_testEnumerator = AZStd::make_unique<TestImpact::TestEnumerator>(
AZStd::nullopt,
m_maxConcurrency,
AZStd::nullopt,
AZStd::nullopt);
// Given a mixture of test enumeration jobs that execute and return either successfully or with failure
for (size_t jobId = 0; jobId < m_testTargetJobArgs.size(); jobId++)
{
JobData jobData(m_testTargetPaths[jobId].second, AZStd::nullopt);
const AZStd::string args = (jobId % 2)
? AZStd::string::format("%s %s", ValidProcessPath, ConstructTestProcessArgs(jobId, AZStd::chrono::milliseconds(0)).c_str())
: m_testTargetJobArgs[jobId];
m_jobInfos.emplace_back(JobInfo({jobId}, args, AZStd::move(jobData)));
}
try
{
// When the test enumeration jobs are executed with different exception policies
const auto enumerationJobs = m_testEnumerator->Enumerate(m_jobInfos, CacheExceptionPolicy::Never, m_jobExceptionPolicy);
// Expect this statement to be reachable only if no exception policy for jobs that return with error
EXPECT_FALSE(::IsFlagSet(m_jobExceptionPolicy, JobExceptionPolicy::OnExecutedWithFailure));
for (const auto& job : enumerationJobs)
{
const JobInfo::IdType jobId = job.GetJobInfo().GetId().m_value;
if (jobId % 2)
{
// Expect failed jobs to have job result ExecutedWithFailure and a non-zero return code
ValidateJobExecutedWithFailure(job);
}
else
{
// Expect the valid jobs job to successfully result in a test enumeration that matches the expected test enumeration data
ValidateJobExecutedSuccessfully(job);
ValidateTestTargetEnumeration(job.GetPayload().value(), m_expectedTestTargetEnumerations[jobId]);
}
}
}
catch ([[maybe_unused]] const TestImpact::TestJobException& e)
{
// Expect this statement to be reachable only if there is an exception policy for jobs that return with error
EXPECT_TRUE(::IsFlagSet(m_jobExceptionPolicy, JobExceptionPolicy::OnExecutedWithFailure));
}
catch (...)
{
// Do not expect any other exceptions
FAIL();
}
}
TEST_P(TestEnumeratorFixtureWithConcurrencyAndCacheExceptionParams, EmptyCacheRead_NoCacheDataButEnumerationsMatchTestSuitesInTarget)
{
// Given a test enumerator with no client callback, enumeration timeout or enumerator timeout
m_testEnumerator = AZStd::make_unique<TestImpact::TestEnumerator>(
AZStd::nullopt,
m_maxConcurrency,
AZStd::nullopt,
AZStd::nullopt);
// Given an test enumeration job for each test target that reads from an enumeration caching
for (size_t jobId = 0; jobId < m_testTargetJobArgs.size(); jobId++)
{
JobData jobData(m_testTargetPaths[jobId].second, JobData::Cache{JobData::CachePolicy::Read, m_cacheFiles[jobId]});
m_jobInfos.emplace_back(JobInfo({jobId}, m_testTargetJobArgs[jobId], AZStd::move(jobData)));
}
try
{
// When the test enumeration jobs are executed with different exception policies
const auto enumerationJobs = m_testEnumerator->Enumerate(m_jobInfos, m_cacheExceptionPolicy, JobExceptionPolicy::Never);
// Expect this statement to be reachable only if no exception policy for read attempts of non-existent caches
EXPECT_FALSE(::IsFlagSet(m_cacheExceptionPolicy, CacheExceptionPolicy::OnCacheNotExist));
// Expect each job to successfully result in a test enumeration that matches the expected test enumeration data for that test target
// even though the cache files could not be read
for (const auto& job : enumerationJobs)
{
const JobInfo::IdType jobId = job.GetJobInfo().GetId().m_value;
ValidateJobExecutedSuccessfully(job);
ValidateTestTargetEnumeration(job.GetPayload().value(), m_expectedTestTargetEnumerations[jobId]);
ValidateInvalidTestEnumerationCache(job.GetJobInfo().GetCache()->m_file);
}
}
catch ([[maybe_unused]] const TestImpact::TestEnumerationException& e)
{
// Expect this statement to be reachable only if there is an exception policy for read attempts of non-existent caches
EXPECT_TRUE(::IsFlagSet(m_cacheExceptionPolicy, CacheExceptionPolicy::OnCacheNotExist));
}
catch (...)
{
// Do not expect any other exceptions
FAIL();
}
}
// Note: this test only cues up one test for enumeration but still runs the permutations for max concurrency so there is duplicated work
TEST_P(
TestEnumeratorFixtureWithConcurrencyAndCacheExceptionParams,
EmptyCacheDataRead_ExpectEnumerationsMatchTestSuitesInTargetOrTestEnumerationException)
{
// Given an enumeration cache for Test Target A with invalid JSON data
WriteTextToFile("", m_cacheFiles[TestTargetA]);
// Given a test enumerator with no client callback, enumeration timeout or enumerator timeout
m_testEnumerator = AZStd::make_unique<TestImpact::TestEnumerator>(
AZStd::nullopt,
m_maxConcurrency,
AZStd::nullopt,
AZStd::nullopt);
// Given an test enumeration job that will attempt to read an invalid enumeration cache
JobData jobData(m_testTargetPaths[TestTargetA].second, JobData::Cache{JobData::CachePolicy::Read, m_cacheFiles[TestTargetA]});
m_jobInfos.emplace_back(JobInfo({TestTargetA}, m_testTargetJobArgs[TestTargetA], AZStd::move(jobData)));
try
{
// When the test enumeration jobs are executed with different exception policies
const auto enumerationJobs = m_testEnumerator->Enumerate(m_jobInfos, m_cacheExceptionPolicy, JobExceptionPolicy::Never);
// Expect this statement to be reachable only if no exception policy for cache reads that fail
EXPECT_FALSE(::IsFlagSet(m_cacheExceptionPolicy, CacheExceptionPolicy::OnCacheReadFailure));
for (const auto& job : enumerationJobs)
{
const JobInfo::IdType jobId = job.GetJobInfo().GetId().m_value;
// Expect the valid jobs job to successfully result in a test enumeration that matches the expected test enumeration data
ValidateJobExecutedSuccessfully(job);
ValidateTestTargetEnumeration(job.GetPayload().value(), m_expectedTestTargetEnumerations[jobId]);
}
}
catch ([[maybe_unused]] const TestImpact::TestEnumerationException& e)
{
// Expect this statement to be reachable only if there is an exception policy for cache reads that fail
EXPECT_TRUE(::IsFlagSet(m_cacheExceptionPolicy, CacheExceptionPolicy::OnCacheReadFailure));
}
catch (...)
{
// Do not expect any other exceptions
FAIL();
}
}
TEST_P(
TestEnumeratorFixtureWithConcurrencyAndCacheExceptionParams,
InvalidCacheWrite_ExpectEnumerationsMatchTestSuitesInTargetOrTestEnumerationException)
{
// Given a test enumerator with no client callback,, enumeration timeout or enumerator timeout
m_testEnumerator = AZStd::make_unique<TestImpact::TestEnumerator>(
AZStd::nullopt,
m_maxConcurrency,
AZStd::nullopt,
AZStd::nullopt);
// Given an test enumeration job that will attempt to write to an invalid enumeration cache
JobData jobData(m_testTargetPaths[TestTargetA].second, JobData::Cache{JobData::CachePolicy::Write, InvalidProcessPath});
m_jobInfos.emplace_back(JobInfo({TestTargetA}, m_testTargetJobArgs[TestTargetA], AZStd::move(jobData)));
try
{
// When the test enumeration job is executed
const auto enumerationJobs = m_testEnumerator->Enumerate(m_jobInfos, m_cacheExceptionPolicy, JobExceptionPolicy::Never);
// Expect this statement to be reachable only if no exception policy for cache writes that fail
EXPECT_FALSE(::IsFlagSet(m_cacheExceptionPolicy, CacheExceptionPolicy::OnCacheWriteFailure));
for (const auto& job : enumerationJobs)
{
const JobInfo::IdType jobId = job.GetJobInfo().GetId().m_value;
// Expect the valid jobs job to successfully result in a test enumeration that matches the expected test enumeration data
ValidateJobExecutedSuccessfully(job);
ValidateTestTargetEnumeration(job.GetPayload().value(), m_expectedTestTargetEnumerations[jobId]);
}
}
catch ([[maybe_unused]] const TestImpact::TestEnumerationException& e)
{
// Expect this statement to be reachable only if there is an exception policy for cache writes that fail
EXPECT_TRUE(::IsFlagSet(m_cacheExceptionPolicy, CacheExceptionPolicy::OnCacheWriteFailure));
}
catch (...)
{
// Do not expect any other exceptions
FAIL();
}
}
TEST_P(TestEnumeratorFixtureWithConcurrencyParams, ValidAndInvalidCacheRead_CachedEnumerationsMatchTestSuitesInTarget)
{
// Given the cache file written for only test target B
WriteTextToFile(TestImpact::SerializeTestEnumeration(m_expectedTestTargetEnumerations[TestTargetB]), m_cacheFiles[TestTargetB]);
// Given a test enumerator with no client callback, enumeration timeout or enumerator timeout
m_testEnumerator = AZStd::make_unique<TestImpact::TestEnumerator>(
AZStd::nullopt,
m_maxConcurrency,
AZStd::nullopt,
AZStd::nullopt);
// Given test enumeration jobs test target A and D with no enumeration caching
m_jobInfos.emplace_back(
JobInfo({TestTargetA}, m_testTargetJobArgs[TestTargetA], JobData{m_testTargetPaths[TestTargetA].second, AZStd::nullopt}));
m_jobInfos.emplace_back(
JobInfo({TestTargetD}, m_testTargetJobArgs[TestTargetD], JobData{m_testTargetPaths[TestTargetD].second, AZStd::nullopt}));
// Given test target B with enumeration cache reading and a valid cache file
m_jobInfos.emplace_back(JobInfo(
{TestTargetB}, m_testTargetJobArgs[TestTargetB],
JobData{m_testTargetPaths[TestTargetB].second, JobInfo::Cache{JobData::CachePolicy::Read, m_cacheFiles[TestTargetB]}}));
// Given test target C with enumeration cache reading and an invalid cache file
m_jobInfos.emplace_back(JobInfo(
{TestTargetC}, m_testTargetJobArgs[TestTargetC],
JobData{m_testTargetPaths[TestTargetC].second, JobInfo::Cache{JobData::CachePolicy::Read, "nothing"}}));
// When the test enumeration jobs are executed
const auto enumerationJobs = m_testEnumerator->Enumerate(m_jobInfos, CacheExceptionPolicy::Never, JobExceptionPolicy::Never);
// Expect each job to successfully result in a test enumeration that matches the expected test enumeration data for that test target
for (const auto& job : enumerationJobs)
{
const JobInfo::IdType jobId = job.GetJobInfo().GetId().m_value;
switch (jobId)
{
case TestTargetA: // No cache read
case TestTargetC: // Cache read, but invalid cache, so re-enumerate anyway
case TestTargetD: // No cache read
{
ValidateJobExecutedSuccessfully(job);
break;
}
case TestTargetB: // Cache read, successful cache read, so job not executed
{
ValidateJobSuccessfulCacheRead(job);
break;
}
default:
{
FAIL();
}
}
// Regardless of cache policy and cache failures all targets should still produce the expected test enumerations
ValidateTestTargetEnumeration(job.GetPayload().value(), m_expectedTestTargetEnumerations[jobId]);
}
}
TEST_F(TestEnumeratorFixture, InvalidCacheDataRead_TestEnumerationException)
{
// Given an enumeration cache for Test Target A with invalid JSON data
WriteTextToFile("There is no valid cache data here", m_cacheFiles[TestTargetA]);
// Given a test enumerator with no client callback, concurrency, enumeration timeout or enumerator timeout
m_testEnumerator = AZStd::make_unique<TestImpact::TestEnumerator>(
AZStd::nullopt,
OneConcurrentProcess,
AZStd::nullopt,
AZStd::nullopt);
// Given an test enumeration job that will attempt to read an invalid enumeration cache
JobData jobData(m_testTargetPaths[TestTargetA].second, JobData::Cache{JobData::CachePolicy::Read, m_cacheFiles[TestTargetA]});
m_jobInfos.emplace_back(JobInfo({TestTargetA}, m_testTargetJobArgs[TestTargetA], AZStd::move(jobData)));
try
{
// When the test enumeration jobs are executed with different exception policies
const auto enumerationJobs = m_testEnumerator->Enumerate(m_jobInfos, CacheExceptionPolicy::Never, JobExceptionPolicy::Never);
}
catch ([[maybe_unused]] const TestImpact::TestEnumerationException& e)
{
// Expect this statement to be reachable only if there is an exception policy for jobs that return with error
SUCCEED();
}
catch (...)
{
// Do not expect any other exceptions
FAIL();
}
}
TEST_P(TestEnumeratorFixtureWithConcurrencyParams, ValidCacheWrite_CachedEnumerationsMatchTestSuitesInTarget)
{
// Given a test enumerator with no client callback, enumeration timeout or enumerator timeout
m_testEnumerator = AZStd::make_unique<TestImpact::TestEnumerator>(
AZStd::nullopt,
m_maxConcurrency,
AZStd::nullopt,
AZStd::nullopt);
// Given an test enumeration job for each test target with write enumeration caching
for (size_t jobId = 0; jobId < m_testTargetJobArgs.size(); jobId++)
{
JobData jobData(m_testTargetPaths[jobId].second, JobData::Cache{JobData::CachePolicy::Write, m_cacheFiles[jobId]});
m_jobInfos.emplace_back(JobInfo({jobId}, m_testTargetJobArgs[jobId], AZStd::move(jobData)));
}
// When the test enumeration jobs are executed
const auto enumerationJobs = m_testEnumerator->Enumerate(m_jobInfos, CacheExceptionPolicy::Never, JobExceptionPolicy::Never);
// Expect each job to successfully result in a test enumeration and cache that matches the expected test enumeration data for that
// test target
for (const auto& job : enumerationJobs)
{
const JobInfo::IdType jobId = job.GetJobInfo().GetId().m_value;
ValidateJobExecutedSuccessfully(job);
ValidateTestTargetEnumeration(job.GetPayload().value(), m_expectedTestTargetEnumerations[jobId]);
ValidateTestEnumerationCache(job.GetJobInfo().GetCache()->m_file, m_expectedTestTargetEnumerations[jobId]);
}
}
TEST_F(TestEnumeratorFixture, EmptyArtifact_ExpectTestEnumerationException)
{
// Given a test enumerator with no client callback, concurrency, enumeration timeout or enumerator timeout
m_testEnumerator = AZStd::make_unique<TestImpact::TestEnumerator>(
AZStd::nullopt,
OneConcurrentProcess,
AZStd::nullopt,
AZStd::nullopt);
// Given an test enumeration job that will return successfully but with an empty artifact string
m_jobInfos.emplace_back(JobInfo({0}, m_testTargetJobArgs[0], JobData("", AZStd::nullopt)));
try
{
// When the test enumeration job is executed
const auto enumerationJobs = m_testEnumerator->Enumerate(m_jobInfos, CacheExceptionPolicy::Never, JobExceptionPolicy::Never);
// Do not expect this statement to be reachable
FAIL();
}
catch ([[maybe_unused]] const TestImpact::TestEnumerationException& e)
{
// Expect an enumeration exception
SUCCEED();
}
catch (...)
{
// Do not expect any other exceptions
FAIL();
}
}
TEST_F(TestEnumeratorFixture, InvalidArtifact_ExpectTestEnumerationException)
{
// Given a test enumerator with no client callback, concurrency, enumeration timeout or enumerator timeout
m_testEnumerator = AZStd::make_unique<TestImpact::TestEnumerator>(
AZStd::nullopt,
OneConcurrentProcess,
AZStd::nullopt,
AZStd::nullopt);
// Given an test enumeration job that will return successfully but not produce an artifact
m_jobInfos.emplace_back(JobInfo(
{0}, AZStd::string::format("%s %s", ValidProcessPath, ConstructTestProcessArgs(0, AZStd::chrono::milliseconds(0)).c_str()),
JobData("", AZStd::nullopt)));
try
{
// When the test enumeration job is executed
const auto enumerationJobs = m_testEnumerator->Enumerate(m_jobInfos, CacheExceptionPolicy::Never, JobExceptionPolicy::Never);
// Do not expect this statement to be reachable
FAIL();
}
catch ([[maybe_unused]] const TestImpact::TestEnumerationException& e)
{
// Expect an enumeration exception
SUCCEED();
}
catch (...)
{
// Do not expect any other exceptions
FAIL();
}
}
TEST_P(TestEnumeratorFixtureWithConcurrencyParams, EnumerateTestTargets_EnumerationsMatchTestSuitesInTarget)
{
// Given a test enumerator with no client callback, enumeration timeout or enumerator timeout
m_testEnumerator = AZStd::make_unique<TestImpact::TestEnumerator>(
AZStd::nullopt,
m_maxConcurrency,
AZStd::nullopt,
AZStd::nullopt);
// Given an test enumeration job for each test target with no enumeration caching
for (size_t jobId = 0; jobId < m_testTargetJobArgs.size(); jobId++)
{
JobData jobData(m_testTargetPaths[jobId].second, AZStd::nullopt);
m_jobInfos.emplace_back(JobInfo({jobId}, m_testTargetJobArgs[jobId], AZStd::move(jobData)));
}
// When the test enumeration jobs are executed
const auto enumerationJobs = m_testEnumerator->Enumerate(m_jobInfos, CacheExceptionPolicy::Never, JobExceptionPolicy::Never);
// Expect each job to successfully result in a test enumeration that matches the expected test enumeration data for that test target
for (const auto& job : enumerationJobs)
{
const JobInfo::IdType jobId = job.GetJobInfo().GetId().m_value;
ValidateJobExecutedSuccessfully(job);
ValidateTestTargetEnumeration(job.GetPayload().value(), m_expectedTestTargetEnumerations[jobId]);
}
}
TEST_P(TestEnumeratorFixtureWithConcurrencyParams, EnumerateTestTargetsWithArbitraryJobIds_EnumerationsMatchTestSuitesInTarget)
{
// Given a set of arbitrary job ids to be used for the test target jobs
enum
{
ArbitraryA = 36,
ArbitraryB = 890,
ArbitraryC = 19,
ArbitraryD = 1
};
const AZStd::unordered_map<JobInfo::IdType, JobInfo::IdType> sequentialToArbitrary =
{
{TestTargetA, ArbitraryA},
{TestTargetB, ArbitraryB},
{TestTargetC, ArbitraryC},
{TestTargetD, ArbitraryD},
};
const AZStd::unordered_map<JobInfo::IdType, JobInfo::IdType> arbitraryToSequential =
{
{ArbitraryA, TestTargetA},
{ArbitraryB, TestTargetB},
{ArbitraryC, TestTargetC},
{ArbitraryD, TestTargetD},
};
// Given a test enumerator with no client callback, enumeration timeout or enumerator timeout
m_testEnumerator = AZStd::make_unique<TestImpact::TestEnumerator>(
AZStd::nullopt,
m_maxConcurrency,
AZStd::nullopt,
AZStd::nullopt);
// Given an test enumeration job for each test target with no enumeration caching
for (size_t jobId = 0; jobId < m_testTargetJobArgs.size(); jobId++)
{
JobData jobData(m_testTargetPaths[jobId].second, AZStd::nullopt);
m_jobInfos.emplace_back(JobInfo({sequentialToArbitrary.at(jobId)}, m_testTargetJobArgs[jobId], AZStd::move(jobData)));
}
// When the test enumeration jobs are executed
const auto enumerationJobs = m_testEnumerator->Enumerate(m_jobInfos, CacheExceptionPolicy::Never, JobExceptionPolicy::Never);
// Expect each job to successfully result in a test enumeration that matches the expected test enumeration data for that test target
for (const auto& job : enumerationJobs)
{
const JobInfo::IdType jobId = arbitraryToSequential.at(job.GetJobInfo().GetId().m_value);
ValidateJobExecutedSuccessfully(job);
ValidateTestTargetEnumeration(job.GetPayload().value(), m_expectedTestTargetEnumerations[jobId]);
}
}
TEST_P(TestEnumeratorFixtureWithConcurrencyParams, EnumerateTestTargetsWithCallback_EnumerationsMatchTestSuitesInTarget)
{
// Given a client callback function that tracks the number of successful enumerations
size_t numSuccesses = 0;
const auto jobCallback = [&numSuccesses]([[maybe_unused]] const TestImpact::TestEnumerator::JobInfo& jobInfo, const TestImpact::JobMeta& meta)
{
if (meta.m_result == TestImpact::JobResult::ExecutedWithSuccess)
{
numSuccesses++;
}
};
// Given a test enumerator with no enumeration timeout or enumerator timeout
m_testEnumerator = AZStd::make_unique<TestImpact::TestEnumerator>(
jobCallback,
m_maxConcurrency,
AZStd::nullopt,
AZStd::nullopt);
// Given an test enumeration job for each test target with no enumeration caching
for (size_t jobId = 0; jobId < m_testTargetJobArgs.size(); jobId++)
{
JobData jobData(m_testTargetPaths[jobId].second, AZStd::nullopt);
m_jobInfos.emplace_back(JobInfo({jobId}, m_testTargetJobArgs[jobId], AZStd::move(jobData)));
}
// When the test enumeration jobs are executed
const auto enumerationJobs = m_testEnumerator->Enumerate(m_jobInfos, CacheExceptionPolicy::Never, JobExceptionPolicy::Never);
// Expect the number of successful enumerations tracked in the callback to match the number of test targets enumerated
EXPECT_EQ(numSuccesses, enumerationJobs.size());
// Expect each job to successfully result in a test enumeration that matches the expected test enumeration data for that test target
for (const auto& job : enumerationJobs)
{
const JobInfo::IdType jobId = job.GetJobInfo().GetId().m_value;
ValidateJobExecutedSuccessfully(job);
ValidateTestTargetEnumeration(job.GetPayload().value(), m_expectedTestTargetEnumerations[jobId]);
}
}
TEST_P(TestEnumeratorFixtureWithConcurrencyParams, JobRunnerTimeout_InFlightJobsTimeoutAndQueuedJobsUnlaunched)
{
// Given a test enumerator with no client callback or enumerator timeout and 500ms enumeration timeout
m_testEnumerator = AZStd::make_unique<TestImpact::TestEnumerator>(
AZStd::nullopt,
m_maxConcurrency,
AZStd::chrono::milliseconds(500),
AZStd::nullopt);
// Given an test enumeration job for each test target with no enumeration caching where half will sleep indefinitely
for (size_t jobId = 0; jobId < m_testTargetJobArgs.size(); jobId++)
{
JobData jobData(m_testTargetPaths[jobId].second, AZStd::nullopt);
const AZStd::string args = (jobId % 2)
? AZStd::string::format("%s %s", ValidProcessPath, ConstructTestProcessArgs(jobId, LongSleep).c_str())
: m_testTargetJobArgs[jobId];
m_jobInfos.emplace_back(JobInfo({jobId}, args, AZStd::move(jobData)));
}
// When the test enumeration jobs are executed
const auto enumerationJobs = m_testEnumerator->Enumerate(m_jobInfos, CacheExceptionPolicy::Never, JobExceptionPolicy::Never);
// Expect half the jobs to successfully result in a test enumeration that matches the expected test enumeration data for that test
// target with the other half having timed out
for (const auto& job : enumerationJobs)
{
const JobInfo::IdType jobId = job.GetJobInfo().GetId().m_value;
if (jobId % 2)
{
ValidateJobTimeout(job);
}
else
{
ValidateJobExecutedSuccessfully(job);
ValidateTestTargetEnumeration(job.GetPayload().value(), m_expectedTestTargetEnumerations[jobId]);
}
}
}
TEST_F(TestEnumeratorFixture, JobTimeout_InFlightJobTimeoutAndQueuedJobsUnlaunched)
{
// Given a test enumerator with no client callback or enumeration timeout and a 5 second enumerator timeout
m_testEnumerator = AZStd::make_unique<TestImpact::TestEnumerator>(
AZStd::nullopt,
FourConcurrentProcesses,
AZStd::nullopt,
AZStd::chrono::milliseconds(5000));
// Given an test enumeration job for each test target with no enumeration caching where half will sleep indefinitely
for (size_t jobId = 0; jobId < m_testTargetJobArgs.size(); jobId++)
{
JobData jobData(m_testTargetPaths[jobId].second, AZStd::nullopt);
const AZStd::string args = (jobId % 2)
? AZStd::string::format("%s %s", ValidProcessPath, ConstructTestProcessArgs(jobId, LongSleep).c_str())
: m_testTargetJobArgs[jobId];
m_jobInfos.emplace_back(JobInfo({jobId}, args, AZStd::move(jobData)));
}
// When the test enumeration jobs are executed
const auto enumerationJobs = m_testEnumerator->Enumerate(m_jobInfos, CacheExceptionPolicy::Never, JobExceptionPolicy::Never);
// Expect half the jobs to successfully result in a test enumeration that matches the expected test enumeration data for that test
// target with the other half having timed out
for (const auto& job : enumerationJobs)
{
const JobInfo::IdType jobId = job.GetJobInfo().GetId().m_value;
if (jobId % 2)
{
ValidateJobTimeout(job);
}
else
{
ValidateJobExecutedSuccessfully(job);
ValidateTestTargetEnumeration(job.GetPayload().value(), m_expectedTestTargetEnumerations[jobId]);
}
}
}
INSTANTIATE_TEST_CASE_P(
,
TestEnumeratorFixtureWithConcurrencyAndJobExceptionParams,
::testing::Combine(
::testing::ValuesIn(MaxConcurrentEnumerations),
::testing::ValuesIn(JobExceptionPolicies))
);
INSTANTIATE_TEST_CASE_P(
,
TestEnumeratorFixtureWithConcurrencyAndCacheExceptionParams,
::testing::Combine(
::testing::ValuesIn(MaxConcurrentEnumerations),
::testing::ValuesIn(CacheExceptionPolicies))
);
INSTANTIATE_TEST_CASE_P(
,
TestEnumeratorFixtureWithConcurrencyParams,
::testing::ValuesIn(MaxConcurrentEnumerations)
);
} // namespace UnitTest
Code/Tools/TestImpactFramework/Runtime/Code/Tests/Test/TestImpactTestEumerationSerializerTest.cpp
-99
View File
@@ -1,99 +0,0 @@
/*
* All or portions of this file Copyright (c) Amazon.com, Inc. or its affiliates or
* its licensors.
*
* For complete copyright and license terms please see the LICENSE at the root of this
* distribution (the "License"). All use of this software is governed by the License,
* or, if provided, by the license below or the license accompanying this file. Do not
* remove or modify any license notices. This file is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
*
*/
#include <TestImpactTestUtils.h>
#include <Test/Enumeration/TestImpactTestEnumeration.h>
#include <Test/Enumeration/TestImpactTestEnumerationSerializer.h>
#include <AzCore/UnitTest/TestTypes.h>
#include <AzTest/AzTest.h>
namespace UnitTest
{
const TestImpact::TestEnumeration EmptyTestEnum(AZStd::vector<TestImpact::TestEnumerationSuite>{});
TEST(TestEnumerationSerializer, EmptyTestEnumerationSuites_ExpectTemptyTestEnumeration)
{
// Given an empty set of test enumeration suites
// When the test enumeration is serialized and deserialized back again
const AZStd::string serializedString = TestImpact::SerializeTestEnumeration(EmptyTestEnum);
const TestImpact::TestEnumeration actualEnumeration = TestImpact::DeserializeTestEnumeration(serializedString);
// Expect the actual test enumeration to match the expected test enumeration
EXPECT_TRUE(actualEnumeration == EmptyTestEnum);
}
TEST(TestEnumerationSerializer, SerializeAndDeserializeSuitesForTestTargetA_ActualSuiteDataMatchesExpectedSuiteData)
{
// Given the test of test enumeration suites for Test target A
const TestImpact::TestEnumeration expectedEnumeration = TestImpact::TestEnumeration(GetTestTargetATestEnumerationSuites());
// When the test enumeration is serialized and deserialized back again
const AZStd::string serializedString = TestImpact::SerializeTestEnumeration(expectedEnumeration);
const TestImpact::TestEnumeration actualEnumeration = TestImpact::DeserializeTestEnumeration(serializedString);
// Expect the actual test enumeration to match the expected test enumeration
EXPECT_TRUE(actualEnumeration == expectedEnumeration);
// Do not expect the actual test enumeration to match the empty test enumeration
EXPECT_FALSE(actualEnumeration == EmptyTestEnum);
}
TEST(TestEnumerationSerializer, SerializeAndDeserializeSuitesForTestTargetB_ActualSuiteDataMatchesExpectedSuiteData)
{
// Given the test of test enumeration suites for Test target B
const TestImpact::TestEnumeration expectedEnumeration = TestImpact::TestEnumeration(GetTestTargetBTestEnumerationSuites());
// When the test enumeration is serialized and deserialized back again
const AZStd::string serializedString = TestImpact::SerializeTestEnumeration(expectedEnumeration);
const TestImpact::TestEnumeration actualEnumeration = TestImpact::DeserializeTestEnumeration(serializedString);
// Expect the actual test enumeration to match the expected test enumeration
EXPECT_TRUE(actualEnumeration == expectedEnumeration);
// Do not expect the actual test enumeration to match the empty test enumeration
EXPECT_FALSE(actualEnumeration == EmptyTestEnum);
}
TEST(TestEnumerationSerializer, SerializeAndDeserializeSuitesForTestTargetC_ActualSuiteDataMatchesExpectedSuiteData)
{
// Given the test of test enumeration suites for Test target B
const TestImpact::TestEnumeration expectedEnumeration = TestImpact::TestEnumeration(GetTestTargetCTestEnumerationSuites());
// When the test enumeration is serialized and deserialized back again
const AZStd::string serializedString = TestImpact::SerializeTestEnumeration(expectedEnumeration);
const TestImpact::TestEnumeration actualEnumeration = TestImpact::DeserializeTestEnumeration(serializedString);
// Expect the actual test enumeration to match the expected test enumeration
EXPECT_TRUE(actualEnumeration == expectedEnumeration);
// Do not expect the actual test enumeration to match the empty test enumeration
EXPECT_FALSE(actualEnumeration == EmptyTestEnum);
}
TEST(TestEnumerationSerializer, SerializeAndDeserializeSuitesForTestTargetD_ActualSuiteDataMatchesExpectedSuiteData)
{
// Given the test of test enumeration suites for Test target B
const TestImpact::TestEnumeration expectedEnumeration = TestImpact::TestEnumeration(GetTestTargetDTestEnumerationSuites());
// When the test enumeration is serialized and deserialized back again
const AZStd::string serializedString = TestImpact::SerializeTestEnumeration(expectedEnumeration);
const TestImpact::TestEnumeration actualEnumeration = TestImpact::DeserializeTestEnumeration(serializedString);
// Expect the actual test enumeration to match the expected test enumeration
EXPECT_TRUE(actualEnumeration == expectedEnumeration);
// Do not expect the actual test enumeration to match the empty test enumeration
EXPECT_FALSE(actualEnumeration == EmptyTestEnum);
}
} // namespace UnitTest
Code/Tools/TestImpactFramework/Runtime/Code/Tests/Test/TestImpactTestRunSerializerTest.cpp
-99
View File
@@ -1,99 +0,0 @@
/*
* All or portions of this file Copyright (c) Amazon.com, Inc. or its affiliates or
* its licensors.
*
* For complete copyright and license terms please see the LICENSE at the root of this
* distribution (the "License"). All use of this software is governed by the License,
* or, if provided, by the license below or the license accompanying this file. Do not
* remove or modify any license notices. This file is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
*
*/
#include <TestImpactTestUtils.h>
#include <Test/Run/TestImpactTestRun.h>
#include <Test/Run/TestImpactTestRunSerializer.h>
#include <AzCore/UnitTest/TestTypes.h>
#include <AzTest/AzTest.h>
namespace UnitTest
{
const TestImpact::TestRun EmptyTestRun(AZStd::vector<TestImpact::TestRunSuite>{}, AZStd::chrono::milliseconds{0});
TEST(TestRunSerializer, EmptyTestRunSuites_ExpectTemptyTestRun)
{
// Given an empty set of test run suites
// When the test run is serialized and deserialized back again
const auto serializedString = TestImpact::SerializeTestRun(EmptyTestRun);
const auto actualRun = TestImpact::DeserializeTestRun(serializedString);
// Expect the actual test run to match the expected test run
EXPECT_TRUE(actualRun == EmptyTestRun);
}
TEST(TestRunSerializer, SerializeAndDeserializeSuitesForTestTargetA_ActualSuiteDataMatchesExpectedSuiteData)
{
// Given the test of test run suites for Test target A
const auto expectedRun = TestImpact::TestRun(GetTestTargetATestRunSuites(), AZStd::chrono::milliseconds{500});
// When the test run is serialized and deserialized back again
const auto serializedString = TestImpact::SerializeTestRun(expectedRun);
const auto actualRun = TestImpact::DeserializeTestRun(serializedString);
// Expect the actual test run to match the expected test run
EXPECT_TRUE(actualRun == expectedRun);
// Do not expect the actual test run to match the empty test run
EXPECT_FALSE(actualRun == EmptyTestRun);
}
TEST(TestRunSerializer, SerializeAndDeserializeSuitesForTestTargetB_ActualSuiteDataMatchesExpectedSuiteData)
{
// Given the test of test run suites for Test target B
const auto expectedRun = TestImpact::TestRun(GetTestTargetBTestRunSuites(), AZStd::chrono::milliseconds{500});
// When the test run is serialized and deserialized back again
const auto serializedString = TestImpact::SerializeTestRun(expectedRun);
const auto actualRun = TestImpact::DeserializeTestRun(serializedString);
// Expect the actual test run to match the expected test run
EXPECT_TRUE(actualRun == expectedRun);
// Do not expect the actual test run to match the empty test run
EXPECT_FALSE(actualRun == EmptyTestRun);
}
TEST(TestRunSerializer, SerializeAndDeserializeSuitesForTestTargetC_ActualSuiteDataMatchesExpectedSuiteData)
{
// Given the test of test run suites for Test target B
const auto expectedRun = TestImpact::TestRun(GetTestTargetCTestRunSuites(), AZStd::chrono::milliseconds{500});
// When the test run is serialized and deserialized back again
const auto serializedString = TestImpact::SerializeTestRun(expectedRun);
const auto actualRun = TestImpact::DeserializeTestRun(serializedString);
// Expect the actual test run to match the expected test run
EXPECT_TRUE(actualRun == expectedRun);
// Do not expect the actual test run to match the empty test run
EXPECT_FALSE(actualRun == EmptyTestRun);
}
TEST(TestRunSerializer, SerializeAndDeserializeSuitesForTestTargetD_ActualSuiteDataMatchesExpectedSuiteData)
{
// Given the test of test run suites for Test target B
const auto expectedRun = TestImpact::TestRun(GetTestTargetDTestRunSuites(), AZStd::chrono::milliseconds{500});
// When the test run is serialized and deserialized back again
const auto serializedString = TestImpact::SerializeTestRun(expectedRun);
const auto actualRun = TestImpact::DeserializeTestRun(serializedString);
// Expect the actual test run to match the expected test run
EXPECT_TRUE(actualRun == expectedRun);
// Do not expect the actual test run to match the empty test run
EXPECT_FALSE(actualRun == EmptyTestRun);
}
} // namespace UnitTest
Code/Tools/TestImpactFramework/Runtime/Code/Tests/Test/TestImpactTestRunnerTest.cpp
-516
View File
@@ -1,516 +0,0 @@
/*
* All or portions of this file Copyright (c) Amazon.com, Inc. or its affiliates or
* its licensors.
*
* For complete copyright and license terms please see the LICENSE at the root of this
* distribution (the "License"). All use of this software is governed by the License,
* or, if provided, by the license below or the license accompanying this file. Do not
* remove or modify any license notices. This file is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
*
*/
#include <TestImpactTestJobRunnerCommon.h>
#include <TestImpactTestUtils.h>
#include <Artifact/TestImpactArtifactException.h>
#include <Test/Run/TestImpactTestRunException.h>
#include <Test/Run/TestImpactTestRunner.h>
#include <AzCore/IO/Path/Path.h>
#include <AzCore/UnitTest/TestTypes.h>
#include <AzCore/std/containers/array.h>
#include <AzCore/std/containers/vector.h>
#include <AzCore/std/optional.h>
#include <AzCore/std/smart_ptr/unique_ptr.h>
#include <AzCore/std/string/string.h>
#include <AzTest/AzTest.h>
namespace UnitTest
{
using JobExceptionPolicy = TestImpact::TestRunner::JobExceptionPolicy;
AZStd::string GetRunCommandForTarget(AZStd::pair<AZ::IO::Path, AZ::IO::Path> testTarget)
{
return AZStd::string::format(
"%s %s AzRunUnitTests --gtest_output=xml:%s", LY_TEST_IMPACT_AZ_TESTRUNNER_BIN, testTarget.first.c_str(),
testTarget.second.c_str());
}
class TestRunnerFixture
: public AllocatorsTestFixture
{
public:
void SetUp() override;
protected:
using JobInfo = TestImpact::TestRunner::JobInfo;
using JobData = TestImpact::TestRunner::JobData;
AZStd::vector<JobInfo> m_jobInfos;
AZStd::unique_ptr<TestImpact::TestRunner> m_testRunner;
AZStd::vector<AZStd::string> m_testTargetJobArgs;
AZStd::vector<AZStd::pair<AZ::IO::Path, AZ::IO::Path>> m_testTargetPaths;
AZStd::vector<TestImpact::TestRun> m_expectedTestTargetRuns;
AZStd::vector<TestImpact::TestRunResult> m_expectedTestTargetResult;
size_t m_maxConcurrency = 0;
};
void TestRunnerFixture::SetUp()
{
UnitTest::AllocatorsTestFixture::SetUp();
DeleteFiles(LY_TEST_IMPACT_TEST_TARGET_RESULTS_DIR, "*.xml");
// first: path to test target bin
// second: path to test target gtest results file in XML format
const AZStd::string runPath = AZStd::string(LY_TEST_IMPACT_TEST_TARGET_RESULTS_DIR) + "/%s.Run.xml";
m_testTargetPaths.emplace_back(
LY_TEST_IMPACT_TEST_TARGET_A_BIN, AZStd::string::format(runPath.c_str(), LY_TEST_IMPACT_TEST_TARGET_A_BASE_NAME));
m_testTargetPaths.emplace_back(
LY_TEST_IMPACT_TEST_TARGET_B_BIN, AZStd::string::format(runPath.c_str(), LY_TEST_IMPACT_TEST_TARGET_B_BASE_NAME));
m_testTargetPaths.emplace_back(
LY_TEST_IMPACT_TEST_TARGET_C_BIN, AZStd::string::format(runPath.c_str(), LY_TEST_IMPACT_TEST_TARGET_C_BASE_NAME));
m_testTargetPaths.emplace_back(
LY_TEST_IMPACT_TEST_TARGET_D_BIN, AZStd::string::format(runPath.c_str(), LY_TEST_IMPACT_TEST_TARGET_D_BASE_NAME));
m_expectedTestTargetRuns.emplace_back(GetTestTargetATestRunSuites(), AZStd::chrono::milliseconds{500});
m_expectedTestTargetRuns.emplace_back(GetTestTargetBTestRunSuites(), AZStd::chrono::milliseconds{500});
m_expectedTestTargetRuns.emplace_back(GetTestTargetCTestRunSuites(), AZStd::chrono::milliseconds{500});
m_expectedTestTargetRuns.emplace_back(GetTestTargetDTestRunSuites(), AZStd::chrono::milliseconds{500});
m_expectedTestTargetResult.emplace_back(TestImpact::TestRunResult::Failed);
m_expectedTestTargetResult.emplace_back(TestImpact::TestRunResult::Passed);
m_expectedTestTargetResult.emplace_back(TestImpact::TestRunResult::Passed);
m_expectedTestTargetResult.emplace_back(TestImpact::TestRunResult::Passed);
for (const auto& testTarget : m_testTargetPaths)
{
m_testTargetJobArgs.emplace_back(GetRunCommandForTarget(testTarget));
}
}
// Fixture parameterized for different max number of concurrent jobs
class TestRunnerFixtureWithConcurrencyParams
: public TestRunnerFixture
, public ::testing::WithParamInterface<size_t>
{
public:
void SetUp() override
{
TestRunnerFixture::SetUp();
m_maxConcurrency = GetParam();
}
};
using ConcurrencyAndJobExceptionPermutation = AZStd::tuple
<
size_t, // Max number of concurrent processes
JobExceptionPolicy // Test job exception policy
>;
// Fixture parameterized for different max number of concurrent jobs and different job exception policies
class TestRunnerFixtureWithConcurrencyAndJobExceptionParams
: public TestRunnerFixture
, public ::testing::WithParamInterface<ConcurrencyAndJobExceptionPermutation>
{
public:
void SetUp() override
{
TestRunnerFixture::SetUp();
const auto& [maxConcurrency, jobExceptionPolicy] = GetParam();
m_maxConcurrency = maxConcurrency;
m_jobExceptionPolicy = jobExceptionPolicy;
}
protected:
JobExceptionPolicy m_jobExceptionPolicy = JobExceptionPolicy::Never;
};
class TestRunnerFixtureWithConcurrencyAndFailedToLaunchExceptionParams
: public TestRunnerFixtureWithConcurrencyAndJobExceptionParams
{
};
class TestRunnerFixtureWithConcurrencyAndExecutedWithFailureExceptionParams
: public TestRunnerFixtureWithConcurrencyAndJobExceptionParams
{
};
namespace
{
AZStd::array<size_t, 4> MaxConcurrentRuns = {1, 2, 3, 4};
AZStd::array<JobExceptionPolicy, 2> FailedToLaunchExceptionPolicies = {
JobExceptionPolicy::Never, JobExceptionPolicy::OnFailedToExecute};
AZStd::array<JobExceptionPolicy, 2> ExecutedWithFailureExceptionPolicies = {
JobExceptionPolicy::Never, JobExceptionPolicy::OnExecutedWithFailure};
} // namespace
TEST_P(
TestRunnerFixtureWithConcurrencyAndFailedToLaunchExceptionParams,
InvalidCommandArgument_ExpectJobResulFailedToExecuteeOrTestJobException)
{
// Given a test runner with no client callback or run timeout or runner timeout
m_testRunner = AZStd::make_unique<TestImpact::TestRunner>(AZStd::nullopt, m_maxConcurrency, AZStd::nullopt, AZStd::nullopt);
// Given a mixture of test run jobs with valid and invalid command arguments
for (size_t jobId = 0; jobId < m_testTargetJobArgs.size(); jobId++)
{
const AZStd::string args = (jobId % 2) ? InvalidProcessPath : m_testTargetJobArgs[jobId];
JobData jobData(m_testTargetPaths[jobId].second);
m_jobInfos.emplace_back(JobInfo({jobId}, args, AZStd::move(jobData)));
}
try
{
// When the test run jobs are executed with different exception policies
const auto runnerJobs = m_testRunner->RunTests(m_jobInfos, m_jobExceptionPolicy);
// Expect this statement to be reachable only if no exception policy for launch failures
EXPECT_FALSE(::IsFlagSet(m_jobExceptionPolicy, JobExceptionPolicy::OnFailedToExecute));
for (const auto& job : runnerJobs)
{
const JobInfo::IdType jobId = job.GetJobInfo().GetId().m_value;
if (jobId % 2)
{
// Expect invalid jobs have a job result of FailedToExecute
ValidateJobFailedToExecute(job);
}
else
{
// Expect the valid jobs to successfully result in a test run that matches the expected test run data
ValidateTestRunCompleted(job, m_expectedTestTargetResult[jobId]);
ValidateTestTargetRun(job.GetPayload().value(), m_expectedTestTargetRuns[jobId]);
}
}
}
catch ([[maybe_unused]] const TestImpact::TestJobException& e)
{
// Expect this statement to be reachable only if there is an exception policy for launch failures
EXPECT_TRUE(::IsFlagSet(m_jobExceptionPolicy, JobExceptionPolicy::OnFailedToExecute));
}
catch (...)
{
// Do not expect any other exceptions
FAIL();
}
}
TEST_P(
TestRunnerFixtureWithConcurrencyAndExecutedWithFailureExceptionParams,
ErroneousReturnCode_ExpectJobResultExecutedWithFailureOrTestJobException)
{
// Given a test runner with no client callback or run timeout or runner timeout
m_testRunner = AZStd::make_unique<TestImpact::TestRunner>(AZStd::nullopt, m_maxConcurrency, AZStd::nullopt, AZStd::nullopt);
// Given a mixture of test run jobs that execute and return either successfully or with failure
for (size_t jobId = 0; jobId < m_testTargetJobArgs.size(); jobId++)
{
JobData jobData(m_testTargetPaths[jobId].second);
m_jobInfos.emplace_back(JobInfo({jobId}, m_testTargetJobArgs[jobId], AZStd::move(jobData)));
}
try
{
// When the test run jobs are executed with different exception policies
const auto runnerJobs = m_testRunner->RunTests(m_jobInfos, m_jobExceptionPolicy);
// Expect this statement to be reachable only if no exception policy for jobs that return with error
EXPECT_FALSE(::IsFlagSet(m_jobExceptionPolicy, JobExceptionPolicy::OnExecutedWithFailure));
for (const auto& job : runnerJobs)
{
const JobInfo::IdType jobId = job.GetJobInfo().GetId().m_value;
// Expect the valid jobs to successfully result in a test run that matches the expected test run data
ValidateTestRunCompleted(job, m_expectedTestTargetResult[jobId]);
ValidateTestTargetRun(job.GetPayload().value(), m_expectedTestTargetRuns[jobId]);
}
}
catch ([[maybe_unused]] const TestImpact::TestJobException& e)
{
// Expect this statement to be reachable only if there is an exception policy for jobs that return with error
EXPECT_TRUE(::IsFlagSet(m_jobExceptionPolicy, JobExceptionPolicy::OnExecutedWithFailure));
}
catch ([[maybe_unused]] const TestImpact::Exception& e)
{
FAIL();
}
catch (...)
{
// Do not expect any other exceptions
FAIL();
}
}
TEST_F(TestRunnerFixture, EmptyArtifact_ExpectTestRunnerException)
{
// Given a test runner with no client callback, concurrency, run timeout or runner timeout
m_testRunner = AZStd::make_unique<TestImpact::TestRunner>(AZStd::nullopt, OneConcurrentProcess, AZStd::nullopt, AZStd::nullopt);
// Given an test runner job that will return successfully but with an empty artifact string
m_jobInfos.emplace_back(JobInfo({0}, m_testTargetJobArgs[0], JobData("")));
try
{
// When the test runner job is executed
const auto runnerJobs = m_testRunner->RunTests(m_jobInfos, JobExceptionPolicy::Never);
// Do not expect this statement to be reachable
FAIL();
}
catch ([[maybe_unused]] const TestImpact::TestRunException& e)
{
// Expect an runner exception
SUCCEED();
}
catch (...)
{
// Do not expect any other exceptions
FAIL();
}
}
TEST_F(TestRunnerFixture, InvalidRunArtifact_ExpectArtifactException)
{
// Given a test run artifact with invalid contents
WriteTextToFile("There is nothing valid here", m_testTargetPaths[TestTargetA].second);
// Given a job command that will write the test run artifact to a different location that what we will read from
auto invalidRunArtifact = m_testTargetPaths[TestTargetA];
invalidRunArtifact.second /= ".xml";
const AZStd::string args = GetRunCommandForTarget(invalidRunArtifact);
// Given a test runner with no client callback, concurrency, run timeout or runner timeout
m_testRunner = AZStd::make_unique<TestImpact::TestRunner>(AZStd::nullopt, OneConcurrentProcess, AZStd::nullopt, AZStd::nullopt);
// Given an test runner job that will return successfully but not produce an artifact
JobData jobData(m_testTargetPaths[TestTargetA].second);
m_jobInfos.emplace_back(JobInfo({TestTargetA}, args, AZStd::move(jobData)));
try
{
// When the test runner job is executed
const auto runnerJobs = m_testRunner->RunTests(m_jobInfos, JobExceptionPolicy::Never);
// Do not expect this statement to be reachable
FAIL();
}
catch ([[maybe_unused]] const TestImpact::ArtifactException& e)
{
// Expect an runner exception
SUCCEED();
}
catch (const TestImpact::Exception& e)
{
std::cout << e.what();
FAIL();
}
catch (...)
{
// Do not expect any other exceptions
FAIL();
}
}
TEST_P(TestRunnerFixtureWithConcurrencyParams, RunTestTargets_RunsMatchTestSuitesInTarget)
{
// Given a test runner with no client callback, runner timeout or runner timeout
m_testRunner = AZStd::make_unique<TestImpact::TestRunner>(AZStd::nullopt, m_maxConcurrency, AZStd::nullopt, AZStd::nullopt);
// Given an test runner job for each test target with no runner caching
for (size_t jobId = 0; jobId < m_testTargetJobArgs.size(); jobId++)
{
JobData jobData(m_testTargetPaths[jobId].second);
m_jobInfos.emplace_back(JobInfo({jobId}, m_testTargetJobArgs[jobId], AZStd::move(jobData)));
}
// When the test runner jobs are executed
const auto runnerJobs = m_testRunner->RunTests(m_jobInfos, JobExceptionPolicy::Never);
// Expect each job to successfully result in a test runner that matches the expected test runner data for that test target
for (const auto& job : runnerJobs)
{
const JobInfo::IdType jobId = job.GetJobInfo().GetId().m_value;
ValidateTestRunCompleted(job, m_expectedTestTargetResult[jobId]);
ValidateTestTargetRun(job.GetPayload().value(), m_expectedTestTargetRuns[jobId]);
}
}
TEST_P(TestRunnerFixtureWithConcurrencyParams, RunTestTargetsWithArbitraryJobIds_RunsMatchTestSuitesInTarget)
{
// Given a set of arbitrary job ids to be used for the test target jobs
enum
{
ArbitraryA = 36,
ArbitraryB = 890,
ArbitraryC = 19,
ArbitraryD = 1
};
const AZStd::unordered_map<JobInfo::IdType, JobInfo::IdType> sequentialToArbitrary =
{
{TestTargetA, ArbitraryA},
{TestTargetB, ArbitraryB},
{TestTargetC, ArbitraryC},
{TestTargetD, ArbitraryD},
};
const AZStd::unordered_map<JobInfo::IdType, JobInfo::IdType> arbitraryToSequential =
{
{ArbitraryA, TestTargetA},
{ArbitraryB, TestTargetB},
{ArbitraryC, TestTargetC},
{ArbitraryD, TestTargetD},
};
// Given a test runner with no client callback, run timeout or runner timeout
m_testRunner = AZStd::make_unique<TestImpact::TestRunner>(AZStd::nullopt, m_maxConcurrency, AZStd::nullopt, AZStd::nullopt);
// Given an test run job for each test target
for (size_t jobId = 0; jobId < m_testTargetJobArgs.size(); jobId++)
{
JobData jobData(m_testTargetPaths[jobId].second);
m_jobInfos.emplace_back(JobInfo({sequentialToArbitrary.at(jobId)}, m_testTargetJobArgs[jobId], AZStd::move(jobData)));
}
// When the test run jobs are executed
const auto runnerJobs = m_testRunner->RunTests(m_jobInfos, JobExceptionPolicy::Never);
// Expect each job to successfully result in a test run that matches the expected test run data for that test target
for (const auto& job : runnerJobs)
{
const JobInfo::IdType jobId = arbitraryToSequential.at(job.GetJobInfo().GetId().m_value);
ValidateTestRunCompleted(job, m_expectedTestTargetResult[jobId]);
ValidateTestTargetRun(job.GetPayload().value(), m_expectedTestTargetRuns[jobId]);
}
}
TEST_P(TestRunnerFixtureWithConcurrencyParams, RunTestTargetsWithCallback_RunsMatchTestSuitesInTarget)
{
// Given a client callback function that tracks the number of successful runs
size_t numSuccesses = 0;
const auto jobCallback =
[&numSuccesses]([[maybe_unused]] const TestImpact::TestRunner::JobInfo& jobInfo, const TestImpact::JobMeta& meta) {
if (meta.m_result == TestImpact::JobResult::ExecutedWithSuccess)
{
numSuccesses++;
}
};
// Given a test runner with no run timeout or runner timeout
m_testRunner = AZStd::make_unique<TestImpact::TestRunner>(jobCallback, m_maxConcurrency, AZStd::nullopt, AZStd::nullopt);
// Given an test run job for each test target
for (size_t jobId = 0; jobId < m_testTargetJobArgs.size(); jobId++)
{
JobData jobData(m_testTargetPaths[jobId].second);
m_jobInfos.emplace_back(JobInfo({jobId}, m_testTargetJobArgs[jobId], AZStd::move(jobData)));
}
// When the test run jobs are executed
const auto runnerJobs = m_testRunner->RunTests(m_jobInfos, JobExceptionPolicy::Never);
// Expect the number of successful runs tracked in the callback to match the number of test targets run with no failures
EXPECT_EQ(
numSuccesses,
AZStd::count_if(m_expectedTestTargetResult.begin(), m_expectedTestTargetResult.end(), [](TestImpact::TestRunResult result) {
return result == TestImpact::TestRunResult::Passed;
}));
// Expect each job to successfully result in a test run that matches the expected test run data for that test target
for (const auto& job : runnerJobs)
{
const JobInfo::IdType jobId = job.GetJobInfo().GetId().m_value;
ValidateTestRunCompleted(job, m_expectedTestTargetResult[jobId]);
ValidateTestTargetRun(job.GetPayload().value(), m_expectedTestTargetRuns[jobId]);
}
}
TEST_P(TestRunnerFixtureWithConcurrencyParams, JobRunnerTimeout_InFlightJobsTimeoutAndQueuedJobsUnlaunched)
{
// Given a test runner with no client callback or runner timeout and 500ms run timeout
m_testRunner =
AZStd::make_unique<TestImpact::TestRunner>(AZStd::nullopt, m_maxConcurrency, AZStd::chrono::milliseconds(500), AZStd::nullopt);
// Given an test run job for each test target where half will sleep indefinitely
for (size_t jobId = 0; jobId < m_testTargetJobArgs.size(); jobId++)
{
JobData jobData(m_testTargetPaths[jobId].second);
const AZStd::string args = (jobId % 2)
? AZStd::string::format("%s %s", ValidProcessPath, ConstructTestProcessArgs(jobId, LongSleep).c_str())
: m_testTargetJobArgs[jobId];
m_jobInfos.emplace_back(JobInfo({jobId}, args, AZStd::move(jobData)));
}
// When the test run jobs are executed
const auto runnerJobs = m_testRunner->RunTests(m_jobInfos, JobExceptionPolicy::Never);
// Expect half the jobs to successfully result in a test run that matches the expected test run data for that test target
// with the other half having timed out
for (const auto& job : runnerJobs)
{
const JobInfo::IdType jobId = job.GetJobInfo().GetId().m_value;
if (jobId % 2)
{
ValidateJobTimeout(job);
}
else
{
ValidateTestRunCompleted(job, m_expectedTestTargetResult[jobId]);
ValidateTestTargetRun(job.GetPayload().value(), m_expectedTestTargetRuns[jobId]);
}
}
}
TEST_F(TestRunnerFixture, JobTimeout_InFlightJobTimeoutAndQueuedJobsUnlaunched)
{
// Given a test runner with no client callback or run timeout and a 5 second runner timeout
m_testRunner = AZStd::make_unique<TestImpact::TestRunner>(
AZStd::nullopt, FourConcurrentProcesses, AZStd::nullopt, AZStd::chrono::milliseconds(5000));
// Given an test run job for each test target where half will sleep indefinitely
for (size_t jobId = 0; jobId < m_testTargetJobArgs.size(); jobId++)
{
JobData jobData(m_testTargetPaths[jobId].second);
const AZStd::string args = (jobId % 2)
? AZStd::string::format("%s %s", ValidProcessPath, ConstructTestProcessArgs(jobId, LongSleep).c_str())
: m_testTargetJobArgs[jobId];
m_jobInfos.emplace_back(JobInfo({jobId}, args, AZStd::move(jobData)));
}
// When the test run jobs are executed
const auto runnerJobs = m_testRunner->RunTests(m_jobInfos, JobExceptionPolicy::Never);
// Expect half the jobs to successfully result in a test run that matches the expected test run data for that test target
// with the other half having timed out
for (const auto& job : runnerJobs)
{
const JobInfo::IdType jobId = job.GetJobInfo().GetId().m_value;
if (jobId % 2)
{
ValidateJobTimeout(job);
}
else
{
ValidateTestRunCompleted(job, m_expectedTestTargetResult[jobId]);
ValidateTestTargetRun(job.GetPayload().value(), m_expectedTestTargetRuns[jobId]);
}
}
}
INSTANTIATE_TEST_CASE_P(
,
TestRunnerFixtureWithConcurrencyAndFailedToLaunchExceptionParams,
::testing::Combine(::testing::ValuesIn(MaxConcurrentRuns), ::testing::ValuesIn(FailedToLaunchExceptionPolicies)));
INSTANTIATE_TEST_CASE_P(
,
TestRunnerFixtureWithConcurrencyAndExecutedWithFailureExceptionParams,
::testing::Combine(::testing::ValuesIn(MaxConcurrentRuns), ::testing::ValuesIn(ExecutedWithFailureExceptionPolicies)));
INSTANTIATE_TEST_CASE_P(, TestRunnerFixtureWithConcurrencyParams, ::testing::ValuesIn(MaxConcurrentRuns));
} // namespace UnitTest
Code/Tools/TestImpactFramework/Runtime/Code/Tests/TestImpactFrameworkPathTest.cpp
-137
View File
@@ -1,137 +0,0 @@
/*
* All or portions of this file Copyright (c) Amazon.com, Inc. or its affiliates or
* its licensors.
*
* For complete copyright and license terms please see the LICENSE at the root of this
* distribution (the "License"). All use of this software is governed by the License,
* or, if provided, by the license below or the license accompanying this file. Do not
* remove or modify any license notices. This file is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
*
*/
#include <AzCore/UnitTest/TestTypes.h>
#include <AzTest/AzTest.h>
#include <TestImpactFramework/TestImpactFrameworkPath.h>
namespace UnitTest
{
class FrameworkPathTestFixture
: public AllocatorsTestFixture
{
public:
void SetUp() override
{
UnitTest::AllocatorsTestFixture::SetUp();
m_parentPathAbs = AZStd::string(AZStd::string("parent") + AZ_TRAIT_OS_PATH_SEPARATOR + "path");
m_childPathRel = AZStd::string(AZStd::string("child") + AZ_TRAIT_OS_PATH_SEPARATOR + "path");
m_childPathAbs = AZStd::string(m_parentPathAbs + AZ_TRAIT_OS_PATH_SEPARATOR + m_childPathRel);
m_pathComponentA = AZStd::string("DirA");
m_pathComponentB = AZStd::string("DirB");
m_pathComponentC = AZStd::string("DirC");
m_posixPath = m_pathComponentA + AZ::IO::PosixPathSeparator + m_pathComponentB + AZ::IO::PosixPathSeparator + m_pathComponentC;
m_windowsPath =
m_pathComponentA + AZ::IO::WindowsPathSeparator + m_pathComponentB + AZ::IO::WindowsPathSeparator + m_pathComponentC;
m_mixedPath =
m_pathComponentA + AZ::IO::WindowsPathSeparator + m_pathComponentB + AZ::IO::PosixPathSeparator + m_pathComponentC;
m_referredPath =
m_pathComponentA + AZ_TRAIT_OS_PATH_SEPARATOR + m_pathComponentB + AZ_TRAIT_OS_PATH_SEPARATOR + m_pathComponentC;
}
void TearDown() override
{
UnitTest::AllocatorsTestFixture::TearDown();
}
AZStd::string m_parentPathAbs;
AZStd::string m_childPathRel;
AZStd::string m_childPathAbs;
AZStd::string m_pathComponentA;
AZStd::string m_pathComponentB;
AZStd::string m_pathComponentC;
AZ::IO::Path m_posixPath;
AZ::IO::Path m_windowsPath;
AZ::IO::Path m_mixedPath;
AZ::IO::Path m_referredPath;
};
TEST_F(FrameworkPathTestFixture, DefaultConstructor_HasEmptyAbsAndRelPaths)
{
// Given an empty framework path
TestImpact::FrameworkPath path;
// Expect the absolute path to be empty
EXPECT_TRUE(path.Absolute().empty());
// Expect the relative path to be empty
EXPECT_TRUE(path.Relative().empty());
}
TEST_F(FrameworkPathTestFixture, OrphanConstructor_HasAbsAndEmptyRelPaths)
{
// Given an orhpan framework path
TestImpact::FrameworkPath path(m_parentPathAbs);
// Expect the absolute path to be equal to the specified path
EXPECT_EQ(path.Absolute().String(), m_parentPathAbs);
// Expect the relative path to be current directory symbol
EXPECT_STREQ(path.Relative().c_str(), ".");
}
TEST_F(FrameworkPathTestFixture, ParentConstructor_HasAbsAndRelPaths)
{
// Given a child framework path
TestImpact::FrameworkPath path(m_childPathAbs, TestImpact::FrameworkPath(m_parentPathAbs));
// Expect the absolute path to be equal to the concatenation of the parent and child path
EXPECT_EQ(path.Absolute().String(), m_childPathAbs);
// Expect the relative path to equal to the specified path
EXPECT_EQ(path.Relative().String(), m_childPathRel);
}
TEST_F(FrameworkPathTestFixture, PosixSeperators_HasUniformPreferredSeperators)
{
// Given an orhpan framework path with Posix separators
TestImpact::FrameworkPath path(m_posixPath);
// Expect the absolute path to be equal to the specified path with preferred separators
EXPECT_EQ(path.Absolute(), m_referredPath);
// Expect the relative path to be current directory symbol
EXPECT_STREQ(path.Relative().c_str(), ".");
}
TEST_F(FrameworkPathTestFixture, WindowsSeperators_HasUniformPreferredSeperators)
{
// Given an orhpan framework path with Windows separators
TestImpact::FrameworkPath path(m_windowsPath);
// Expect the absolute path to be equal to the specified path with preferred separators
EXPECT_EQ(path.Absolute(), m_referredPath);
// Expect the relative path to be current directory symbol
EXPECT_STREQ(path.Relative().c_str(), ".");
}
TEST_F(FrameworkPathTestFixture, MixedSeperators_HasUniformPreferredSeperators)
{
// Given an orhpan framework path with mixed separators
TestImpact::FrameworkPath path(m_windowsPath);
// Expect the absolute path to be equal to the specified path with preferred separators
EXPECT_EQ(path.Absolute(), m_referredPath);
// Expect the relative path to be current directory symbol
EXPECT_STREQ(path.Relative().c_str(), ".");
}
} // namespace UnitTest
Code/Tools/TestImpactFramework/Runtime/Code/Tests/TestImpactTestMain.cpp
-33
View File
@@ -1,33 +0,0 @@
/*
* All or portions of this file Copyright (c) Amazon.com, Inc. or its affiliates or
* its licensors.
*
* For complete copyright and license terms please see the LICENSE at the root of this
* distribution (the "License"). All use of this software is governed by the License,
* or, if provided, by the license below or the license accompanying this file. Do not
* remove or modify any license notices. This file is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
*
*/
#include <AzCore/Memory/OSAllocator.h>
#include <AzCore/Memory/SystemAllocator.h>
#include <AzTest/AzTest.h>
class TestImpactTestEnvironment : public AZ::Test::ITestEnvironment
{
protected:
void SetupEnvironment() override
{
AZ::AllocatorInstance<AZ::OSAllocator>::Create();
AZ::AllocatorInstance<AZ::SystemAllocator>::Create();
}
void TeardownEnvironment() override
{
AZ::AllocatorInstance<AZ::OSAllocator>::Destroy();
AZ::AllocatorInstance<AZ::SystemAllocator>::Destroy();
}
};
AZ_UNIT_TEST_HOOK(new TestImpactTestEnvironment);
cmake/LYTestWrappers.cmake
+20 -9
View File
@@ -151,12 +151,6 @@ function(ly_add_test)
set(LY_ADDED_TEST_NAME ${qualified_test_run_name_with_suite}::TEST_RUN)
set(LY_ADDED_TEST_NAME ${LY_ADDED_TEST_NAME} PARENT_SCOPE)
# Store the test so we can walk through all of them in LYTestImpactFramework.cmake
set_property(GLOBAL APPEND PROPERTY LY_ALL_TESTS ${LY_ADDED_TEST_NAME})
set_property(GLOBAL APPEND PROPERTY LY_ALL_TESTS_${LY_ADDED_TEST_NAME}_TEST_NAME ${ly_add_test_NAME})
set_property(GLOBAL APPEND PROPERTY LY_ALL_TESTS_${LY_ADDED_TEST_NAME}_TEST_SUITE ${ly_add_test_TEST_SUITE})
set_property(GLOBAL APPEND PROPERTY LY_ALL_TESTS_${LY_ADDED_TEST_NAME}_TEST_LIBRARY ${ly_add_test_TEST_LIBRARY})
set(final_labels SUITE_${ly_add_test_TEST_SUITE})
if (ly_add_test_TEST_REQUIRES)
@@ -247,6 +241,12 @@ function(ly_add_test)
endif()
# Store the test so we can walk through all of them in LYTestImpactFramework.cmake
set_property(GLOBAL APPEND PROPERTY LY_ALL_TESTS ${ly_add_test_NAME})
set_property(GLOBAL APPEND PROPERTY LY_ALL_TESTS_${ly_add_test_NAME}_TEST_SUITE ${ly_add_test_TEST_SUITE})
set_property(GLOBAL APPEND PROPERTY LY_ALL_TESTS_${ly_add_test_NAME}_TEST_LIBRARY ${ly_add_test_TEST_LIBRARY})
set_property(GLOBAL APPEND PROPERTY LY_ALL_TESTS_${ly_add_test_NAME}_TEST_TIMEOUT ${ly_add_test_TIMEOUT})
endfunction()
#! ly_add_pytest: registers target PyTest-based test with CTest
@@ -298,8 +298,8 @@ function(ly_add_pytest)
${ly_add_pytest_UNPARSED_ARGUMENTS}
)
set_property(GLOBAL APPEND PROPERTY LY_ALL_TESTS_${ly_add_pytest_NAME}_SCRIPT_PATH ${ly_add_pytest_PATH})
set_tests_properties(${LY_ADDED_TEST_NAME} PROPERTIES RUN_SERIAL "${ly_add_pytest_TEST_SERIAL}")
set_property(GLOBAL APPEND PROPERTY LY_ALL_TESTS_${LY_ADDED_TEST_NAME}_SCRIPT_PATH ${ly_add_pytest_PATH})
endfunction()
#! ly_add_editor_python_test: registers target Editor Python Bindings test with CTest
@@ -363,8 +363,8 @@ function(ly_add_editor_python_test)
COMPONENT ${ly_add_editor_python_test_COMPONENT}
)
set_property(GLOBAL APPEND PROPERTY LY_ALL_TESTS_${ly_add_editor_python_test_NAME}_SCRIPT_PATH ${ly_add_editor_python_test_PATH})
set_tests_properties(${LY_ADDED_TEST_NAME} PROPERTIES RUN_SERIAL "${ly_add_editor_python_test_TEST_SERIAL}")
set_property(GLOBAL APPEND PROPERTY LY_ALL_TESTS_${LY_ADDED_TEST_NAME}_SCRIPT_PATH ${ly_add_editor_python_test_PATH})
endfunction()
#! ly_add_googletest: Adds a new RUN_TEST using for the specified target using the supplied command or fallback to running
@@ -424,8 +424,14 @@ function(ly_add_googletest)
set(full_test_command $<TARGET_FILE:AZ::AzTestRunner> $<TARGET_FILE:${build_target}> AzRunUnitTests)
# Add AzTestRunner as a build dependency
ly_add_dependencies(${build_target} AZ::AzTestRunner)
# Ideally, we would populate the full command procedurally but the generator expressions won't be expanded by the time we need this data
set_property(GLOBAL APPEND PROPERTY LY_ALL_TESTS_${ly_add_googletest_NAME}_TEST_COMMAND "AzRunUnitTests")
else()
set(full_test_command ${ly_add_googletest_TEST_COMMAND})
# Remove the generator expressions so we are left with the argument(s) required to run unit tests for executable targets
string(REPLACE ";" "" stripped_test_command ${full_test_command})
string(GENEX_STRIP ${stripped_test_command} stripped_test_command)
set_property(GLOBAL APPEND PROPERTY LY_ALL_TESTS_${ly_add_googletest_NAME}_TEST_COMMAND ${stripped_test_command})
endif()
string(REPLACE "::" "_" report_directory "${GTEST_XML_OUTPUT_DIR}/${ly_add_googletest_NAME}.xml")
@@ -507,8 +513,14 @@ function(ly_add_googlebenchmark)
# If command is not supplied attempts, uses the AzTestRunner to run googlebenchmarks on the supplied TARGET
set(full_test_command $<TARGET_FILE:AZ::AzTestRunner> $<TARGET_FILE:${build_target}> AzRunBenchmarks ${output_format_args})
# Ideally, we would populate the full command procedurally but the generator expressions won't be expanded by the time we need this data
set_property(GLOBAL APPEND PROPERTY LY_ALL_TESTS_${ly_add_googlebenchmark_NAME}_TEST_COMMAND "AzRunUnitTests")
else()
set(full_test_command ${ly_add_googlebenchmark_TEST_COMMAND})
# Remove the generator expressions so we are left with the argument(s) required to run unit tests for executable targets
string(REPLACE ";" "" stripped_test_command ${full_test_command})
string(GENEX_STRIP ${stripped_test_command} stripped_test_command)
set_property(GLOBAL APPEND PROPERTY LY_ALL_TESTS_${ly_add_googletest_NAME}_TEST_COMMAND ${stripped_test_command})
endif()
ly_add_test(
@@ -524,6 +536,5 @@ function(ly_add_googlebenchmark)
AZ::AzTestRunner
COMPONENT ${ly_add_googlebenchmark_COMPONENT}
)
endfunction()
cmake/TestImpactFramework/ConsoleFrontendConfig.in
+188 -111
View File
@@ -1,111 +1,188 @@
[path.configuration]
repo_dir = "${repo_dir}"
working_dir = "${working_dir}"
bin_dir = "${runtime_bin_dir}"
tests_dir = "${tests_dir}"
temp_dir = "${temp_dir}"
target_mappings_dir = "${source_target_mapping_dir}"
test_type_dir = "${test_type_dir}"
dependencies_dir = "${target_dependency_dir}"
[sourcetree.configuration.filters.autogen]
# E.g. matches input /Foo/{Bar}.FooBar.xml with output /Baz/{Bar}.BazBar.cpp
input_output_pairer = "(.*)\\..*"
[sourcetree.configuration.filters.autogen.input]
exclude_filter = [".jinja"]
[sourcetree.configuration.filters.source]
exclude_filter = [".cmake"]
[sourcetree.configuration.testtype.enumerated]
file = "All.tests"
# The table to read from the test enumeration file that contains the test targets
target_table = "google.test"
[sourcetree.configuration.dependency]
# E.g. matches WhiteBox.Editor.Static\n(Gem::WhiteBox.Editor.Static) or WhiteBox.Editor.Static
target_dependency_file_matcher = "target\\.(.*)\\.(dependers)?"
# E.g. matches target.WhiteBox.Editor.Static (for dependency) target.WhiteBox.Editor.Static.dependers (for dependers)
target_vertex_matcher = "(?:(.*)\\n|(.*)"
[spartia.configuration]
test_impact_Data_file = "TestImpactData.spartia"
test_run_coverage_file = "{test_dir}\\{test_target}.coverage.xml"
test_run_results_file = "{test_dir}\\{test_target}.results.xml"
test_enumeration_file = "{temp_dir}\\{test_target}.enum"
test_shard_selection_file = "{temp_dir}\\{test_target}.filter.{shard_id}"
exclude_filter = [
{ target = "AssetBundler.Tests", tests = ["*"] },
{ target = "AssetProcessor.Tests", tests = ["*"] },
{ target = "CryRenderD3D11.Tests", tests = ["*"] },
{ target = "CryRenderD3D12.Tests", tests = ["*"] },
{ target = "LyzardApplicationDescriptors.Tests", tests = ["*"] },
{ target = "EMotionFX.Editor.Tests", tests = ["UIFixture.*", "SimulatedObjectModelTestsFixture.*", "TestParametersFixture.*", "CanSeeJointsFixture.*", "LODSkinnedMeshFixtureTests/LODSkinnedMeshFixture.CheckLODLevels/*"] },
{ target = "EMotionFX.Tests", tests = ["UIFixture.*", "SimulatedObjectModelTestsFixture.*", "TestParametersFixture.*", "CanSeeJointsFixture.*"] },
{ target = "AzCore.Tests", tests = ["AllocatorsTestFixtureLeakDetectionDeathTest_SKIPCODECOVERAGE.AllocatorLeak"] },
]
[spartia.configuration.shard]
# Long tests that will be sharded
include_filter = [
{ target = "AzCore.Tests", policy = "fixture_contiguous" },
{ target = "AzToolsFramework.Tests", policy = "fixture_contiguous" },
{ target = "Framework.Tests", policy = "test_interleaved" },
{ target = "LmbrCentral.Editor.Tests", policy = "test_interleaved" },
{ target = "EditorLib.Tests", policy = "test_interleaved" },
{ target = "PhysX.Tests", policy = "test_interleaved" },
{ target = "ImageProcessing.Tests", policy = "test_interleaved" },
{ target = "Atom_RPI.Tests", policy = "test_interleaved" },
{ target = "Atom_RHI.Tests", policy = "test_interleaved" },
{ target = "AzManipulatorFramework.Tests", policy = "test_interleaved" },
{ target = "WhiteBox.Editor.Tests", policy = "test_interleaved" },
{ target = "AzManipulatorTestFramework.Tests", policy = "test_interleaved" },
{ target = "AtomCore.Tests", policy = "test_interleaved" },
{ target = "ImageProcessingAtom.Editor.Tests", policy = "test_interleaved" },
{ target = "EditorPythonBindings.Tests", policy = "test_interleaved" },
{ target = "Atom_Utils.Tests", policy = "test_interleaved" },
{ target = "AudioEngineWwise.Editor.Tests", policy = "test_interleaved" },
{ target = "Multiplayer.Tests", policy = "test_interleaved" },
{ target = "LmbrCentral.Tests", policy = "test_interleaved" },
{ target = "LyMetricsShared.Tests", policy = "fixture_contiguous" },
{ target = "PhysX.Editor.Tests", policy = "test_interleaved" },
{ target = "ComponentEntityEditorPlugin.Tests", policy = "test_interleaved" },
{ target = "DeltaCataloger.Tests", policy = "test_interleaved" },
{ target = "GradientSignal.Tests", policy = "test_interleaved" },
{ target = "LyShine.Tests", policy = "test_interleaved" },
{ target = "EMotionFX.Editor.Tests", policy = "test_interleaved" },
{ target = "EMotionFX.Tests", policy = "test_interleaved" },
{ target = "CrySystem.Tests", policy = "test_interleaved" },
]
[spartia.configuration.instrumentation]
abs_bin = "${instrumentation_bin}"
[spartia.configuration.instrumentation.errors]
# AzCppCoverage error codes
incorrect_args = -1618178468
[spartia.configuration.instrumentation.test_coverage]
args = "--export_type cobertura:\"{test_run_coverage_file}\""
[spartia.configuration.instrumentation.test_selection]
args = "--gtest_filter={test_selection}"
[spartia.configuration.instrumentation.test_enumeration]
args = "--gtest_list_tests"
[spartia.configuration.instrumentation.test_results]
args = "--gtest_output=xml:\"{test_run_results_file}\""
[spartia.configuration.instrumentation.test_results.errors]
test_success = 0
test_failures = 1
[spartia.configuration.instrumentation.binary_type.dynlib]
abs_bin = "{bin_dir}\\AzTestRunner.exe"
args = "\"{bin_dir}\\{test_target}.dll\" AzRunUnitTests"
[spartia.configuration.instrumentation.binary_type.dynlib.test_enumeration]
args = "--stdout_to_file \"{test_enumeration_file}\" {test_enumeration}"
[spartia.configuration.instrumentation.binary_type.dynlib.test_shard_selection]
args = "--args_from_file \"{test_shard_selection_file}\""
[spartia.configuration.instrumentation.binary_type.dynlib.errors]
# AzTestRunner error codes
failed_to_find_target_bin = 102
incorrect_args = 101
known_errors = [ 103, 104]
[spartia.configuration.instrumentation.binary_type.executable]
abs_bin = "{bin_dir}\\{test_target}.exe"
[spartia.configuration.instrumentation.binary_type.executable.test_enumeration]
args = "--stdout_to_file \"{test_enumeration_file}\" {test_enumeration}"
[spartia.configuration.instrumentation.binary_type.executable.test_shard_selection]
args = "--args_from_file \"{test_shard_selection_file}\""
[spartia.configuration.test_run.seed]
instrumentation_args = "--modules \"{bin_dir}\" --excluded_modules \"{binary_type.dynlib.abs_bin}\" --sources \"{repo_dir}\" --no_breakpoints {test_coverage} -- "
{
"meta": {
"platform": "${platform}",
"timestamp": "${timestamp}"
},
"repo": {
"root": "${repo_dir}"
},
"workspace": {
"temp": {
"root": "${temp_dir}",
"relative_paths": {
"artifact_dir": "RuntimeArtifact"
}
},
"persistent": {
"root": "${persistent_dir}",
"relative_paths": {
"test_impact_data_file": "TestImpactData.spartia",
"enumeration_cache_dir": "EnumerationCache"
}
}
},
"artifacts": {
"static": {
"build_target_descriptor": {
"dir": "${source_target_mapping_dir}",
"target_sources": {
"static": {
"include_filters": [
".h", ".hpp", ".hxx", ".inl", ".c", ".cpp", ".cxx"
]
},
"autogen": {
"input_output_pairer": "(.*)\\..*",
"input": {
"include_filters": [
".xml"
]
}
}
}
},
"dependency_graph_data": {
"dir": "${target_dependency_dir}",
"matchers": {
"target_dependency_file": "target\\.(.*)\\.(dependers)?",
"target_vertex": "(?:(.*)\\n|(.*)"
}
},
"test_target_meta": {
"file": "${test_target_type_file}"
}
}
},
"test_engine": {
"test_runner": {
"bin": "${test_runner_bin}"
},
"instrumentation": {
"bin": "${instrumentation_bin}"
}
},
"target": {
"dir": "${bin_dir}",
"exclude": [
],
"shard": [
{
"policy": "fixture_contiguous",
"target": "AzCore.Tests"
},
{
"policy": "fixture_contiguous",
"target": "AzToolsFramework.Tests"
},
{
"policy": "test_interleaved",
"target": "Framework.Tests"
},
{
"policy": "test_interleaved",
"target": "LmbrCentral.Editor.Tests"
},
{
"policy": "test_interleaved",
"target": "EditorLib.Tests"
},
{
"policy": "test_interleaved",
"target": "PhysX.Tests"
},
{
"policy": "test_interleaved",
"target": "ImageProcessing.Tests"
},
{
"policy": "test_interleaved",
"target": "Atom_RPI.Tests"
},
{
"policy": "test_interleaved",
"target": "Atom_RHI.Tests"
},
{
"policy": "test_interleaved",
"target": "AzManipulatorFramework.Tests"
},
{
"policy": "test_interleaved",
"target": "WhiteBox.Editor.Tests"
},
{
"policy": "test_interleaved",
"target": "ImageProcessing.Tests"
},
{
"policy": "test_interleaved",
"target": "AzManipulatorTestFramework.Tests"
},
{
"policy": "test_interleaved",
"target": "AtomCore.Tests"
},
{
"policy": "test_interleaved",
"target": "ImageProcessingAtom.Editor.Tests"
},
{
"policy": "test_interleaved",
"target": "EditorPythonBindings.Tests"
},
{
"policy": "test_interleaved",
"target": "Atom_Utils.Tests"
},
{
"policy": "test_interleaved",
"target": "AudioEngineWwise.Editor.Tests"
},
{
"policy": "test_interleaved",
"target": "Multiplayer.Tests"
},
{
"policy": "test_interleaved",
"target": "LmbrCentral.Tests"
},
{
"policy": "fixture_contiguous",
"target": "LyMetricsShared.Tests"
},
{
"policy": "test_interleaved",
"target": "PhysX.Editor.Tests"
},
{
"policy": "test_interleaved",
"target": "ComponentEntityEditorPlugin.Tests"
},
{
"policy": "test_interleaved",
"target": "DeltaCataloger.Tests"
},
{
"policy": "test_interleaved",
"target": "GradientSignal.Tests"
},
{
"policy": "test_interleaved",
"target": "LyShine.Tests"
},
{
"policy": "test_interleaved",
"target": "EMotionFX.Editor.Tests"
},
{
"policy": "test_interleaved",
"target": "EMotionFX.Tests"
},
{
"policy": "test_interleaved",
"target": "CrySystem.Tests"
}
]
}
}
cmake/TestImpactFramework/LYTestImpactFramework.cmake
+94 -87
View File
@@ -15,6 +15,9 @@ option(LY_TEST_IMPACT_ACTIVE "Enable test impact framework" OFF)
# Path to test instrumentation binary
option(LY_TEST_IMPACT_INSTRUMENTATION_BIN "Path to test impact framework instrumentation binary" OFF)
# Name of test impact framework console static library target
set(LY_TEST_IMPACT_CONSOLE_STATIC_TARGET "TestImpact.Frontend.Console.Static")
# Name of test impact framework console target
set(LY_TEST_IMPACT_CONSOLE_TARGET "TestImpact.Frontend.Console")
@@ -33,11 +36,8 @@ set(LY_TEST_IMPACT_SOURCE_TARGET_MAPPING_DIR "${LY_TEST_IMPACT_ARTIFACT_DIR}/Map
# Directory for build target dependency/depender graphs
set(LY_TEST_IMPACT_TARGET_DEPENDENCY_DIR "${LY_TEST_IMPACT_ARTIFACT_DIR}/Dependency")
# Directory for test type enumeration files
set(LY_TEST_IMPACT_TEST_TYPE_DIR "${LY_TEST_IMPACT_ARTIFACT_DIR}/TestType")
# Master test enumeration file for all test types
set(LY_TEST_IMPACT_TEST_TYPE_FILE "${LY_TEST_IMPACT_TEST_TYPE_DIR}/All.tests")
set(LY_TEST_IMPACT_TEST_TYPE_FILE "${LY_TEST_IMPACT_ARTIFACT_DIR}/TestType/All.tests")
#! ly_test_impact_rebase_file_to_repo_root: rebases the relative and/or absolute path to be relative to repo root directory and places the resulting path in quotes.
#
@@ -93,14 +93,13 @@ function(ly_test_impact_get_test_launch_method TARGET_NAME LAUNCH_METHOD)
endif()
endfunction()
#! ly_test_impact_extract_google_test: explodes a composite google test string into namespace, test and suite components.
#! ly_test_impact_extract_google_test_name: extracts the google test name from the composite 'namespace::test_name' string
#
# \arg:COMPOSITE_TEST test in the form 'namespace::test'
# \arg:TEST_NAMESPACE namespace for the test
# \arg:TEST_NAME name of test
function(ly_test_impact_extract_google_test COMPOSITE_TEST TEST_NAMESPACE TEST_NAME)
get_property(test_components GLOBAL PROPERTY LY_ALL_TESTS_${COMPOSITE_TEST}_TEST_NAME)
# Namespace and test are mandetiry
# Namespace and test are mandatory
string(REPLACE "::" ";" test_components ${test_components})
list(LENGTH test_components num_test_components)
if(num_test_components LESS 2)
@@ -113,37 +112,72 @@ function(ly_test_impact_extract_google_test COMPOSITE_TEST TEST_NAMESPACE TEST_N
set(${TEST_NAME} ${test_name} PARENT_SCOPE)
endfunction()
#! ly_test_impact_extract_python_test: explodes a composite python test string into filename, namespace, test and suite components.
#! ly_test_impact_extract_python_test_name: extracts the python test name from the composite 'namespace::test_name' string
#
# \arg:COMPOSITE_TEST test in form 'namespace::test' or 'test'
# \arg:TEST_NAMESPACE namespace for the test (optional)
# \arg:TEST_NAME name of test
# \arg:TEST_FILE the Python script path for this test
function(ly_test_impact_extract_python_test COMPOSITE_TEST TEST_NAMESPACE TEST_NAME TEST_FILE)
function(ly_test_impact_extract_python_test COMPOSITE_TEST TEST_NAME)
get_property(test_components GLOBAL PROPERTY LY_ALL_TESTS_${COMPOSITE_TEST}_TEST_NAME)
get_property(test_file GLOBAL PROPERTY LY_ALL_TESTS_${COMPOSITE_TEST}_SCRIPT_PATH)
# namespace is optional, in which case this component will be simply the test name
string(REPLACE "::" ";" test_components ${test_components})
list(LENGTH test_components num_test_components)
if(num_test_components GREATER 1)
list(GET test_components 0 test_namespace)
list(GET test_components 1 test_name)
else()
set(test_namespace "")
set(test_name ${test_components})
endif()
set(${TEST_NAME} ${test_name} PARENT_SCOPE)
endfunction()
#! ly_test_impact_extract_google_test_params: extracts the google test name and command parameters.
#
# \arg:COMPOSITE_TEST test in the form 'namespace::test'
# \arg:TEST_NAME name of test
# \arg:TEST_COMMAND optional command arguments to run the test
function(ly_test_impact_extract_google_test_params COMPOSITE_TEST TEST_NAME TEST_COMMAND)
get_property(test_command GLOBAL PROPERTY LY_ALL_TESTS_${COMPOSITE_TEST}_TEST_COMMAND)
# Namespace and test are mandatory
string(REPLACE "::" ";" test_components ${COMPOSITE_TEST})
list(LENGTH test_components num_test_components)
if(num_test_components LESS 2)
message(FATAL_ERROR "The test ${test_components} appears to have been specified without a namespace, i.e.:\ly_add_googletest/benchmark(NAME ${test_components})\nInstead of (perhaps):\ly_add_googletest/benchmark(NAME Gem::${test_components})\nPlease add the missing namespace before proceeding.")
endif()
list(GET test_components 0 test_namespace)
list(GET test_components 1 test_name)
set(${TEST_NAMESPACE} ${test_namespace} PARENT_SCOPE)
set(${TEST_NAME} ${test_name} PARENT_SCOPE)
set(${TEST_COMMAND} ${test_command} PARENT_SCOPE)
endfunction()
#! ly_test_impact_extract_python_test_params: extracts the python test name and relative script path parameters.
#
# \arg:COMPOSITE_TEST test in form 'namespace::test' or 'test'
# \arg:TEST_NAME name of test
# \arg:SCRIPT_PATH name of test
function(ly_test_impact_extract_python_test_params COMPOSITE_TEST TEST_NAME SCRIPT_PATH)
get_property(script_path GLOBAL PROPERTY LY_ALL_TESTS_${COMPOSITE_TEST}_SCRIPT_PATH)
# namespace is optional, in which case this component will be simply the test name
string(REPLACE "::" ";" test_components ${COMPOSITE_TEST})
list(LENGTH test_components num_test_components)
if(num_test_components GREATER 1)
list(GET test_components 1 test_name)
else()
set(test_name ${test_components})
endif()
# Get python script path relative to repo root
ly_test_impact_rebase_file_to_repo_root(
${test_file}
test_file
${script_path}
script_path
${LY_ROOT_FOLDER}
)
set(${TEST_NAMESPACE} ${test_namespace} PARENT_SCOPE)
set(${TEST_NAME} ${test_name} PARENT_SCOPE)
set(${TEST_FILE} ${test_file} PARENT_SCOPE)
set(${SCRIPT_PATH} ${script_path} PARENT_SCOPE)
endfunction()
#! ly_test_impact_write_test_enumeration_file: exports the master test lists to file.
@@ -164,25 +198,26 @@ function(ly_test_impact_write_test_enumeration_file TEST_ENUMERATION_TEMPLATE_FI
message(TRACE "Parsing ${test}")
get_property(test_type GLOBAL PROPERTY LY_ALL_TESTS_${test}_TEST_LIBRARY)
get_property(test_suite GLOBAL PROPERTY LY_ALL_TESTS_${test}_TEST_SUITE)
get_property(test_timeout GLOBAL PROPERTY LY_ALL_TESTS_${test}_TEST_TIMEOUT)
if("${test_type}" STREQUAL "pytest")
# Python tests
ly_test_impact_extract_python_test(${test} test_namespace test_name test_file)
list(APPEND python_tests " { \"name\": \"${test_name}\", \"namespace\": \"${test_namespace}\", \"suite\": \"${test_suite}\", \"path\": \"${test_file}\" }")
ly_test_impact_extract_python_test_params(${test} test_name script_path)
list(APPEND python_tests " { \"name\": \"${test_name}\", \"suite\": \"${test_suite}\", \"script\": \"${script_path}\", \"timeout\":${test_timeout} }")
elseif("${test_type}" STREQUAL "pytest_editor")
# Python editor tests
ly_test_impact_extract_python_test(${test} test_namespace test_name test_file)
list(APPEND python_editor_tests " { \"name\": \"${test_name}\", \"namespace\": \"${test_namespace}\", \"suite\": \"${test_suite}\", \"path\": \"${test_file}\" }")
ly_test_impact_extract_python_test_params(${test} test_name script_path)
list(APPEND python_editor_tests " { \"name\": \"${test_name}\", \"suite\": \"${test_suite}\", \"script\": \"${script_path}\", \"timeout\":${test_timeout} }")
elseif("${test_type}" STREQUAL "googletest")
# Google tests
ly_test_impact_extract_google_test(${test} test_namespace test_name)
ly_test_impact_extract_google_test_params(${test} test_name test_command)
ly_test_impact_get_test_launch_method(${test_name} launch_method)
list(APPEND google_tests " { \"name\": \"${test_name}\", \"namespace\": \"${test_namespace}\", \"suite\": \"${test_suite}\", \"launch_method\": \"${launch_method}\" }")
list(APPEND google_tests " { \"name\": \"${test_name}\", \"suite\": \"${test_suite}\", \"command\": \"${test_command}\", \"timeout\":${test_timeout}, \"launch_method\": \"${launch_method}\" }")
elseif("${test_type}" STREQUAL "googlebenchmark")
# Google benchmarks
ly_test_impact_extract_google_test(${test} test_namespace test_name)
list(APPEND google_benchmarks " { \"name\": \"${test_name}\", \"namespace\": \"${test_namespace}\", \"suite\": \"${test_suite}\" }")
ly_test_impact_extract_google_test_params(${test} test_name test_command)
list(APPEND google_benchmarks " { \"name\": \"${test_name}\", \"suite\": \"${test_suite}\", \"command\": \"${test_command}\", \"timeout\":${test_timeout} }")
else()
message("${test} is of unknown type (TEST_LIBRARY property is empty)")
message("${test_name} is of unknown type (TEST_LIBRARY property is empty)")
list(APPEND unknown_tests " { \"name\": \"${test}\" }")
endif()
endforeach()
@@ -242,12 +277,7 @@ function(ly_test_impact_export_source_target_mappings MAPPING_TEMPLATE_FILE)
endif()
# Static source file mappings
get_target_property(target_type ${target} TYPE)
if("${target_type}" STREQUAL "INTERFACE_LIBRARY")
get_target_property(static_sources ${target}_HEADERS SOURCES)
else()
get_target_property(static_sources ${target} SOURCES)
endif()
get_target_property(static_sources ${target} SOURCES)
# Rebase static source files to repo root
ly_test_impact_rebase_files_to_repo_root(
@@ -271,75 +301,52 @@ endfunction()
#
# \arg:CONFIG_TEMPLATE_FILE path to the runtime configuration template file
# \arg:PERSISTENT_DATA_DIR path to the test impact framework persistent data directory
# \arg:RUNTIME_BIN_DIR path to repo binary ourput directory
function(ly_test_impact_write_config_file CONFIG_TEMPLATE_FILE PERSISTENT_DATA_DIR RUNTIME_BIN_DIR)
set(repo_dir ${LY_ROOT_FOLDER})
# \arg:BIN_DIR path to repo binary output directory
function(ly_test_impact_write_config_file CONFIG_TEMPLATE_FILE PERSISTENT_DATA_DIR BIN_DIR)
# Platform this config file is being generated for
set(platform ${PAL_PLATFORM_NAME})
# SparTIA instrumentation binary
# Timestamp this config file was generated at
string(TIMESTAMP timestamp "%Y-%m-%d %H:%M:%S")
# Instrumentation binary
if(NOT LY_TEST_IMPACT_INSTRUMENTATION_BIN)
message(FATAL_ERROR "No test impact framework instrumentation binary was specified, please provide the path with option LY_TEST_IMPACT_INSTRUMENTATION_BIN")
endif()
file(TO_CMAKE_PATH ${LY_TEST_IMPACT_INSTRUMENTATION_BIN} instrumentation_bin)
# test impact framework working dir
ly_test_impact_rebase_file_to_repo_root(
${LY_TEST_IMPACT_WORKING_DIR}
working_dir
${LY_ROOT_FOLDER}
)
# test impact framework console binary dir
ly_test_impact_rebase_file_to_repo_root(
${RUNTIME_BIN_DIR}
runtime_bin_dir
${LY_ROOT_FOLDER}
)
# Test dir
ly_test_impact_rebase_file_to_repo_root(
"${PERSISTENT_DATA_DIR}/Tests"
tests_dir
${LY_ROOT_FOLDER}
)
# Testrunner binary
set(test_runner_bin $<TARGET_FILE:AzTestRunner>)
# Repository root
set(repo_dir ${LY_ROOT_FOLDER})
# Test impact framework output binary dir
set(bin_dir ${BIN_DIR})
# Temp dir
ly_test_impact_rebase_file_to_repo_root(
"${LY_TEST_IMPACT_TEMP_DIR}"
temp_dir
${LY_ROOT_FOLDER}
)
set(temp_dir "${LY_TEST_IMPACT_TEMP_DIR}")
# Persistent dir
set(persistent_dir "${PERSISTENT_DATA_DIR}")
# Source to target mappings dir
ly_test_impact_rebase_file_to_repo_root(
"${LY_TEST_IMPACT_SOURCE_TARGET_MAPPING_DIR}"
source_target_mapping_dir
${LY_ROOT_FOLDER}
)
set(source_target_mapping_dir "${LY_TEST_IMPACT_SOURCE_TARGET_MAPPING_DIR}")
# Test type artifact dir
ly_test_impact_rebase_file_to_repo_root(
"${LY_TEST_IMPACT_TEST_TYPE_DIR}"
test_type_dir
${LY_ROOT_FOLDER}
)
# Test type artifact file
set(test_target_type_file "${LY_TEST_IMPACT_TEST_TYPE_FILE}")
# Build dependency artifact dir
ly_test_impact_rebase_file_to_repo_root(
"${LY_TEST_IMPACT_TARGET_DEPENDENCY_DIR}"
target_dependency_dir
${LY_ROOT_FOLDER}
)
set(target_dependency_dir "${LY_TEST_IMPACT_TARGET_DEPENDENCY_DIR}")
# Substitute config file template with above vars
file(READ "${CONFIG_TEMPLATE_FILE}" config_file)
string(CONFIGURE ${config_file} config_file)
# Write out entire config contents to a file in the build directory of the test impact framework console target
string(TIMESTAMP timestamp "%Y-%m-%d %H:%M:%S")
set(header "# Test Impact Framework configuration file for Lumberyard\n# Platform: ${CMAKE_SYSTEM_NAME}\n# Build: $<CONFIG>\n# ${timestamp}")
file(GENERATE
OUTPUT "${CMAKE_RUNTIME_OUTPUT_DIRECTORY}/$<CONFIG>/$<TARGET_FILE_BASE_NAME:${LY_TEST_IMPACT_CONSOLE_TARGET}>.$<CONFIG>.cfg"
CONTENT "${header}\n\n${config_file}"
OUTPUT "${CMAKE_RUNTIME_OUTPUT_DIRECTORY}/$<CONFIG>/$<TARGET_FILE_BASE_NAME:${LY_TEST_IMPACT_CONSOLE_TARGET}>.$<CONFIG>.json"
CONTENT ${config_file}
)
endfunction()
@@ -353,7 +360,7 @@ function(ly_test_impact_post_step)
set(persistent_data_dir "${LY_ROOT_FOLDER}/Tests/test_impact_framework/${CMAKE_SYSTEM_NAME}/$<CONFIG>")
# Directory for binaries built for this profile
set(runtime_bin_dir "${CMAKE_RUNTIME_OUTPUT_DIRECTORY}/$<CONFIG>")
set(bin_dir "${CMAKE_RUNTIME_OUTPUT_DIRECTORY}/$<CONFIG>")
# Erase any existing non-persistent data to avoid getting test impact framework out of sync with current repo state
file(REMOVE_RECURSE "${LY_TEST_IMPACT_WORKING_DIR}")
@@ -372,7 +379,7 @@ function(ly_test_impact_post_step)
ly_test_impact_write_config_file(
"cmake/TestImpactFramework/ConsoleFrontendConfig.in"
${persistent_data_dir}
${runtime_bin_dir}
${bin_dir}
)
# Copy over the graphviz options file for the build dependency graphs
@@ -380,6 +387,6 @@ function(ly_test_impact_post_step)
file(COPY "cmake/TestImpactFramework/CMakeGraphVizOptions.cmake" DESTINATION ${CMAKE_BINARY_DIR})
# Set the above config file as the default config file to use for the test impact framework console target
target_compile_definitions(${LY_TEST_IMPACT_CONSOLE_TARGET} PRIVATE "LY_TEST_IMPACT_DEFAULT_CONFIG_FILE=\"$<TARGET_FILE_BASE_NAME:${LY_TEST_IMPACT_CONSOLE_TARGET}>.$<CONFIG>.cfg\"")
target_compile_definitions(${LY_TEST_IMPACT_CONSOLE_STATIC_TARGET} PUBLIC "LY_TEST_IMPACT_DEFAULT_CONFIG_FILE=\"${CMAKE_RUNTIME_OUTPUT_DIRECTORY}/$<CONFIG>/$<TARGET_FILE_BASE_NAME:${LY_TEST_IMPACT_CONSOLE_TARGET}>.$<CONFIG>.json\"")
message(DEBUG "Test impact framework post steps complete")
endfunction()
endfunction()