Delete moved and obselete runtime files
This commit is contained in:
jonawals
-35
@@ -1,35 +0,0 @@
|
||||
/*
|
||||
* All or portions of this file Copyright (c) Amazon.com, Inc. or its affiliates or
|
||||
* its licensors.
|
||||
*
|
||||
* For complete copyright and license terms please see the LICENSE at the root of this
|
||||
* distribution (the "License"). All use of this software is governed by the License,
|
||||
* or, if provided, by the license below or the license accompanying this file. Do not
|
||||
* remove or modify any license notices. This file is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
*
|
||||
*/
|
||||
|
||||
#pragma once
|
||||
|
||||
namespace TestImpact
|
||||
{
|
||||
//! Convinience namespace for allowing bitwise operations on enum classes.
|
||||
//! @note Any types declared in this namespace will have the bitwise operator overloads declared within.
|
||||
namespace Bitwise
|
||||
{
|
||||
template<typename Flags>
|
||||
Flags operator|(Flags lhs, Flags rhs)
|
||||
{
|
||||
return static_cast<Flags>(
|
||||
static_cast<std::underlying_type<Flags>::type>(lhs) | static_cast<std::underlying_type<Flags>::type>(rhs));
|
||||
}
|
||||
|
||||
template<typename Flags>
|
||||
bool IsFlagSet(Flags flags, Flags flag)
|
||||
{
|
||||
return static_cast<bool>(
|
||||
static_cast<std::underlying_type<Flags>::type>(flags) & static_cast<std::underlying_type<Flags>::type>(flag));
|
||||
}
|
||||
} // namespace Bitwise
|
||||
} // namespace TestImpact
|
||||
-23
@@ -1,23 +0,0 @@
|
||||
/*
|
||||
* All or portions of this file Copyright (c) Amazon.com, Inc. or its affiliates or
|
||||
* its licensors.
|
||||
*
|
||||
* For complete copyright and license terms please see the LICENSE at the root of this
|
||||
* distribution (the "License"). All use of this software is governed by the License,
|
||||
* or, if provided, by the license below or the license accompanying this file. Do not
|
||||
* remove or modify any license notices. This file is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
*
|
||||
*/
|
||||
|
||||
#pragma once
|
||||
|
||||
namespace TestImpact
|
||||
{
|
||||
//! Generic callback result used by test impact systems.
|
||||
enum class CallbackResult : bool
|
||||
{
|
||||
Continue,
|
||||
Abort
|
||||
};
|
||||
} // namespace TestImpact
|
||||
-41
@@ -1,41 +0,0 @@
|
||||
/*
|
||||
* All or portions of this file Copyright (c) Amazon.com, Inc. or its affiliates or
|
||||
* its licensors.
|
||||
*
|
||||
* For complete copyright and license terms please see the LICENSE at the root of this
|
||||
* distribution (the "License"). All use of this software is governed by the License,
|
||||
* or, if provided, by the license below or the license accompanying this file. Do not
|
||||
* remove or modify any license notices. This file is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
*
|
||||
*/
|
||||
|
||||
#pragma once
|
||||
|
||||
#include <AzCore/IO/Path/Path.h>
|
||||
|
||||
namespace TestImpact
|
||||
{
|
||||
//! Wrapper for OS paths relative to a specified parent path.
|
||||
//! @note Mimics path semantics only, makes no guarantees about validity.
|
||||
class FrameworkPath
|
||||
{
|
||||
public:
|
||||
FrameworkPath() = default;
|
||||
//! Creates a path with no parent.
|
||||
explicit FrameworkPath(const AZ::IO::Path& absolutePath);
|
||||
//! Creates a path with an absolute path and path relative to the specified parent path.
|
||||
explicit FrameworkPath(const AZ::IO::Path& absolutePath, const FrameworkPath& relativeTo);
|
||||
|
||||
//! Retrieves the absolute path.
|
||||
const AZ::IO::Path& Absolute() const;
|
||||
//! Retrieves the path relative to the specified parent path.
|
||||
const AZ::IO::Path& Relative() const;
|
||||
|
||||
private:
|
||||
//! The absolute path value.
|
||||
AZ::IO::Path m_absolutePath;
|
||||
//! The path value relative to the specified parent path.
|
||||
AZ::IO::Path m_relativePath;
|
||||
};
|
||||
} // namespace TestImpact
|
||||
-27
@@ -1,27 +0,0 @@
|
||||
/*
|
||||
* All or portions of this file Copyright (c) Amazon.com, Inc. or its affiliates or
|
||||
* its licensors.
|
||||
*
|
||||
* For complete copyright and license terms please see the LICENSE at the root of this
|
||||
* distribution (the "License"). All use of this software is governed by the License,
|
||||
* or, if provided, by the license below or the license accompanying this file. Do not
|
||||
* remove or modify any license notices. This file is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
*
|
||||
*/
|
||||
|
||||
#pragma once
|
||||
|
||||
#include <AzCore/std/containers/vector.h>
|
||||
#include <AzCore/std/string/string.h>
|
||||
|
||||
namespace TestImpact
|
||||
{
|
||||
//! Artifact produced by the unified diff parsing process representing the file CRUD operations of a given diff.
|
||||
struct ChangeList
|
||||
{
|
||||
AZStd::vector<AZStd::string> m_createdFiles;
|
||||
AZStd::vector<AZStd::string> m_updatedFiles;
|
||||
AZStd::vector<AZStd::string> m_deletedFiles;
|
||||
};
|
||||
} // namespace TestImpact
|
||||
-175
@@ -1,175 +0,0 @@
|
||||
/*
|
||||
* All or portions of this file Copyright (c) Amazon.com, Inc. or its affiliates or
|
||||
* its licensors.
|
||||
*
|
||||
* For complete copyright and license terms please see the LICENSE at the root of this
|
||||
* distribution (the "License"). All use of this software is governed by the License,
|
||||
* or, if provided, by the license below or the license accompanying this file. Do not
|
||||
* remove or modify any license notices. This file is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
*
|
||||
*/
|
||||
|
||||
#include <Artifact/Factory/TestImpactChangeListFactory.h>
|
||||
#include <Artifact/TestImpactArtifactException.h>
|
||||
|
||||
#include <AzCore/std/optional.h>
|
||||
|
||||
namespace TestImpact
|
||||
{
|
||||
namespace Utils
|
||||
{
|
||||
template<typename C>
|
||||
auto split(C&& str, const AZStd::string& delimiter)
|
||||
{
|
||||
AZStd::vector<C> strings;
|
||||
|
||||
for (auto p = str.data(), end = p + str.length(); p != end; p += ((p == end) ? 0 : delimiter.length()))
|
||||
{
|
||||
const auto pre = p;
|
||||
p = AZStd::search(pre, end, delimiter.cbegin(), delimiter.cend());
|
||||
|
||||
if (p != pre)
|
||||
{
|
||||
strings.emplace_back(pre, p - pre);
|
||||
}
|
||||
}
|
||||
|
||||
return strings;
|
||||
}
|
||||
} // namespace Utils
|
||||
|
||||
namespace UnifiedDiff
|
||||
{
|
||||
class UnifiedDiffParser
|
||||
{
|
||||
public:
|
||||
ChangeList Parse(const AZStd::string& unifiedDiff);
|
||||
|
||||
private:
|
||||
AZStd::optional<AZStd::string_view> GetTargetFile(const AZStd::string_view& targetFile);
|
||||
ChangeList GenerateChangelist(
|
||||
const AZStd::vector<AZStd::optional<AZStd::string_view>>& src,
|
||||
const AZStd::vector<AZStd::optional<AZStd::string_view>>& dst);
|
||||
|
||||
const AZStd::string m_srcFilePrefix = "--- ";
|
||||
const AZStd::string m_dstFilePrefix = "+++ ";
|
||||
const AZStd::string m_gitTargetPrefix = "b/";
|
||||
const AZStd::string m_perforceTargetPrefix = "/b/";
|
||||
const AZStd::string m_renameFromPrefix = "rename from ";
|
||||
const AZStd::string m_renameToPrefix = "rename to ";
|
||||
const AZStd::string m_nullFile = "/dev/null";
|
||||
bool m_hasGitHeader = false;
|
||||
};
|
||||
|
||||
AZStd::optional<AZStd::string_view> UnifiedDiffParser::GetTargetFile(const AZStd::string_view& targetFile)
|
||||
{
|
||||
size_t startIndex = 0;
|
||||
|
||||
if (targetFile.starts_with(m_renameFromPrefix))
|
||||
{
|
||||
startIndex = m_renameFromPrefix.length();
|
||||
}
|
||||
else if (targetFile.starts_with(m_renameToPrefix))
|
||||
{
|
||||
startIndex = m_renameToPrefix.length();
|
||||
}
|
||||
else if (targetFile.find(m_nullFile) != AZStd::string::npos)
|
||||
{
|
||||
return AZStd::nullopt;
|
||||
}
|
||||
else
|
||||
{
|
||||
startIndex = m_hasGitHeader ? m_gitTargetPrefix.size() + m_dstFilePrefix.size()
|
||||
: m_perforceTargetPrefix.size() + m_dstFilePrefix.size();
|
||||
}
|
||||
|
||||
const auto endIndex = targetFile.find('\t');
|
||||
|
||||
if (endIndex != AZStd::string::npos)
|
||||
{
|
||||
return targetFile.substr(startIndex, endIndex - startIndex);
|
||||
}
|
||||
|
||||
return targetFile.substr(startIndex);
|
||||
}
|
||||
|
||||
ChangeList UnifiedDiffParser::GenerateChangelist(
|
||||
const AZStd::vector<AZStd::optional<AZStd::string_view>>& src, const AZStd::vector<AZStd::optional<AZStd::string_view>>& dst)
|
||||
{
|
||||
AZ_TestImpact_Eval(src.size() == dst.size(), ArtifactException, "Change list source and destination file count mismatch");
|
||||
ChangeList changelist;
|
||||
|
||||
for (size_t i = 0; i < src.size(); i++)
|
||||
{
|
||||
if (!src[i].has_value())
|
||||
{
|
||||
changelist.m_createdFiles.emplace_back(dst[i].value());
|
||||
}
|
||||
else if (!dst[i].has_value())
|
||||
{
|
||||
changelist.m_deletedFiles.emplace_back(src[i].value());
|
||||
}
|
||||
else if (src[i] != dst[i])
|
||||
{
|
||||
changelist.m_deletedFiles.emplace_back(src[i].value());
|
||||
changelist.m_createdFiles.emplace_back(dst[i].value());
|
||||
}
|
||||
else
|
||||
{
|
||||
changelist.m_updatedFiles.emplace_back(src[i].value());
|
||||
}
|
||||
}
|
||||
|
||||
return changelist;
|
||||
}
|
||||
|
||||
ChangeList UnifiedDiffParser::Parse(const AZStd::string& unifiedDiff)
|
||||
{
|
||||
const AZStd::string GitHeader = "diff --git";
|
||||
const auto lines = Utils::split<AZStd::string_view>(unifiedDiff, "\n");
|
||||
|
||||
AZStd::vector<AZStd::optional<AZStd::string_view>> src;
|
||||
AZStd::vector<AZStd::optional<AZStd::string_view>> dst;
|
||||
|
||||
for (const auto& line : lines)
|
||||
{
|
||||
if (line.starts_with(GitHeader))
|
||||
{
|
||||
m_hasGitHeader = true;
|
||||
}
|
||||
else if (line.starts_with(m_srcFilePrefix))
|
||||
{
|
||||
src.emplace_back(GetTargetFile(line));
|
||||
}
|
||||
else if (line.starts_with(m_dstFilePrefix))
|
||||
{
|
||||
dst.emplace_back(GetTargetFile(line));
|
||||
}
|
||||
else if (line.starts_with(m_renameFromPrefix))
|
||||
{
|
||||
src.emplace_back(GetTargetFile(line));
|
||||
}
|
||||
else if (line.starts_with(m_renameToPrefix))
|
||||
{
|
||||
dst.emplace_back(GetTargetFile(line));
|
||||
}
|
||||
}
|
||||
|
||||
return GenerateChangelist(src, dst);
|
||||
}
|
||||
|
||||
ChangeList ChangeListFactory(const AZStd::string& unifiedDiff)
|
||||
{
|
||||
AZ_TestImpact_Eval(!unifiedDiff.empty(), ArtifactException, "Unified diff is empty");
|
||||
UnifiedDiffParser diff;
|
||||
ChangeList changeList = diff.Parse(unifiedDiff);
|
||||
AZ_TestImpact_Eval(
|
||||
!changeList.m_createdFiles.empty() ||
|
||||
!changeList.m_updatedFiles.empty() ||
|
||||
!changeList.m_deletedFiles.empty(),
|
||||
ArtifactException, "The unified diff contained no changes");
|
||||
return changeList;
|
||||
}
|
||||
} // namespace UnifiedDiff
|
||||
} // namespace TestImpact
|
||||
-29
@@ -1,29 +0,0 @@
|
||||
/*
|
||||
* All or portions of this file Copyright (c) Amazon.com, Inc. or its affiliates or
|
||||
* its licensors.
|
||||
*
|
||||
* For complete copyright and license terms please see the LICENSE at the root of this
|
||||
* distribution (the "License"). All use of this software is governed by the License,
|
||||
* or, if provided, by the license below or the license accompanying this file. Do not
|
||||
* remove or modify any license notices. This file is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
*
|
||||
*/
|
||||
|
||||
#pragma once
|
||||
|
||||
#include <Artifact/Dynamic/TestImpactChangeList.h>
|
||||
|
||||
#include <AzCore/std/containers/vector.h>
|
||||
#include <AzCore/std/string/string.h>
|
||||
|
||||
namespace TestImpact
|
||||
{
|
||||
namespace UnifiedDiff
|
||||
{
|
||||
//! Constructs a change list artifact from the specified unified diff data.
|
||||
//! @param unifiedDiffData The raw change list data in unified diff format.
|
||||
//! @return The constructed change list artifact.
|
||||
ChangeList ChangeListFactory(const AZStd::string& unifiedDiffData);
|
||||
} // namespace UnifiedDiff
|
||||
} // namespace TestImpact
|
||||
@@ -1,38 +0,0 @@
|
||||
/*
|
||||
* All or portions of this file Copyright (c) Amazon.com, Inc. or its affiliates or
|
||||
* its licensors.
|
||||
*
|
||||
* For complete copyright and license terms please see the LICENSE at the root of this
|
||||
* distribution (the "License"). All use of this software is governed by the License,
|
||||
* or, if provided, by the license below or the license accompanying this file. Do not
|
||||
* remove or modify any license notices. This file is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
*
|
||||
*/
|
||||
|
||||
#include <TestImpactFramework/TestImpactFrameworkPath.h>
|
||||
|
||||
namespace TestImpact
|
||||
{
|
||||
FrameworkPath::FrameworkPath(const AZ::IO::Path& absolutePath)
|
||||
{
|
||||
m_absolutePath = AZ::IO::Path(absolutePath).MakePreferred();
|
||||
m_relativePath = m_absolutePath.LexicallyRelative(m_absolutePath);
|
||||
}
|
||||
|
||||
FrameworkPath::FrameworkPath(const AZ::IO::Path& absolutePath, const FrameworkPath& relativeTo)
|
||||
{
|
||||
m_absolutePath = AZ::IO::Path(absolutePath).MakePreferred();
|
||||
m_relativePath = m_absolutePath.LexicallyRelative(relativeTo.Absolute());
|
||||
}
|
||||
|
||||
const AZ::IO::Path& FrameworkPath::Absolute() const
|
||||
{
|
||||
return m_absolutePath;
|
||||
}
|
||||
|
||||
const AZ::IO::Path& FrameworkPath::Relative() const
|
||||
{
|
||||
return m_relativePath;
|
||||
}
|
||||
} // namespace TestImpact
|
||||
-288
@@ -1,288 +0,0 @@
|
||||
/*
|
||||
* All or portions of this file Copyright (c) Amazon.com, Inc. or its affiliates or
|
||||
* its licensors.
|
||||
*
|
||||
* For complete copyright and license terms please see the LICENSE at the root of this
|
||||
* distribution (the "License"). All use of this software is governed by the License,
|
||||
* or, if provided, by the license below or the license accompanying this file. Do not
|
||||
* remove or modify any license notices. This file is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
*
|
||||
*/
|
||||
|
||||
#include <Artifact/Factory/TestImpactChangeListFactory.h>
|
||||
#include <Artifact/TestImpactArtifactException.h>
|
||||
|
||||
#include <AzCore/UnitTest/TestTypes.h>
|
||||
#include <AzTest/AzTest.h>
|
||||
|
||||
namespace UnitTest
|
||||
{
|
||||
TEST(ChangeListFactoryTest, NoRawData_ExpectArtifactException)
|
||||
{
|
||||
// Given an empty unified diff string
|
||||
const AZStd::string unifiedDiff;
|
||||
|
||||
try
|
||||
{
|
||||
// When attempting to construct the change list
|
||||
const TestImpact::ChangeList changeList = TestImpact::UnifiedDiff::ChangeListFactory(unifiedDiff);
|
||||
|
||||
// Do not expect this statement to be reachable
|
||||
FAIL();
|
||||
}
|
||||
catch ([[maybe_unused]] const TestImpact::ArtifactException& e)
|
||||
{
|
||||
// Expect an artifact exception
|
||||
SUCCEED();
|
||||
}
|
||||
catch (...)
|
||||
{
|
||||
// Do not expect any other exceptions
|
||||
FAIL();
|
||||
}
|
||||
}
|
||||
|
||||
TEST(ChangeListFactoryTest, NoChanges_ExpectArtifactException)
|
||||
{
|
||||
// Given a unified diff string with no changes
|
||||
const AZStd::string unifiedDiff = "On this day in 1738 absolutely nothing happened";
|
||||
|
||||
try
|
||||
{
|
||||
// When attempting to construct the change list
|
||||
const TestImpact::ChangeList changeList = TestImpact::UnifiedDiff::ChangeListFactory(unifiedDiff);
|
||||
|
||||
// Do not expect this statement to be reachable
|
||||
FAIL();
|
||||
}
|
||||
catch ([[maybe_unused]] const TestImpact::ArtifactException& e)
|
||||
{
|
||||
// Expect an artifact exception
|
||||
SUCCEED();
|
||||
}
|
||||
catch (...)
|
||||
{
|
||||
// Do not expect any other exceptions
|
||||
FAIL();
|
||||
}
|
||||
}
|
||||
|
||||
TEST(ChangeListFactoryTest, CreateOnly_ExpectValidChangeListWithFileCreateOperations)
|
||||
{
|
||||
// Given a unified diff with only one file creation and no file updates or deletions
|
||||
const AZStd::string unifiedDiff =
|
||||
"From f642a2f698452fc18484758b0046132415f09467 Mon Sep 17 00:00:00 2001\n"
|
||||
"From: user <user@website.com>\n"
|
||||
"Date: Sat, 13 Mar 2021 22:58:07 +0000\n"
|
||||
"Subject: Test\n"
|
||||
"\n"
|
||||
"---\n"
|
||||
" New.txt | 1 +\n"
|
||||
" create mode 100644 New.txt\n"
|
||||
"diff --git a/New.txt b/New.txt\n"
|
||||
"new file mode 100644\n"
|
||||
"index 0000000..30d74d2\n"
|
||||
"--- /dev/null\n"
|
||||
"+++ b/New.txt\n"
|
||||
"@@ -0,0 +1 @@\n"
|
||||
"+test\n"
|
||||
"\\ No newline at end of file\n"
|
||||
"-- \n"
|
||||
"2.30.0.windows.2\n"
|
||||
"\n"
|
||||
"\n";
|
||||
|
||||
// When attempting to construct the change list
|
||||
const TestImpact::ChangeList changeList = TestImpact::UnifiedDiff::ChangeListFactory(unifiedDiff);
|
||||
|
||||
// Expect the change list to contain the 1 created file
|
||||
EXPECT_EQ(changeList.m_createdFiles.size(), 1);
|
||||
EXPECT_TRUE(
|
||||
AZStd::find(changeList.m_createdFiles.begin(), changeList.m_createdFiles.end(), "New.txt") != changeList.m_createdFiles.end());
|
||||
|
||||
// Expect the change list to contain no updated files
|
||||
EXPECT_TRUE(changeList.m_updatedFiles.empty());
|
||||
|
||||
// Expect the change list to contain no deleted files
|
||||
EXPECT_TRUE(changeList.m_deletedFiles.empty());
|
||||
}
|
||||
|
||||
TEST(ChangeListFactoryTest, UpdateOnly_ExpectValidChangeListWithFileUpdateOperations)
|
||||
{
|
||||
// Given a unified diff with only one file update and no file creations or deletions
|
||||
const AZStd::string unifiedDiff =
|
||||
"From f642a2f698452fc18484758b0046132415f09467 Mon Sep 17 00:00:00 2001\n"
|
||||
"From: user <user@website.com>\n"
|
||||
"Date: Sat, 13 Mar 2021 22:58:07 +0000\n"
|
||||
"Subject: Test\n"
|
||||
"\n"
|
||||
"---\n"
|
||||
" A.txt | 2 +-\n"
|
||||
"diff --git a/A.txt b/A.txt\n"
|
||||
"index 7c4a013..e132db2 100644\n"
|
||||
"--- a/A.txt\n"
|
||||
"+++ b/A.txt\n"
|
||||
"@@ -1 +1 @@\n"
|
||||
"-aaa\n"
|
||||
"\\ No newline at end of file\n"
|
||||
"+zzz\n"
|
||||
"\\ No newline at end of file\n"
|
||||
"-- \n"
|
||||
"2.30.0.windows.2\n"
|
||||
"\n"
|
||||
"\n";
|
||||
|
||||
// When attempting to construct the change list
|
||||
const TestImpact::ChangeList changeList = TestImpact::UnifiedDiff::ChangeListFactory(unifiedDiff);
|
||||
|
||||
// Expect the change list to contain no created files
|
||||
EXPECT_TRUE(changeList.m_createdFiles.empty());
|
||||
|
||||
// Expect the change list to contain one updated file
|
||||
EXPECT_EQ(changeList.m_updatedFiles.size(), 1);
|
||||
EXPECT_TRUE(
|
||||
AZStd::find(changeList.m_updatedFiles.begin(), changeList.m_updatedFiles.end(), "A.txt") != changeList.m_updatedFiles.end());
|
||||
|
||||
// Expect the change list to contain no deleted files
|
||||
EXPECT_TRUE(changeList.m_deletedFiles.empty());
|
||||
}
|
||||
|
||||
TEST(ChangeListFactoryTest, DeleteOnly_ExpectValidChangeListWithFileDeleteOperations)
|
||||
{
|
||||
// Given a unified diff with only one file deletion and no file creations or updates
|
||||
const AZStd::string unifiedDiff =
|
||||
"From f642a2f698452fc18484758b0046132415f09467 Mon Sep 17 00:00:00 2001\n"
|
||||
"From: user <user@website.com>\n"
|
||||
"Date: Sat, 13 Mar 2021 22:58:07 +0000\n"
|
||||
"Subject: Test\n"
|
||||
"\n"
|
||||
"---\n"
|
||||
" B.txt | 1 -\n"
|
||||
" delete mode 100644 B.txt\n"
|
||||
"diff --git a/B.txt b/B.txt\n"
|
||||
"deleted file mode 100644\n"
|
||||
"index 01f02e3..0000000\n"
|
||||
"--- a/B.txt\n"
|
||||
"+++ /dev/null\n"
|
||||
"@@ -1 +0,0 @@\n"
|
||||
"-bbb\n"
|
||||
"\\ No newline at end of file\n"
|
||||
"-- \n"
|
||||
"2.30.0.windows.2\n"
|
||||
"\n"
|
||||
"\n";
|
||||
|
||||
// When attempting to construct the change list
|
||||
const TestImpact::ChangeList changeList = TestImpact::UnifiedDiff::ChangeListFactory(unifiedDiff);
|
||||
|
||||
// Expect the change list to contain no created files
|
||||
EXPECT_TRUE(changeList.m_createdFiles.empty());
|
||||
|
||||
// Expect the change list to contain no updated files
|
||||
EXPECT_TRUE(changeList.m_updatedFiles.empty());
|
||||
|
||||
// Expect the change list to contain one deleted file
|
||||
EXPECT_EQ(changeList.m_deletedFiles.size(), 1);
|
||||
EXPECT_TRUE(
|
||||
AZStd::find(changeList.m_deletedFiles.begin(), changeList.m_deletedFiles.end(), "B.txt") != changeList.m_deletedFiles.end());
|
||||
}
|
||||
|
||||
TEST(ChangeListFactoryTest, ParseUnifiedDiffWithAllPossibleOperations_ExpectChangeListMatchingOperations)
|
||||
{
|
||||
// Given a unified diff with created files, updated files, deleted files, renamed files and moved files
|
||||
const AZStd::string unifiedDiff =
|
||||
"From f642a2f698452fc18484758b0046132415f09467 Mon Sep 17 00:00:00 2001\n"
|
||||
"From: user <user@website.com>\n"
|
||||
"Date: Sat, 13 Mar 2021 22:58:07 +0000\n"
|
||||
"Subject: Test\n"
|
||||
"\n"
|
||||
"---\n"
|
||||
" A.txt | 2 +-\n"
|
||||
" B.txt | 1 -\n"
|
||||
" D.txt => Foo/D.txt | 0\n"
|
||||
" E.txt => Foo/Y.txt | 0\n"
|
||||
" New.txt | 1 +\n"
|
||||
" C.txt => X.txt | 0\n"
|
||||
" 6 files changed, 2 insertions(+), 2 deletions(-)\n"
|
||||
" delete mode 100644 B.txt\n"
|
||||
" rename D.txt => Foo/D.txt (100%)\n"
|
||||
" rename E.txt => Foo/Y.txt (100%)\n"
|
||||
" create mode 100644 New.txt\n"
|
||||
" rename C.txt => X.txt (100%)\n"
|
||||
"\n"
|
||||
"diff --git a/A.txt b/A.txt\n"
|
||||
"index 7c4a013..e132db2 100644\n"
|
||||
"--- a/A.txt\n"
|
||||
"+++ b/A.txt\n"
|
||||
"@@ -1 +1 @@\n"
|
||||
"-aaa\n"
|
||||
"\\ No newline at end of file\n"
|
||||
"+zzz\n"
|
||||
"\\ No newline at end of file\n"
|
||||
"diff --git a/B.txt b/B.txt\n"
|
||||
"deleted file mode 100644\n"
|
||||
"index 01f02e3..0000000\n"
|
||||
"--- a/B.txt\n"
|
||||
"+++ /dev/null\n"
|
||||
"@@ -1 +0,0 @@\n"
|
||||
"-bbb\n"
|
||||
"\\ No newline at end of file\n"
|
||||
"diff --git a/D.txt b/Foo/D.txt\n"
|
||||
"similarity index 100%\n"
|
||||
"rename from D.txt\n"
|
||||
"rename to Foo/D.txt\n"
|
||||
"diff --git a/E.txt b/Foo/Y.txt\n"
|
||||
"similarity index 100%\n"
|
||||
"rename from E.txt\n"
|
||||
"rename to Foo/Y.txt\n"
|
||||
"diff --git a/New.txt b/New.txt\n"
|
||||
"new file mode 100644\n"
|
||||
"index 0000000..30d74d2\n"
|
||||
"--- /dev/null\n"
|
||||
"+++ b/New.txt\n"
|
||||
"@@ -0,0 +1 @@\n"
|
||||
"+test\n"
|
||||
"\\ No newline at end of file\n"
|
||||
"diff --git a/C.txt b/X.txt\n"
|
||||
"similarity index 100%\n"
|
||||
"rename from C.txt\n"
|
||||
"rename to X.txt\n"
|
||||
"-- \n"
|
||||
"2.30.0.windows.2\n"
|
||||
"\n"
|
||||
"\n";
|
||||
|
||||
// When attempting to construct the change list
|
||||
const TestImpact::ChangeList changeList = TestImpact::UnifiedDiff::ChangeListFactory(unifiedDiff);
|
||||
|
||||
// Expect the change list to contain the 4 created files
|
||||
EXPECT_EQ(changeList.m_createdFiles.size(), 4);
|
||||
EXPECT_TRUE(
|
||||
AZStd::find(changeList.m_createdFiles.begin(), changeList.m_createdFiles.end(), "Foo/D.txt") !=
|
||||
changeList.m_createdFiles.end());
|
||||
EXPECT_TRUE(
|
||||
AZStd::find(changeList.m_createdFiles.begin(), changeList.m_createdFiles.end(), "Foo/Y.txt") !=
|
||||
changeList.m_createdFiles.end());
|
||||
EXPECT_TRUE(
|
||||
AZStd::find(changeList.m_createdFiles.begin(), changeList.m_createdFiles.end(), "X.txt") != changeList.m_createdFiles.end());
|
||||
EXPECT_TRUE(
|
||||
AZStd::find(changeList.m_createdFiles.begin(), changeList.m_createdFiles.end(), "New.txt") != changeList.m_createdFiles.end());
|
||||
|
||||
// Expect the change list to contain the 1 updated file
|
||||
EXPECT_EQ(changeList.m_updatedFiles.size(), 1);
|
||||
EXPECT_TRUE(
|
||||
AZStd::find(changeList.m_updatedFiles.begin(), changeList.m_updatedFiles.end(), "A.txt") != changeList.m_updatedFiles.end());
|
||||
|
||||
// Expect the change list to contain the 4 deleted files
|
||||
EXPECT_EQ(changeList.m_deletedFiles.size(), 4);
|
||||
EXPECT_TRUE(
|
||||
AZStd::find(changeList.m_deletedFiles.begin(), changeList.m_deletedFiles.end(), "B.txt") != changeList.m_deletedFiles.end());
|
||||
EXPECT_TRUE(
|
||||
AZStd::find(changeList.m_deletedFiles.begin(), changeList.m_deletedFiles.end(), "D.txt") != changeList.m_deletedFiles.end());
|
||||
EXPECT_TRUE(
|
||||
AZStd::find(changeList.m_deletedFiles.begin(), changeList.m_deletedFiles.end(), "E.txt") != changeList.m_deletedFiles.end());
|
||||
EXPECT_TRUE(
|
||||
AZStd::find(changeList.m_deletedFiles.begin(), changeList.m_deletedFiles.end(), "C.txt") != changeList.m_deletedFiles.end());
|
||||
}
|
||||
} // namespace UnitTest
|
||||
-822
@@ -1,822 +0,0 @@
|
||||
/*
|
||||
* All or portions of this file Copyright (c) Amazon.com, Inc. or its affiliates or
|
||||
* its licensors.
|
||||
*
|
||||
* For complete copyright and license terms please see the LICENSE at the root of this
|
||||
* distribution (the "License"). All use of this software is governed by the License,
|
||||
* or, if provided, by the license below or the license accompanying this file. Do not
|
||||
* remove or modify any license notices. This file is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
*
|
||||
*/
|
||||
|
||||
#include <TestImpactTestJobRunnerCommon.h>
|
||||
#include <TestImpactTestUtils.h>
|
||||
|
||||
#include <Artifact/TestImpactArtifactException.h>
|
||||
#include <Test/Run/TestImpactInstrumentedTestRunner.h>
|
||||
#include <Test/Run/TestImpactTestRunException.h>
|
||||
|
||||
#include <AzCore/IO/Path/Path.h>
|
||||
#include <AzCore/UnitTest/TestTypes.h>
|
||||
#include <AzCore/std/containers/array.h>
|
||||
#include <AzCore/std/containers/vector.h>
|
||||
#include <AzCore/std/optional.h>
|
||||
#include <AzCore/std/smart_ptr/unique_ptr.h>
|
||||
#include <AzCore/std/string/regex.h>
|
||||
#include <AzCore/std/string/string.h>
|
||||
#include <AzTest/AzTest.h>
|
||||
|
||||
namespace UnitTest
|
||||
{
|
||||
using JobExceptionPolicy = TestImpact::InstrumentedTestRunner::JobExceptionPolicy;
|
||||
using CoverageExceptionPolicy = TestImpact::InstrumentedTestRunner::CoverageExceptionPolicy;
|
||||
|
||||
struct TargetPaths
|
||||
{
|
||||
AZ::IO::Path m_targetBinary;
|
||||
AZ::IO::Path m_testRunArtifact;
|
||||
AZ::IO::Path m_testCoverageArtifact;
|
||||
};
|
||||
|
||||
// Indices for looking up job command arguments for the different coverage levels
|
||||
enum CoverageLevel : uint8_t
|
||||
{
|
||||
LineLevel = 0,
|
||||
SourceLevel
|
||||
};
|
||||
|
||||
// Get the job command for an instrumented test run
|
||||
AZStd::string GetRunCommandForTarget(const TargetPaths& testTarget, CoverageLevel coverageLevel, const char* sourcesFilter)
|
||||
{
|
||||
AZStd::string args = AZStd::string::format(
|
||||
"%s " // 1. Instrumented test runner
|
||||
"--coverage_level %s " // 2. Coverage level
|
||||
"--export_type cobertura:\"%s\" " // 3. Test coverage artifact path
|
||||
"--modules \"%s\" " // 4. Modules path
|
||||
"--excluded_modules \"%s\" " // 5. Exclude modules
|
||||
"--sources \"%s\" -- " // 6. Sources path
|
||||
"\"%s\" " // 7. Test runner binary
|
||||
"\"%s\" " // 8. Test target bin
|
||||
"AzRunUnitTests "
|
||||
"--gtest_output=xml:%s", // 9. Test run result artifact
|
||||
|
||||
LY_TEST_IMPACT_INSTRUMENTATION_BIN, // 1.
|
||||
(coverageLevel == CoverageLevel::LineLevel ? "line" : "source"), // 2.
|
||||
testTarget.m_testCoverageArtifact.c_str(), // 3.
|
||||
LY_TEST_IMPACT_MODULES_DIR, // 4.
|
||||
LY_TEST_IMPACT_AZ_TESTRUNNER_BIN, // 5.
|
||||
sourcesFilter, // 6.
|
||||
LY_TEST_IMPACT_AZ_TESTRUNNER_BIN, // 7.
|
||||
testTarget.m_targetBinary.c_str(), // 8.
|
||||
testTarget.m_testRunArtifact.c_str() // 9.
|
||||
);
|
||||
|
||||
// OpenCppCoverage doesn't support forward slash directory separators so replace all with escaped backslashes
|
||||
return AZStd::regex_replace(args, AZStd::regex("/"), "\\");
|
||||
}
|
||||
|
||||
// Get the job command for an instrumented test run with valid source filters to produce coverage artifact
|
||||
AZStd::string GetRunCommandForTargetWithSources(const TargetPaths& testTarget, CoverageLevel coverageLevel)
|
||||
{
|
||||
return GetRunCommandForTarget(testTarget, coverageLevel, LY_TEST_IMPACT_COVERAGE_SOURCES_DIR);
|
||||
}
|
||||
|
||||
// Get the job command for an instrumented test run without valid source filters to produce empty coverage artifact
|
||||
AZStd::string GetRunCommandForTargetWithoutSources(const TargetPaths& testTarget, CoverageLevel coverageLevel)
|
||||
{
|
||||
return GetRunCommandForTarget(testTarget, coverageLevel, "C:\\No\\Sources\\Here\\At\\All\\Ever\\Ever\\Ever");
|
||||
}
|
||||
|
||||
class InstrumentedTestRunnerFixture
|
||||
: public AllocatorsTestFixture
|
||||
{
|
||||
public:
|
||||
void SetUp() override;
|
||||
void TearDown() override;
|
||||
|
||||
protected:
|
||||
using JobInfo = TestImpact::InstrumentedTestRunner::JobInfo;
|
||||
using JobData = TestImpact::InstrumentedTestRunner::JobData;
|
||||
|
||||
AZStd::vector<JobInfo> m_jobInfos;
|
||||
AZStd::unique_ptr<TestImpact::InstrumentedTestRunner> m_testRunner;
|
||||
AZStd::vector<AZStd::array<AZStd::string, 2>> m_testTargetJobArgs;
|
||||
AZStd::vector<TargetPaths> m_testTargetPaths;
|
||||
AZStd::vector<TestImpact::TestRun> m_expectedTestTargetRuns;
|
||||
AZStd::vector<AZStd::array<TestImpact::TestCoverage, 2>> m_expectedTestTargetCoverages;
|
||||
AZStd::vector<TestImpact::TestRunResult> m_expectedTestTargetResult;
|
||||
size_t m_maxConcurrency = 0;
|
||||
CoverageLevel m_coverageLevel = CoverageLevel::LineLevel;
|
||||
inline static AZ::u32 s_uniqueTestCaseId = 0; // Unique id for each test case to be used as the suffix for the written files
|
||||
};
|
||||
|
||||
void InstrumentedTestRunnerFixture::SetUp()
|
||||
{
|
||||
AllocatorsTestFixture::SetUp();
|
||||
|
||||
// Suffix each artifact file with the unique test case id to ensure that there are no possible race conditions between cleaning
|
||||
// up the artifact files after each test case and those artifact files being written and read again in future test cases
|
||||
const AZStd::string fileExtension = AZStd::string::format(".%u.xml", s_uniqueTestCaseId++);
|
||||
|
||||
const AZStd::string runPath = AZStd::string(LY_TEST_IMPACT_TEST_TARGET_RESULTS_DIR) + "/%s.Run" + fileExtension;
|
||||
const AZStd::string coveragePath = AZStd::string(LY_TEST_IMPACT_TEST_TARGET_COVERAGE_DIR) + "/%s.Coverage" + fileExtension;
|
||||
|
||||
// TestTargetA
|
||||
m_testTargetPaths.emplace_back(TargetPaths{
|
||||
LY_TEST_IMPACT_TEST_TARGET_A_BIN, AZStd::string::format(runPath.c_str(), LY_TEST_IMPACT_TEST_TARGET_A_BASE_NAME),
|
||||
AZStd::string::format(coveragePath.c_str(), LY_TEST_IMPACT_TEST_TARGET_A_BASE_NAME)});
|
||||
|
||||
// TestTargetB
|
||||
m_testTargetPaths.emplace_back(TargetPaths{
|
||||
LY_TEST_IMPACT_TEST_TARGET_B_BIN, AZStd::string::format(runPath.c_str(), LY_TEST_IMPACT_TEST_TARGET_B_BASE_NAME),
|
||||
AZStd::string::format(coveragePath.c_str(), LY_TEST_IMPACT_TEST_TARGET_B_BASE_NAME)});
|
||||
|
||||
// TestTargetC
|
||||
m_testTargetPaths.emplace_back(TargetPaths{
|
||||
LY_TEST_IMPACT_TEST_TARGET_C_BIN, AZStd::string::format(runPath.c_str(), LY_TEST_IMPACT_TEST_TARGET_C_BASE_NAME),
|
||||
AZStd::string::format(coveragePath.c_str(), LY_TEST_IMPACT_TEST_TARGET_C_BASE_NAME)});
|
||||
|
||||
// TestTargetD
|
||||
m_testTargetPaths.emplace_back(TargetPaths{
|
||||
LY_TEST_IMPACT_TEST_TARGET_D_BIN, AZStd::string::format(runPath.c_str(), LY_TEST_IMPACT_TEST_TARGET_D_BASE_NAME),
|
||||
AZStd::string::format(coveragePath.c_str(), LY_TEST_IMPACT_TEST_TARGET_D_BASE_NAME)});
|
||||
|
||||
m_expectedTestTargetRuns.emplace_back(GetTestTargetATestRunSuites(), AZStd::chrono::milliseconds{500}); // TestTargetA
|
||||
m_expectedTestTargetRuns.emplace_back(GetTestTargetBTestRunSuites(), AZStd::chrono::milliseconds{500}); // TestTargetB
|
||||
m_expectedTestTargetRuns.emplace_back(GetTestTargetCTestRunSuites(), AZStd::chrono::milliseconds{500}); // TestTargetC
|
||||
m_expectedTestTargetRuns.emplace_back(GetTestTargetDTestRunSuites(), AZStd::chrono::milliseconds{500}); // TestTargetD
|
||||
|
||||
// TestTargetA
|
||||
m_expectedTestTargetCoverages.emplace_back(AZStd::array<TestImpact::TestCoverage, 2>{
|
||||
TestImpact::TestCoverage(GetTestTargetALineModuleCoverages()),
|
||||
TestImpact::TestCoverage(GetTestTargetASourceModuleCoverages())});
|
||||
|
||||
// TestTargetB
|
||||
m_expectedTestTargetCoverages.emplace_back(AZStd::array<TestImpact::TestCoverage, 2>{
|
||||
TestImpact::TestCoverage(GetTestTargetBLineModuleCoverages()),
|
||||
TestImpact::TestCoverage(GetTestTargetBSourceModuleCoverages())});
|
||||
|
||||
// TestTargetC
|
||||
m_expectedTestTargetCoverages.emplace_back(AZStd::array<TestImpact::TestCoverage, 2>{
|
||||
TestImpact::TestCoverage(GetTestTargetCLineModuleCoverages()),
|
||||
TestImpact::TestCoverage(GetTestTargetCSourceModuleCoverages())});
|
||||
|
||||
// TestTargetD
|
||||
m_expectedTestTargetCoverages.emplace_back(AZStd::array<TestImpact::TestCoverage, 2>{
|
||||
TestImpact::TestCoverage(GetTestTargetDLineModuleCoverages()),
|
||||
TestImpact::TestCoverage(GetTestTargetDSourceModuleCoverages())});
|
||||
|
||||
m_expectedTestTargetResult.emplace_back(TestImpact::TestRunResult::Failed); // TestTargetA
|
||||
m_expectedTestTargetResult.emplace_back(TestImpact::TestRunResult::Passed); // TestTargetB
|
||||
m_expectedTestTargetResult.emplace_back(TestImpact::TestRunResult::Passed); // TestTargetC
|
||||
m_expectedTestTargetResult.emplace_back(TestImpact::TestRunResult::Passed); // TestTargetD
|
||||
|
||||
// Generate the job command arguments for both line level and source level coverage permutations
|
||||
for (const auto& testTarget : m_testTargetPaths)
|
||||
{
|
||||
m_testTargetJobArgs.emplace_back(AZStd::array<AZStd::string, 2>{
|
||||
GetRunCommandForTargetWithSources(testTarget, CoverageLevel::LineLevel),
|
||||
GetRunCommandForTargetWithSources(testTarget, CoverageLevel::SourceLevel)});
|
||||
}
|
||||
}
|
||||
|
||||
void InstrumentedTestRunnerFixture::TearDown()
|
||||
{
|
||||
DeleteFiles(LY_TEST_IMPACT_TEST_TARGET_COVERAGE_DIR, "*.xml");
|
||||
DeleteFiles(LY_TEST_IMPACT_TEST_TARGET_RESULTS_DIR, "*.xml");
|
||||
|
||||
AllocatorsTestFixture::TearDown();
|
||||
}
|
||||
|
||||
using ConcurrencyAndCoveragePermutation = AZStd::tuple
|
||||
<
|
||||
size_t, // Max number of concurrent processes
|
||||
CoverageLevel // Coverage level
|
||||
>;
|
||||
|
||||
// Fixture parameterized for different max number of concurrent jobs and coverage levels
|
||||
class InstrumentedTestRunnerFixtureWithConcurrencyAndCoverageParams
|
||||
: public InstrumentedTestRunnerFixture
|
||||
, public ::testing::WithParamInterface<ConcurrencyAndCoveragePermutation>
|
||||
{
|
||||
public:
|
||||
void SetUp() override
|
||||
{
|
||||
InstrumentedTestRunnerFixture::SetUp();
|
||||
auto [maxConcurrency, coverageLevel] = GetParam();
|
||||
m_maxConcurrency = maxConcurrency;
|
||||
m_coverageLevel = coverageLevel;
|
||||
}
|
||||
};
|
||||
|
||||
using ConcurrencyAndJobExceptionPermutation = AZStd::tuple
|
||||
<
|
||||
size_t, // Max number of concurrent processes
|
||||
CoverageLevel, // Coverage level
|
||||
JobExceptionPolicy // Test job exception policy
|
||||
>;
|
||||
|
||||
// Fixture parameterized for different max number of concurrent jobs, coverage levels and different job exception policies
|
||||
class InstrumentedTestRunnerFixtureWithConcurrencyAndCoverageAndJobExceptionParams
|
||||
: public InstrumentedTestRunnerFixture
|
||||
, public ::testing::WithParamInterface<ConcurrencyAndJobExceptionPermutation>
|
||||
{
|
||||
public:
|
||||
void SetUp() override
|
||||
{
|
||||
InstrumentedTestRunnerFixture::SetUp();
|
||||
const auto& [maxConcurrency, coverageLevel, jobExceptionPolicy] = GetParam();
|
||||
m_maxConcurrency = maxConcurrency;
|
||||
m_coverageLevel = coverageLevel;
|
||||
m_jobExceptionPolicy = jobExceptionPolicy;
|
||||
}
|
||||
|
||||
protected:
|
||||
JobExceptionPolicy m_jobExceptionPolicy = JobExceptionPolicy::Never;
|
||||
};
|
||||
|
||||
class InstrumentedTestRunnerFixtureWithConcurrencyAndCoverageAndFailedToLaunchExceptionParams
|
||||
: public InstrumentedTestRunnerFixtureWithConcurrencyAndCoverageAndJobExceptionParams
|
||||
{
|
||||
};
|
||||
|
||||
class InstrumentedTestRunnerFixtureWithConcurrencyAndCoverageAndExecutedWithFailureExceptionParams
|
||||
: public InstrumentedTestRunnerFixtureWithConcurrencyAndCoverageAndJobExceptionParams
|
||||
{
|
||||
};
|
||||
|
||||
using ConcurrencyAndCoverageExceptionPermutation = AZStd::tuple
|
||||
<
|
||||
size_t, // Max number of concurrent processes
|
||||
CoverageExceptionPolicy // Test coverage exception policy
|
||||
>;
|
||||
|
||||
// Fixture parameterized for different max number of concurrent jobs and different coverage exception policies
|
||||
class InstrumentedTestRunnerFixtureWithConcurrencyAndCoverageAndCoverageExceptionParams
|
||||
: public InstrumentedTestRunnerFixture
|
||||
, public ::testing::WithParamInterface<ConcurrencyAndCoverageExceptionPermutation>
|
||||
{
|
||||
public:
|
||||
void SetUp() override
|
||||
{
|
||||
InstrumentedTestRunnerFixture::SetUp();
|
||||
const auto& [maxConcurrency, coverageExceptionPolicy] = GetParam();
|
||||
m_maxConcurrency = maxConcurrency;
|
||||
m_coverageExceptionPolicy = coverageExceptionPolicy;
|
||||
}
|
||||
|
||||
protected:
|
||||
CoverageExceptionPolicy m_coverageExceptionPolicy = CoverageExceptionPolicy::Never;
|
||||
};
|
||||
|
||||
namespace
|
||||
{
|
||||
AZStd::array<size_t, 4> MaxConcurrentRuns = {1, 2, 3, 4};
|
||||
|
||||
AZStd::array<CoverageLevel, 2> CoverageLevels = {CoverageLevel::LineLevel, CoverageLevel::SourceLevel};
|
||||
|
||||
AZStd::array<JobExceptionPolicy, 2> FailedToLaunchExceptionPolicies = {
|
||||
JobExceptionPolicy::Never, JobExceptionPolicy::OnFailedToExecute};
|
||||
|
||||
AZStd::array<JobExceptionPolicy, 2> ExecutedWithFailureExceptionPolicies = {
|
||||
JobExceptionPolicy::Never, JobExceptionPolicy::OnExecutedWithFailure};
|
||||
} // namespace
|
||||
|
||||
// Validates that the specified test coverage matches the expected output
|
||||
void ValidateTestTargetCoverage(const TestImpact::TestCoverage& actualResult, const TestImpact::TestCoverage& expectedResult)
|
||||
{
|
||||
EXPECT_TRUE(actualResult == expectedResult);
|
||||
}
|
||||
|
||||
// Validates that the specified test coverage is empty
|
||||
void ValidateEmptyTestTargetCoverage(const TestImpact::TestCoverage& actualResult)
|
||||
{
|
||||
EXPECT_TRUE(actualResult.GetSourcesCovered().empty());
|
||||
EXPECT_TRUE(actualResult.GetModuleCoverages().empty());
|
||||
EXPECT_EQ(actualResult.GetNumSourcesCovered(), 0);
|
||||
EXPECT_EQ(actualResult.GetNumModulesCovered(), 0);
|
||||
}
|
||||
|
||||
TEST_P(
|
||||
InstrumentedTestRunnerFixtureWithConcurrencyAndCoverageAndCoverageExceptionParams,
|
||||
EmptyTestCoverages_ExpectEmptyTestCoveragesOrTestRunException)
|
||||
{
|
||||
// Given a test runner with no client callback or run timeout or runner timeout
|
||||
m_testRunner =
|
||||
AZStd::make_unique<TestImpact::InstrumentedTestRunner>(AZStd::nullopt, m_maxConcurrency, AZStd::nullopt, AZStd::nullopt);
|
||||
|
||||
// Given a mixture of instrumented test run jobs with and without coverage sources
|
||||
for (size_t jobId = 0; jobId < m_testTargetJobArgs.size(); jobId++)
|
||||
{
|
||||
const AZStd::string args = (jobId % 2) ? GetRunCommandForTargetWithoutSources(m_testTargetPaths[jobId], m_coverageLevel)
|
||||
: m_testTargetJobArgs[jobId][m_coverageLevel];
|
||||
|
||||
JobData jobData(m_testTargetPaths[jobId].m_testRunArtifact, m_testTargetPaths[jobId].m_testCoverageArtifact);
|
||||
m_jobInfos.emplace_back(JobInfo({jobId}, args, AZStd::move(jobData)));
|
||||
}
|
||||
|
||||
try
|
||||
{
|
||||
// When the instrumented test run jobs are executed with different exception policies
|
||||
const auto runnerJobs = m_testRunner->RunInstrumentedTests(m_jobInfos, m_coverageExceptionPolicy, JobExceptionPolicy::Never);
|
||||
|
||||
// Expect this statement to be reachable only if no exception policy for empty coverages
|
||||
EXPECT_FALSE(::IsFlagSet(m_coverageExceptionPolicy, CoverageExceptionPolicy::OnEmptyCoverage));
|
||||
|
||||
for (const auto& job : runnerJobs)
|
||||
{
|
||||
const JobInfo::IdType jobId = job.GetJobInfo().GetId().m_value;
|
||||
ValidateTestRunCompleted(job, m_expectedTestTargetResult[jobId]);
|
||||
ValidateTestTargetRun(job.GetPayload().value().first, m_expectedTestTargetRuns[jobId]);
|
||||
|
||||
if (jobId % 2)
|
||||
{
|
||||
// Expect jobs have a empty test coverages
|
||||
ValidateEmptyTestTargetCoverage(job.GetPayload().value().second);
|
||||
}
|
||||
else
|
||||
{
|
||||
// Expect the jobs to successfully result in a test run and coverage that matches the expected test run data
|
||||
ValidateTestTargetCoverage(job.GetPayload().value().second, m_expectedTestTargetCoverages[jobId][m_coverageLevel]);
|
||||
}
|
||||
}
|
||||
}
|
||||
catch ([[maybe_unused]] const TestImpact::TestRunException& e)
|
||||
{
|
||||
// Expect this statement to be reachable only if there is an exception policy for empty coverages
|
||||
EXPECT_TRUE(::IsFlagSet(m_coverageExceptionPolicy, CoverageExceptionPolicy::OnEmptyCoverage));
|
||||
}
|
||||
catch (...)
|
||||
{
|
||||
// Do not expect any other exceptions
|
||||
FAIL();
|
||||
}
|
||||
}
|
||||
|
||||
TEST_P(
|
||||
InstrumentedTestRunnerFixtureWithConcurrencyAndCoverageAndFailedToLaunchExceptionParams,
|
||||
InvalidCommandArgument_ExpectJobResulFailedToExecuteeOrTestJobException)
|
||||
{
|
||||
// Given a test runner with no client callback or run timeout or runner timeout
|
||||
m_testRunner =
|
||||
AZStd::make_unique<TestImpact::InstrumentedTestRunner>(AZStd::nullopt, m_maxConcurrency, AZStd::nullopt, AZStd::nullopt);
|
||||
|
||||
// Given a mixture of instrumented test run jobs with valid and invalid command arguments
|
||||
for (size_t jobId = 0; jobId < m_testTargetJobArgs.size(); jobId++)
|
||||
{
|
||||
const AZStd::string args = (jobId % 2) ? InvalidProcessPath : m_testTargetJobArgs[jobId][m_coverageLevel];
|
||||
JobData jobData(m_testTargetPaths[jobId].m_testRunArtifact, m_testTargetPaths[jobId].m_testCoverageArtifact);
|
||||
m_jobInfos.emplace_back(JobInfo({jobId}, args, AZStd::move(jobData)));
|
||||
}
|
||||
|
||||
try
|
||||
{
|
||||
// When the instrumented test run jobs are executed with different exception policies
|
||||
const auto runnerJobs = m_testRunner->RunInstrumentedTests(m_jobInfos, CoverageExceptionPolicy::Never, m_jobExceptionPolicy);
|
||||
|
||||
// Expect this statement to be reachable only if no exception policy for launch failures
|
||||
EXPECT_FALSE(::IsFlagSet(m_jobExceptionPolicy, JobExceptionPolicy::OnFailedToExecute));
|
||||
|
||||
for (const auto& job : runnerJobs)
|
||||
{
|
||||
const JobInfo::IdType jobId = job.GetJobInfo().GetId().m_value;
|
||||
if (jobId % 2)
|
||||
{
|
||||
// Expect invalid jobs have a job result of FailedToExecute
|
||||
ValidateJobFailedToExecute(job);
|
||||
}
|
||||
else
|
||||
{
|
||||
// Expect the valid jobs to successfully result in a test run that matches the expected test run data
|
||||
ValidateTestRunCompleted(job, m_expectedTestTargetResult[jobId]);
|
||||
ValidateTestTargetRun(job.GetPayload().value().first, m_expectedTestTargetRuns[jobId]);
|
||||
ValidateTestTargetCoverage(job.GetPayload().value().second, m_expectedTestTargetCoverages[jobId][m_coverageLevel]);
|
||||
}
|
||||
}
|
||||
}
|
||||
catch ([[maybe_unused]] const TestImpact::TestJobException& e)
|
||||
{
|
||||
// Expect this statement to be reachable only if there is an exception policy for launch failures
|
||||
EXPECT_TRUE(::IsFlagSet(m_jobExceptionPolicy, JobExceptionPolicy::OnFailedToExecute));
|
||||
}
|
||||
catch (...)
|
||||
{
|
||||
// Do not expect any other exceptions
|
||||
FAIL();
|
||||
}
|
||||
}
|
||||
|
||||
TEST_P(
|
||||
InstrumentedTestRunnerFixtureWithConcurrencyAndCoverageAndExecutedWithFailureExceptionParams,
|
||||
ErroneousReturnCode_ExpectJobResultExecutedWithFailureOrTestJobException)
|
||||
{
|
||||
// Given a test runner with no client callback or run timeout or runner timeout
|
||||
m_testRunner =
|
||||
AZStd::make_unique<TestImpact::InstrumentedTestRunner>(AZStd::nullopt, m_maxConcurrency, AZStd::nullopt, AZStd::nullopt);
|
||||
|
||||
// Given a mixture of instrumented test run jobs that execute and return either successfully or with failure
|
||||
for (size_t jobId = 0; jobId < m_testTargetJobArgs.size(); jobId++)
|
||||
{
|
||||
JobData jobData(m_testTargetPaths[jobId].m_testRunArtifact, m_testTargetPaths[jobId].m_testCoverageArtifact);
|
||||
m_jobInfos.emplace_back(JobInfo({jobId}, m_testTargetJobArgs[jobId][m_coverageLevel], AZStd::move(jobData)));
|
||||
}
|
||||
|
||||
try
|
||||
{
|
||||
// When the instrumented test run jobs are executed with different exception policies
|
||||
const auto runnerJobs = m_testRunner->RunInstrumentedTests(m_jobInfos, CoverageExceptionPolicy::Never, m_jobExceptionPolicy);
|
||||
|
||||
// Expect this statement to be reachable only if no exception policy for jobs that return with error
|
||||
EXPECT_FALSE(::IsFlagSet(m_jobExceptionPolicy, JobExceptionPolicy::OnExecutedWithFailure));
|
||||
|
||||
for (const auto& job : runnerJobs)
|
||||
{
|
||||
const JobInfo::IdType jobId = job.GetJobInfo().GetId().m_value;
|
||||
|
||||
// Expect the valid jobs to successfully result in a test run that matches the expected test run data
|
||||
ValidateTestRunCompleted(job, m_expectedTestTargetResult[jobId]);
|
||||
ValidateTestTargetRun(job.GetPayload().value().first, m_expectedTestTargetRuns[jobId]);
|
||||
ValidateTestTargetCoverage(job.GetPayload().value().second, m_expectedTestTargetCoverages[jobId][m_coverageLevel]);
|
||||
}
|
||||
}
|
||||
catch ([[maybe_unused]] const TestImpact::TestJobException& e)
|
||||
{
|
||||
// Expect this statement to be reachable only if there is an exception policy for jobs that return with error
|
||||
EXPECT_TRUE(::IsFlagSet(m_jobExceptionPolicy, JobExceptionPolicy::OnExecutedWithFailure));
|
||||
}
|
||||
catch ([[maybe_unused]] const TestImpact::Exception& e)
|
||||
{
|
||||
FAIL();
|
||||
}
|
||||
catch (...)
|
||||
{
|
||||
// Do not expect any other exceptions
|
||||
FAIL();
|
||||
}
|
||||
}
|
||||
|
||||
TEST_F(InstrumentedTestRunnerFixture, EmptyRunRawData_ExpectTestRunnerException)
|
||||
{
|
||||
// Given a test runner with no client callback, concurrency, run timeout or runner timeout
|
||||
m_testRunner =
|
||||
AZStd::make_unique<TestImpact::InstrumentedTestRunner>(AZStd::nullopt, OneConcurrentProcess, AZStd::nullopt, AZStd::nullopt);
|
||||
|
||||
// Given an test runner job that will return successfully but with an empty artifact string
|
||||
JobData jobData("", m_testTargetPaths[TestTargetA].m_testCoverageArtifact);
|
||||
m_jobInfos.emplace_back(JobInfo({TestTargetA}, m_testTargetJobArgs[TestTargetA][LineLevel], AZStd::move(jobData)));
|
||||
|
||||
try
|
||||
{
|
||||
// When the test runner job is executed
|
||||
const auto runnerJobs =
|
||||
m_testRunner->RunInstrumentedTests(m_jobInfos, CoverageExceptionPolicy::Never, JobExceptionPolicy::Never);
|
||||
|
||||
// Do not expect this statement to be reachable
|
||||
FAIL();
|
||||
}
|
||||
catch ([[maybe_unused]] const TestImpact::TestRunException& e)
|
||||
{
|
||||
// Expect an runner exception
|
||||
SUCCEED();
|
||||
}
|
||||
catch (...)
|
||||
{
|
||||
// Do not expect any other exceptions
|
||||
FAIL();
|
||||
}
|
||||
}
|
||||
|
||||
TEST_F(InstrumentedTestRunnerFixture, EmptyCoverageRawData_ExpectTestRunnerException)
|
||||
{
|
||||
// Given a test runner with no client callback, concurrency, run timeout or runner timeout
|
||||
m_testRunner =
|
||||
AZStd::make_unique<TestImpact::InstrumentedTestRunner>(AZStd::nullopt, OneConcurrentProcess, AZStd::nullopt, AZStd::nullopt);
|
||||
|
||||
// Given an test runner job that will return successfully but with an empty artifact string
|
||||
JobData jobData(m_testTargetPaths[TestTargetA].m_testRunArtifact, "");
|
||||
m_jobInfos.emplace_back(JobInfo({TestTargetA}, m_testTargetJobArgs[TestTargetA][LineLevel], AZStd::move(jobData)));
|
||||
|
||||
try
|
||||
{
|
||||
// When the test runner job is executed
|
||||
const auto runnerJobs =
|
||||
m_testRunner->RunInstrumentedTests(m_jobInfos, CoverageExceptionPolicy::Never, JobExceptionPolicy::Never);
|
||||
|
||||
// Do not expect this statement to be reachable
|
||||
FAIL();
|
||||
}
|
||||
catch ([[maybe_unused]] const TestImpact::TestRunException& e)
|
||||
{
|
||||
// Expect an runner exception
|
||||
SUCCEED();
|
||||
}
|
||||
catch (...)
|
||||
{
|
||||
// Do not expect any other exceptions
|
||||
FAIL();
|
||||
}
|
||||
}
|
||||
|
||||
TEST_F(InstrumentedTestRunnerFixture, InvalidRunArtifact_ExpectArtifactException)
|
||||
{
|
||||
// Given a run artifact with invalid contents
|
||||
WriteTextToFile("There is nothing valid here", m_testTargetPaths[TestTargetA].m_testRunArtifact);
|
||||
|
||||
// Given a job command that will write the run artifact to a different location that what we will read from
|
||||
TargetPaths invalidRunArtifact = m_testTargetPaths[TestTargetA];
|
||||
invalidRunArtifact.m_testRunArtifact /= ".xml";
|
||||
const AZStd::string args = GetRunCommandForTargetWithSources(invalidRunArtifact, LineLevel);
|
||||
|
||||
// Given a test runner with no client callback, concurrency, run timeout or runner timeout
|
||||
m_testRunner =
|
||||
AZStd::make_unique<TestImpact::InstrumentedTestRunner>(AZStd::nullopt, OneConcurrentProcess, AZStd::nullopt, AZStd::nullopt);
|
||||
|
||||
// Given an test runner job that will return successfully but not produce a run artifact
|
||||
JobData jobData(m_testTargetPaths[TestTargetA].m_testRunArtifact, m_testTargetPaths[TestTargetA].m_testCoverageArtifact);
|
||||
m_jobInfos.emplace_back(JobInfo({TestTargetA}, args, AZStd::move(jobData)));
|
||||
|
||||
try
|
||||
{
|
||||
// When the test runner job is executed
|
||||
const auto runnerJobs =
|
||||
m_testRunner->RunInstrumentedTests(m_jobInfos, CoverageExceptionPolicy::Never, JobExceptionPolicy::Never);
|
||||
|
||||
// Do not expect this statement to be reachable
|
||||
FAIL();
|
||||
}
|
||||
catch ([[maybe_unused]] const TestImpact::ArtifactException& e)
|
||||
{
|
||||
// Expect an runner exception
|
||||
SUCCEED();
|
||||
}
|
||||
catch (...)
|
||||
{
|
||||
// Do not expect any other exceptions
|
||||
FAIL();
|
||||
}
|
||||
}
|
||||
|
||||
TEST_F(InstrumentedTestRunnerFixture, InvalidCoverageArtifact_ExpectArtifactException)
|
||||
{
|
||||
// Given a coverage artifact with invalid contents
|
||||
WriteTextToFile("There is nothing valid here", m_testTargetPaths[TestTargetA].m_testCoverageArtifact);
|
||||
|
||||
// Given a job command that will write the coverage artifact to a different location that what we will read from
|
||||
TargetPaths invalidCoverageArtifact = m_testTargetPaths[TestTargetA];
|
||||
invalidCoverageArtifact.m_testCoverageArtifact /= ".xml";
|
||||
const AZStd::string args = GetRunCommandForTargetWithSources(invalidCoverageArtifact, LineLevel);
|
||||
|
||||
// Given a test runner with no client callback, concurrency, run timeout or runner timeout
|
||||
m_testRunner =
|
||||
AZStd::make_unique<TestImpact::InstrumentedTestRunner>(AZStd::nullopt, OneConcurrentProcess, AZStd::nullopt, AZStd::nullopt);
|
||||
|
||||
// Given an test runner job that will return successfully but not produce a run artifact
|
||||
JobData jobData(m_testTargetPaths[TestTargetA].m_testRunArtifact, m_testTargetPaths[TestTargetA].m_testCoverageArtifact);
|
||||
m_jobInfos.emplace_back(JobInfo({TestTargetA}, args, AZStd::move(jobData)));
|
||||
|
||||
try
|
||||
{
|
||||
// When the test runner job is executed
|
||||
const auto runnerJobs =
|
||||
m_testRunner->RunInstrumentedTests(m_jobInfos, CoverageExceptionPolicy::Never, JobExceptionPolicy::Never);
|
||||
|
||||
// Do not expect this statement to be reachable
|
||||
FAIL();
|
||||
}
|
||||
catch ([[maybe_unused]] const TestImpact::ArtifactException& e)
|
||||
{
|
||||
// Expect an runner exception
|
||||
SUCCEED();
|
||||
}
|
||||
catch (...)
|
||||
{
|
||||
// Do not expect any other exceptions
|
||||
FAIL();
|
||||
}
|
||||
}
|
||||
|
||||
TEST_P(InstrumentedTestRunnerFixtureWithConcurrencyAndCoverageParams, RunTestTargets_RunsAndCoverageMatchTestSuitesInTarget)
|
||||
{
|
||||
// Given a test runner with no client callback, runner timeout or runner timeout
|
||||
m_testRunner =
|
||||
AZStd::make_unique<TestImpact::InstrumentedTestRunner>(AZStd::nullopt, m_maxConcurrency, AZStd::nullopt, AZStd::nullopt);
|
||||
|
||||
// Given an test runner job for each test target with no runner caching
|
||||
for (size_t jobId = 0; jobId < m_testTargetJobArgs.size(); jobId++)
|
||||
{
|
||||
JobData jobData(m_testTargetPaths[jobId].m_testRunArtifact, m_testTargetPaths[jobId].m_testCoverageArtifact);
|
||||
m_jobInfos.emplace_back(JobInfo({jobId}, m_testTargetJobArgs[jobId][m_coverageLevel], AZStd::move(jobData)));
|
||||
}
|
||||
|
||||
// When the test runner jobs are executed
|
||||
const auto runnerJobs = m_testRunner->RunInstrumentedTests(m_jobInfos, CoverageExceptionPolicy::Never, JobExceptionPolicy::Never);
|
||||
|
||||
// Expect each job to successfully result in a test runner that matches the expected test runner data for that test target
|
||||
for (const auto& job : runnerJobs)
|
||||
{
|
||||
const JobInfo::IdType jobId = job.GetJobInfo().GetId().m_value;
|
||||
ValidateTestRunCompleted(job, m_expectedTestTargetResult[jobId]);
|
||||
ValidateTestTargetRun(job.GetPayload().value().first, m_expectedTestTargetRuns[jobId]);
|
||||
ValidateTestTargetCoverage(job.GetPayload().value().second, m_expectedTestTargetCoverages[jobId][m_coverageLevel]);
|
||||
}
|
||||
}
|
||||
|
||||
TEST_P(
|
||||
InstrumentedTestRunnerFixtureWithConcurrencyAndCoverageParams,
|
||||
RunTestTargetsWithArbitraryJobIds_RunsAndCoverageMatchTestSuitesInTarget)
|
||||
{
|
||||
// Given a set of arbitrary job ids to be used for the test target jobs
|
||||
enum
|
||||
{
|
||||
ArbitraryA = 36,
|
||||
ArbitraryB = 890,
|
||||
ArbitraryC = 19,
|
||||
ArbitraryD = 1
|
||||
};
|
||||
|
||||
const AZStd::unordered_map<JobInfo::IdType, JobInfo::IdType> sequentialToArbitrary =
|
||||
{
|
||||
{TestTargetA, ArbitraryA},
|
||||
{TestTargetB, ArbitraryB},
|
||||
{TestTargetC, ArbitraryC},
|
||||
{TestTargetD, ArbitraryD},
|
||||
};
|
||||
|
||||
const AZStd::unordered_map<JobInfo::IdType, JobInfo::IdType> arbitraryToSequential =
|
||||
{
|
||||
{ArbitraryA, TestTargetA},
|
||||
{ArbitraryB, TestTargetB},
|
||||
{ArbitraryC, TestTargetC},
|
||||
{ArbitraryD, TestTargetD},
|
||||
};
|
||||
|
||||
// Given a test runner with no client callback, run timeout or runner timeout
|
||||
m_testRunner =
|
||||
AZStd::make_unique<TestImpact::InstrumentedTestRunner>(AZStd::nullopt, m_maxConcurrency, AZStd::nullopt, AZStd::nullopt);
|
||||
|
||||
// Given an test run job for each test target
|
||||
for (size_t jobId = 0; jobId < m_testTargetJobArgs.size(); jobId++)
|
||||
{
|
||||
JobData jobData(m_testTargetPaths[jobId].m_testRunArtifact, m_testTargetPaths[jobId].m_testCoverageArtifact);
|
||||
m_jobInfos.emplace_back(
|
||||
JobInfo({sequentialToArbitrary.at(jobId)}, m_testTargetJobArgs[jobId][m_coverageLevel], AZStd::move(jobData)));
|
||||
}
|
||||
|
||||
// When the instrumented test run jobs are executed
|
||||
const auto runnerJobs = m_testRunner->RunInstrumentedTests(m_jobInfos, CoverageExceptionPolicy::Never, JobExceptionPolicy::Never);
|
||||
|
||||
// Expect each job to successfully result in a test run that matches the expected test run data for that test target
|
||||
for (const auto& job : runnerJobs)
|
||||
{
|
||||
const JobInfo::IdType jobId = arbitraryToSequential.at(job.GetJobInfo().GetId().m_value);
|
||||
ValidateTestRunCompleted(job, m_expectedTestTargetResult[jobId]);
|
||||
ValidateTestTargetRun(job.GetPayload().value().first, m_expectedTestTargetRuns[jobId]);
|
||||
ValidateTestTargetCoverage(job.GetPayload().value().second, m_expectedTestTargetCoverages[jobId][m_coverageLevel]);
|
||||
}
|
||||
}
|
||||
|
||||
TEST_P(InstrumentedTestRunnerFixtureWithConcurrencyAndCoverageParams, RunTestTargetsWithCallback_RunsAndCoverageMatchTestSuitesInTarget)
|
||||
{
|
||||
// Given a client callback function that tracks the number of successful runs
|
||||
size_t numSuccesses = 0;
|
||||
const auto jobCallback =
|
||||
[&numSuccesses]([[maybe_unused]] const TestImpact::InstrumentedTestRunner::JobInfo& jobInfo, const TestImpact::JobMeta& meta)
|
||||
{
|
||||
if (meta.m_result == TestImpact::JobResult::ExecutedWithSuccess)
|
||||
{
|
||||
numSuccesses++;
|
||||
}
|
||||
};
|
||||
|
||||
// Given a test runner with no run timeout or runner timeout
|
||||
m_testRunner =
|
||||
AZStd::make_unique<TestImpact::InstrumentedTestRunner>(jobCallback, m_maxConcurrency, AZStd::nullopt, AZStd::nullopt);
|
||||
|
||||
// Given an test run job for each test target
|
||||
for (size_t jobId = 0; jobId < m_testTargetJobArgs.size(); jobId++)
|
||||
{
|
||||
JobData jobData(m_testTargetPaths[jobId].m_testRunArtifact, m_testTargetPaths[jobId].m_testCoverageArtifact);
|
||||
m_jobInfos.emplace_back(JobInfo({jobId}, m_testTargetJobArgs[jobId][m_coverageLevel], AZStd::move(jobData)));
|
||||
}
|
||||
|
||||
// When the instrumented test run jobs are executed
|
||||
const auto runnerJobs = m_testRunner->RunInstrumentedTests(m_jobInfos, CoverageExceptionPolicy::Never, JobExceptionPolicy::Never);
|
||||
|
||||
// Expect the number of successful runs tracked in the callback to match the number of test targets run with no failures
|
||||
EXPECT_EQ(
|
||||
numSuccesses,
|
||||
AZStd::count_if(m_expectedTestTargetResult.begin(), m_expectedTestTargetResult.end(), [](TestImpact::TestRunResult result) {
|
||||
return result == TestImpact::TestRunResult::Passed;
|
||||
}));
|
||||
|
||||
// Expect each job to successfully result in a test run that matches the expected test run data for that test target
|
||||
for (const auto& job : runnerJobs)
|
||||
{
|
||||
const JobInfo::IdType jobId = job.GetJobInfo().GetId().m_value;
|
||||
ValidateTestRunCompleted(job, m_expectedTestTargetResult[jobId]);
|
||||
ValidateTestTargetRun(job.GetPayload().value().first, m_expectedTestTargetRuns[jobId]);
|
||||
ValidateTestTargetCoverage(job.GetPayload().value().second, m_expectedTestTargetCoverages[jobId][m_coverageLevel]);
|
||||
}
|
||||
}
|
||||
|
||||
TEST_P(InstrumentedTestRunnerFixtureWithConcurrencyAndCoverageParams, JobRunnerTimeout_InFlightJobsTimeoutAndQueuedJobsUnlaunched)
|
||||
{
|
||||
// Given a test runner with no client callback or runner timeout and 2 second run timeout
|
||||
m_testRunner = AZStd::make_unique<TestImpact::InstrumentedTestRunner>(
|
||||
AZStd::nullopt, m_maxConcurrency, AZStd::chrono::seconds(2), AZStd::nullopt);
|
||||
|
||||
// Given an test run job for each test target where half will sleep indefinitely
|
||||
for (size_t jobId = 0; jobId < m_testTargetJobArgs.size(); jobId++)
|
||||
{
|
||||
JobData jobData(m_testTargetPaths[jobId].m_testRunArtifact, m_testTargetPaths[jobId].m_testCoverageArtifact);
|
||||
const AZStd::string args = (jobId % 2)
|
||||
? AZStd::string::format("%s %s", ValidProcessPath, ConstructTestProcessArgs(jobId, LongSleep).c_str())
|
||||
: m_testTargetJobArgs[jobId][m_coverageLevel];
|
||||
m_jobInfos.emplace_back(JobInfo({jobId}, args, AZStd::move(jobData)));
|
||||
}
|
||||
|
||||
// When the instrumented test run jobs are executed
|
||||
const auto runnerJobs = m_testRunner->RunInstrumentedTests(m_jobInfos, CoverageExceptionPolicy::Never, JobExceptionPolicy::Never);
|
||||
|
||||
// Expect half the jobs to successfully result in a test run that matches the expected test run data for that test target
|
||||
// with the other half having timed out
|
||||
for (const auto& job : runnerJobs)
|
||||
{
|
||||
const JobInfo::IdType jobId = job.GetJobInfo().GetId().m_value;
|
||||
if (jobId % 2)
|
||||
{
|
||||
ValidateJobTimeout(job);
|
||||
}
|
||||
else
|
||||
{
|
||||
ValidateTestRunCompleted(job, m_expectedTestTargetResult[jobId]);
|
||||
ValidateTestTargetRun(job.GetPayload().value().first, m_expectedTestTargetRuns[jobId]);
|
||||
ValidateTestTargetCoverage(job.GetPayload().value().second, m_expectedTestTargetCoverages[jobId][m_coverageLevel]);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
TEST_F(InstrumentedTestRunnerFixture, JobTimeout_InFlightJobTimeoutAndQueuedJobsUnlaunched)
|
||||
{
|
||||
// Given a test runner with no client callback or run timeout and a 5 second runner timeout
|
||||
m_testRunner = AZStd::make_unique<TestImpact::InstrumentedTestRunner>(
|
||||
AZStd::nullopt, FourConcurrentProcesses, AZStd::nullopt, AZStd::chrono::seconds(5));
|
||||
|
||||
// Given an test run job for each test target where half will sleep indefinitely
|
||||
for (size_t jobId = 0; jobId < m_testTargetJobArgs.size(); jobId++)
|
||||
{
|
||||
JobData jobData(m_testTargetPaths[jobId].m_testRunArtifact, m_testTargetPaths[jobId].m_testCoverageArtifact);
|
||||
const AZStd::string args = (jobId % 2)
|
||||
? AZStd::string::format("%s %s", ValidProcessPath, ConstructTestProcessArgs(jobId, LongSleep).c_str())
|
||||
: m_testTargetJobArgs[jobId][m_coverageLevel];
|
||||
m_jobInfos.emplace_back(JobInfo({jobId}, args, AZStd::move(jobData)));
|
||||
}
|
||||
|
||||
// When the instrumented test run jobs are executed
|
||||
const auto runnerJobs = m_testRunner->RunInstrumentedTests(m_jobInfos, CoverageExceptionPolicy::Never, JobExceptionPolicy::Never);
|
||||
|
||||
// Expect half the jobs to successfully result in a test run that matches the expected test run data for that test target
|
||||
// with the other half having timed out
|
||||
for (const auto& job : runnerJobs)
|
||||
{
|
||||
const JobInfo::IdType jobId = job.GetJobInfo().GetId().m_value;
|
||||
if (jobId % 2)
|
||||
{
|
||||
ValidateJobTimeout(job);
|
||||
}
|
||||
else
|
||||
{
|
||||
ValidateTestRunCompleted(job, m_expectedTestTargetResult[jobId]);
|
||||
ValidateTestTargetRun(job.GetPayload().value().first, m_expectedTestTargetRuns[jobId]);
|
||||
ValidateTestTargetCoverage(job.GetPayload().value().second, m_expectedTestTargetCoverages[jobId][m_coverageLevel]);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
INSTANTIATE_TEST_CASE_P(
|
||||
,
|
||||
InstrumentedTestRunnerFixtureWithConcurrencyAndCoverageAndCoverageExceptionParams,
|
||||
::testing::Combine(
|
||||
::testing::ValuesIn(MaxConcurrentRuns),
|
||||
::testing::Values(CoverageExceptionPolicy::Never, CoverageExceptionPolicy::OnEmptyCoverage)));
|
||||
|
||||
INSTANTIATE_TEST_CASE_P(
|
||||
,
|
||||
InstrumentedTestRunnerFixtureWithConcurrencyAndCoverageAndFailedToLaunchExceptionParams,
|
||||
::testing::Combine(
|
||||
::testing::ValuesIn(MaxConcurrentRuns), ::testing::ValuesIn(CoverageLevels),
|
||||
::testing::ValuesIn(FailedToLaunchExceptionPolicies)));
|
||||
|
||||
INSTANTIATE_TEST_CASE_P(
|
||||
,
|
||||
InstrumentedTestRunnerFixtureWithConcurrencyAndCoverageAndExecutedWithFailureExceptionParams,
|
||||
::testing::Combine(
|
||||
::testing::ValuesIn(MaxConcurrentRuns), ::testing::ValuesIn(CoverageLevels),
|
||||
::testing::ValuesIn(ExecutedWithFailureExceptionPolicies)));
|
||||
|
||||
INSTANTIATE_TEST_CASE_P(
|
||||
,
|
||||
InstrumentedTestRunnerFixtureWithConcurrencyAndCoverageParams,
|
||||
::testing::Combine(::testing::ValuesIn(MaxConcurrentRuns), ::testing::ValuesIn(CoverageLevels)));
|
||||
} // namespace UnitTest
|
||||
-208
@@ -1,208 +0,0 @@
|
||||
/*
|
||||
* All or portions of this file Copyright (c) Amazon.com, Inc. or its affiliates or
|
||||
* its licensors.
|
||||
*
|
||||
* For complete copyright and license terms please see the LICENSE at the root of this
|
||||
* distribution (the "License"). All use of this software is governed by the License,
|
||||
* or, if provided, by the license below or the license accompanying this file. Do not
|
||||
* remove or modify any license notices. This file is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
*
|
||||
*/
|
||||
|
||||
#include <Test/Run/TestImpactTestCoverage.h>
|
||||
|
||||
#include <AzCore/UnitTest/TestTypes.h>
|
||||
#include <AzTest/AzTest.h>
|
||||
|
||||
namespace UnitTest
|
||||
{
|
||||
namespace
|
||||
{
|
||||
AZStd::string GenerateSourcePath(AZ::u32 index)
|
||||
{
|
||||
return AZStd::string::format("SourceFile%u", index);
|
||||
}
|
||||
|
||||
AZStd::string GenerateModulePath(AZ::u32 index)
|
||||
{
|
||||
return AZStd::string::format("Module%u", index);
|
||||
}
|
||||
|
||||
AZStd::vector<TestImpact::LineCoverage> GenerateLineCoverages(AZ::u32 numLines)
|
||||
{
|
||||
AZStd::vector<TestImpact::LineCoverage> lineCoverages;
|
||||
for (size_t i = 0; i < numLines; i++)
|
||||
{
|
||||
// Fudge some superficially different but trivially checkable line coverage data
|
||||
lineCoverages.emplace_back(TestImpact::LineCoverage{i, i * 2});
|
||||
}
|
||||
|
||||
return lineCoverages;
|
||||
}
|
||||
|
||||
TestImpact::SourceCoverage GenerateSourceCoverage(AZ::u32 index, TestImpact::CoverageLevel coverageLevel)
|
||||
{
|
||||
TestImpact::SourceCoverage sourceCoverage;
|
||||
sourceCoverage.m_path = GenerateSourcePath(index);
|
||||
if (coverageLevel == TestImpact::CoverageLevel::Line)
|
||||
{
|
||||
sourceCoverage.m_coverage = GenerateLineCoverages(index + 1);
|
||||
}
|
||||
|
||||
return sourceCoverage;
|
||||
}
|
||||
|
||||
AZStd::vector<TestImpact::SourceCoverage> GenerateSourceCoverages(AZ::u32 numSources, TestImpact::CoverageLevel coverageLevel)
|
||||
{
|
||||
AZStd::vector<TestImpact::SourceCoverage> sourceCoverages;
|
||||
for (AZ::u32 i = 0; i < numSources; i++)
|
||||
{
|
||||
sourceCoverages.emplace_back(GenerateSourceCoverage(i, coverageLevel));
|
||||
}
|
||||
|
||||
return sourceCoverages;
|
||||
}
|
||||
|
||||
TestImpact::ModuleCoverage GenerateModuleCoverage(AZ::u32 index, AZ::u32 numSources, TestImpact::CoverageLevel coverageLevel)
|
||||
{
|
||||
TestImpact::ModuleCoverage moduleCoverage;
|
||||
moduleCoverage.m_path = GenerateModulePath(index);
|
||||
moduleCoverage.m_sources = GenerateSourceCoverages(numSources, coverageLevel);
|
||||
return moduleCoverage;
|
||||
}
|
||||
|
||||
AZStd::vector<TestImpact::ModuleCoverage> GenerateModuleCoverages(AZ::u32 numModules, TestImpact::CoverageLevel coverageLevel)
|
||||
{
|
||||
AZStd::vector<TestImpact::ModuleCoverage> moduleCoverages;
|
||||
for (AZ::u32 i = 0; i < numModules; i++)
|
||||
{
|
||||
// Fudge some superficially different but trivially deducible module coverage data
|
||||
moduleCoverages.emplace_back(GenerateModuleCoverage(i, i + 1, coverageLevel));
|
||||
}
|
||||
|
||||
return moduleCoverages;
|
||||
}
|
||||
} // namespace
|
||||
|
||||
using CoveragePermutation = AZStd::tuple
|
||||
<
|
||||
unsigned, // Number of modules covered
|
||||
TestImpact::CoverageLevel // Test coverage level
|
||||
>;
|
||||
|
||||
// Fixture parameterized for different max number of concurrent jobs
|
||||
class TestCoverageFixtureWithCoverageParams
|
||||
: public AllocatorsTestFixture
|
||||
, public ::testing::WithParamInterface<CoveragePermutation>
|
||||
{
|
||||
public:
|
||||
void SetUp() override;
|
||||
|
||||
protected:
|
||||
void ValidateTestCoverage(const TestImpact::TestCoverage& testCoverage);
|
||||
|
||||
size_t m_numModulesCovered;
|
||||
TestImpact::CoverageLevel m_coverageLevel;
|
||||
};
|
||||
|
||||
void TestCoverageFixtureWithCoverageParams::SetUp()
|
||||
{
|
||||
AllocatorsTestFixture::SetUp();
|
||||
const auto& [numModulesCovered, coverageLevel] = GetParam();
|
||||
m_numModulesCovered = numModulesCovered;
|
||||
m_coverageLevel = coverageLevel;
|
||||
}
|
||||
|
||||
void TestCoverageFixtureWithCoverageParams::ValidateTestCoverage(const TestImpact::TestCoverage& testCoverage)
|
||||
{
|
||||
// Expect the coverage level to match that which was used to generate the module coverages generated
|
||||
EXPECT_EQ(testCoverage.GetCoverageLevel(), m_coverageLevel);
|
||||
|
||||
// Expect the number of modules covered to match the number of module coverages generated
|
||||
EXPECT_EQ(testCoverage.GetNumModulesCovered(), m_numModulesCovered);
|
||||
|
||||
// Expect the number of unique sources covered to match the number of modules coverages generated
|
||||
EXPECT_EQ(testCoverage.GetNumSourcesCovered(), m_numModulesCovered);
|
||||
|
||||
// Expect the unique sources covered to match the procedurally generated source paths
|
||||
for (size_t sourceIndex = 0; sourceIndex < testCoverage.GetNumSourcesCovered(); sourceIndex++)
|
||||
{
|
||||
EXPECT_EQ(testCoverage.GetSourcesCovered()[sourceIndex], GenerateSourcePath(sourceIndex));
|
||||
}
|
||||
|
||||
// Expect each module covered to match that of the corresponding procedurally generated modules
|
||||
for (size_t moduleIndex = 0; moduleIndex < testCoverage.GetNumModulesCovered(); moduleIndex++)
|
||||
{
|
||||
const TestImpact::ModuleCoverage& moduleCoverage = testCoverage.GetModuleCoverages()[moduleIndex];
|
||||
|
||||
// Expect the module path to match that of the corresponding procedurally generated module
|
||||
EXPECT_EQ(moduleCoverage.m_path, GenerateModulePath(moduleIndex));
|
||||
|
||||
// Expect the module's number of sources to match that of the corresponding procedurally generated module
|
||||
EXPECT_EQ(moduleCoverage.m_sources.size(), moduleIndex + 1);
|
||||
|
||||
for (size_t sourceIndex = 0; sourceIndex < moduleCoverage.m_sources.size(); sourceIndex++)
|
||||
{
|
||||
const TestImpact::SourceCoverage& sourceCoverage = moduleCoverage.m_sources[sourceIndex];
|
||||
|
||||
// Expect the source path to match the procedurally generated source path
|
||||
EXPECT_EQ(sourceCoverage.m_path, GenerateSourcePath(sourceIndex));
|
||||
|
||||
if (m_coverageLevel == TestImpact::CoverageLevel::Line)
|
||||
{
|
||||
// Expect there to actually be line coverage data if this coverage was procedurally generated with line data
|
||||
EXPECT_FALSE(sourceCoverage.m_coverage.empty());
|
||||
|
||||
const AZStd::vector<TestImpact::LineCoverage>& lineCoverages = sourceCoverage.m_coverage;
|
||||
|
||||
// Expect the source's number of lines to match that of the corresponding procedurally generated source
|
||||
EXPECT_EQ(lineCoverages.size(), sourceIndex + 1);
|
||||
|
||||
for (size_t lineIndex = 0; lineIndex < lineCoverages.size(); lineIndex++)
|
||||
{
|
||||
// The expected line number and hit count are deduced as follows:
|
||||
// Line number: line index
|
||||
// Hit count: 2x line index
|
||||
EXPECT_EQ(lineCoverages[lineIndex].m_lineNumber, lineIndex);
|
||||
EXPECT_EQ(lineCoverages[lineIndex].m_hitCount, lineIndex * 2);
|
||||
}
|
||||
}
|
||||
else
|
||||
{
|
||||
// Do not expect there to actually be line coverage data if this coverage was not procedurally generated with line data
|
||||
EXPECT_TRUE(sourceCoverage.m_coverage.empty());
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
TEST(TestCoverage, EmptyCoverage_ExpectTestRunException)
|
||||
{
|
||||
// When constructing a test coverage from the empty module coverages
|
||||
TestImpact::TestCoverage testCoverage(AZStd::vector<TestImpact::ModuleCoverage>{});
|
||||
|
||||
// Expect the test coverage fields to be empty
|
||||
EXPECT_EQ(testCoverage.GetNumModulesCovered(), 0);
|
||||
EXPECT_EQ(testCoverage.GetNumSourcesCovered(), 0);
|
||||
EXPECT_TRUE(testCoverage.GetModuleCoverages().empty());
|
||||
EXPECT_TRUE(testCoverage.GetSourcesCovered().empty());
|
||||
}
|
||||
|
||||
TEST_P(TestCoverageFixtureWithCoverageParams, AllCoveragePermutations_ExpectTestCoverageMetaDatasToMatchPermutations)
|
||||
{
|
||||
// Given a procedurally generated test coverage
|
||||
const TestImpact::TestCoverage testCoverage(GenerateModuleCoverages(m_numModulesCovered, m_coverageLevel));
|
||||
|
||||
// Expect the test coverage data and meta-data to match that of the rules used to procedurally generate the coverage data
|
||||
ValidateTestCoverage(testCoverage);
|
||||
}
|
||||
|
||||
INSTANTIATE_TEST_CASE_P(
|
||||
,
|
||||
TestCoverageFixtureWithCoverageParams,
|
||||
::testing::Combine(
|
||||
::testing::Range(1u, 11u),
|
||||
::testing::Values(TestImpact::CoverageLevel::Line, TestImpact::CoverageLevel::Source))
|
||||
);
|
||||
} // namespace UnitTest
|
||||
-890
@@ -1,890 +0,0 @@
|
||||
/*
|
||||
* All or portions of this file Copyright (c) Amazon.com, Inc. or its affiliates or
|
||||
* its licensors.
|
||||
*
|
||||
* For complete copyright and license terms please see the LICENSE at the root of this
|
||||
* distribution (the "License"). All use of this software is governed by the License,
|
||||
* or, if provided, by the license below or the license accompanying this file. Do not
|
||||
* remove or modify any license notices. This file is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
*
|
||||
*/
|
||||
|
||||
#include <TestImpactTestJobRunnerCommon.h>
|
||||
#include <TestImpactTestUtils.h>
|
||||
|
||||
#include <Test/Enumeration/TestImpactTestEnumerationException.h>
|
||||
#include <Test/Enumeration/TestImpactTestEnumerationSerializer.h>
|
||||
#include <Test/Enumeration/TestImpactTestEnumerator.h>
|
||||
|
||||
#include <AzCore/IO/Path/Path.h>
|
||||
#include <AzCore/UnitTest/TestTypes.h>
|
||||
#include <AzCore/std/containers/array.h>
|
||||
#include <AzCore/std/containers/vector.h>
|
||||
#include <AzCore/std/optional.h>
|
||||
#include <AzCore/std/smart_ptr/unique_ptr.h>
|
||||
#include <AzCore/std/string/string.h>
|
||||
#include <AzTest/AzTest.h>
|
||||
|
||||
namespace UnitTest
|
||||
{
|
||||
using JobExceptionPolicy = TestImpact::TestEnumerator::JobExceptionPolicy;
|
||||
using CacheExceptionPolicy = TestImpact::TestEnumerator::CacheExceptionPolicy;
|
||||
|
||||
// Generates the command to run the given test target through AzTestRunner and get gtest to output the enumeration file
|
||||
AZStd::string GetEnumerateCommandForTarget(AZStd::pair<AZ::IO::Path, AZ::IO::Path> testTarget)
|
||||
{
|
||||
return AZStd::string::format(
|
||||
"%s %s AzRunUnitTests --gtest_list_tests --gtest_output=xml:%s",
|
||||
LY_TEST_IMPACT_AZ_TESTRUNNER_BIN,
|
||||
testTarget.first.c_str(), // Path to test target bin
|
||||
testTarget.second.c_str()); // Path to test target gtest enumeration file
|
||||
}
|
||||
|
||||
class TestEnumeratorFixture
|
||||
: public AllocatorsTestFixture
|
||||
{
|
||||
public:
|
||||
void SetUp() override;
|
||||
|
||||
protected:
|
||||
using JobInfo = TestImpact::TestEnumerator::JobInfo;
|
||||
using JobData = TestImpact::TestEnumerator::JobData;
|
||||
|
||||
AZStd::vector<JobInfo> m_jobInfos;
|
||||
AZStd::unique_ptr<TestImpact::TestEnumerator> m_testEnumerator;
|
||||
AZStd::vector<AZStd::string> m_testTargetJobArgs;
|
||||
AZStd::vector<AZStd::pair<AZ::IO::Path, AZ::IO::Path>> m_testTargetPaths;
|
||||
AZStd::vector<TestImpact::TestEnumeration> m_expectedTestTargetEnumerations;
|
||||
AZStd::vector<AZStd::string> m_cacheFiles;
|
||||
size_t m_maxConcurrency = 0;
|
||||
};
|
||||
|
||||
void TestEnumeratorFixture::SetUp()
|
||||
{
|
||||
UnitTest::AllocatorsTestFixture::SetUp();
|
||||
|
||||
DeleteFiles(LY_TEST_IMPACT_TEST_TARGET_ENUMERATION_DIR, "*.cache");
|
||||
DeleteFiles(LY_TEST_IMPACT_TEST_TARGET_ENUMERATION_DIR, "*.xml");
|
||||
|
||||
// first: path to test target bin
|
||||
// second: path to test target gtest enumeration file in XML format
|
||||
const AZStd::string enumPath = AZStd::string(LY_TEST_IMPACT_TEST_TARGET_ENUMERATION_DIR) + "/%s.Enumeration.xml";
|
||||
m_testTargetPaths.emplace_back(
|
||||
LY_TEST_IMPACT_TEST_TARGET_A_BIN, AZStd::string::format(enumPath.c_str(), LY_TEST_IMPACT_TEST_TARGET_A_BASE_NAME));
|
||||
m_testTargetPaths.emplace_back(
|
||||
LY_TEST_IMPACT_TEST_TARGET_B_BIN, AZStd::string::format(enumPath.c_str(), LY_TEST_IMPACT_TEST_TARGET_B_BASE_NAME));
|
||||
m_testTargetPaths.emplace_back(
|
||||
LY_TEST_IMPACT_TEST_TARGET_C_BIN, AZStd::string::format(enumPath.c_str(), LY_TEST_IMPACT_TEST_TARGET_C_BASE_NAME));
|
||||
m_testTargetPaths.emplace_back(
|
||||
LY_TEST_IMPACT_TEST_TARGET_D_BIN, AZStd::string::format(enumPath.c_str(), LY_TEST_IMPACT_TEST_TARGET_D_BASE_NAME));
|
||||
|
||||
m_expectedTestTargetEnumerations.emplace_back(GetTestTargetATestEnumerationSuites());
|
||||
m_expectedTestTargetEnumerations.emplace_back(GetTestTargetBTestEnumerationSuites());
|
||||
m_expectedTestTargetEnumerations.emplace_back(GetTestTargetCTestEnumerationSuites());
|
||||
m_expectedTestTargetEnumerations.emplace_back(GetTestTargetDTestEnumerationSuites());
|
||||
|
||||
// Path to enumeration file in TIAF internal JSON format
|
||||
m_cacheFiles.emplace_back(
|
||||
AZStd::string::format("%s/%s.cache", LY_TEST_IMPACT_TEST_TARGET_ENUMERATION_DIR, LY_TEST_IMPACT_TEST_TARGET_A_BASE_NAME));
|
||||
m_cacheFiles.emplace_back(
|
||||
AZStd::string::format("%s/%s.cache", LY_TEST_IMPACT_TEST_TARGET_ENUMERATION_DIR, LY_TEST_IMPACT_TEST_TARGET_B_BASE_NAME));
|
||||
m_cacheFiles.emplace_back(
|
||||
AZStd::string::format("%s/%s.cache", LY_TEST_IMPACT_TEST_TARGET_ENUMERATION_DIR, LY_TEST_IMPACT_TEST_TARGET_C_BASE_NAME));
|
||||
m_cacheFiles.emplace_back(
|
||||
AZStd::string::format("%s/%s.cache", LY_TEST_IMPACT_TEST_TARGET_ENUMERATION_DIR, LY_TEST_IMPACT_TEST_TARGET_D_BASE_NAME));
|
||||
|
||||
for (const auto& testTarget : m_testTargetPaths)
|
||||
{
|
||||
m_testTargetJobArgs.emplace_back(GetEnumerateCommandForTarget(testTarget));
|
||||
}
|
||||
}
|
||||
|
||||
// Fixture parameterized for different max number of concurrent jobs
|
||||
class TestEnumeratorFixtureWithConcurrencyParams
|
||||
: public TestEnumeratorFixture
|
||||
, public ::testing::WithParamInterface<size_t>
|
||||
{
|
||||
public:
|
||||
void SetUp() override
|
||||
{
|
||||
TestEnumeratorFixture::SetUp();
|
||||
m_maxConcurrency = GetParam();
|
||||
}
|
||||
};
|
||||
|
||||
using ConcurrencyAndJobExceptionPermutation = AZStd::tuple
|
||||
<
|
||||
size_t, // Max number of concurrent processes
|
||||
JobExceptionPolicy // Test job exception policy
|
||||
>;
|
||||
|
||||
// Fixture parameterized for different max number of concurrent jobs and different job exception policies
|
||||
class TestEnumeratorFixtureWithConcurrencyAndJobExceptionParams
|
||||
: public TestEnumeratorFixture
|
||||
, public ::testing::WithParamInterface<ConcurrencyAndJobExceptionPermutation>
|
||||
{
|
||||
public:
|
||||
void SetUp() override
|
||||
{
|
||||
TestEnumeratorFixture::SetUp();
|
||||
const auto& [maxConcurrency, jobExceptionPolicy] = GetParam();
|
||||
m_maxConcurrency = maxConcurrency;
|
||||
m_jobExceptionPolicy = jobExceptionPolicy;
|
||||
}
|
||||
|
||||
protected:
|
||||
JobExceptionPolicy m_jobExceptionPolicy = JobExceptionPolicy::Never;
|
||||
};
|
||||
|
||||
using ConcurrencyAndCacheExceptionPermutation = AZStd::tuple
|
||||
<
|
||||
size_t, // Max number of concurrent processes
|
||||
CacheExceptionPolicy // Test enumeration exception policy
|
||||
>;
|
||||
|
||||
// Fixture parameterized for different max number of concurrent jobs and different enumeration exception policies
|
||||
class TestEnumeratorFixtureWithConcurrencyAndCacheExceptionParams
|
||||
: public TestEnumeratorFixture
|
||||
, public ::testing::WithParamInterface<ConcurrencyAndCacheExceptionPermutation>
|
||||
{
|
||||
public:
|
||||
void SetUp() override
|
||||
{
|
||||
TestEnumeratorFixture::SetUp();
|
||||
const auto& [maxConcurrency, cacheExceptionPolicy] = GetParam();
|
||||
m_maxConcurrency = maxConcurrency;
|
||||
m_cacheExceptionPolicy = cacheExceptionPolicy;
|
||||
}
|
||||
|
||||
protected:
|
||||
CacheExceptionPolicy m_cacheExceptionPolicy = CacheExceptionPolicy::Never;
|
||||
};
|
||||
|
||||
namespace
|
||||
{
|
||||
AZStd::array<size_t, 4> MaxConcurrentEnumerations = {{1, 2, 3, 4}};
|
||||
|
||||
AZStd::array<JobExceptionPolicy, 3> JobExceptionPolicies =
|
||||
{
|
||||
JobExceptionPolicy::Never,
|
||||
JobExceptionPolicy::OnExecutedWithFailure,
|
||||
JobExceptionPolicy::OnFailedToExecute
|
||||
};
|
||||
|
||||
AZStd::array<CacheExceptionPolicy, 4> CacheExceptionPolicies =
|
||||
{
|
||||
CacheExceptionPolicy::Never,
|
||||
CacheExceptionPolicy::OnCacheNotExist,
|
||||
CacheExceptionPolicy::OnCacheReadFailure,
|
||||
CacheExceptionPolicy::OnCacheWriteFailure
|
||||
};
|
||||
}
|
||||
|
||||
// Validates that the specified job successfully read from its test enumeration cache
|
||||
void ValidateJobSuccessfulCacheRead(const TestImpact::TestEnumerator::Job& job)
|
||||
{
|
||||
EXPECT_EQ(job.GetResult(), TestImpact::JobResult::NotExecuted);
|
||||
EXPECT_EQ(job.GetStartTime(), AZStd::chrono::high_resolution_clock::time_point());
|
||||
EXPECT_EQ(job.GetEndTime(), AZStd::chrono::high_resolution_clock::time_point());
|
||||
EXPECT_EQ(job.GetDuration(), AZStd::chrono::milliseconds(0));
|
||||
EXPECT_FALSE(job.GetReturnCode().has_value());
|
||||
EXPECT_TRUE(job.GetPayload().has_value());
|
||||
}
|
||||
|
||||
// Validates that the specified test enumeration matched the expected output
|
||||
void ValidateTestTargetEnumeration(const TestImpact::TestEnumeration& actualResult, const TestImpact::TestEnumeration& expectedResult)
|
||||
{
|
||||
EXPECT_TRUE(actualResult == expectedResult);
|
||||
EXPECT_EQ(actualResult.GetNumTestSuites(), CalculateNumTestSuites(expectedResult.GetTestSuites()));
|
||||
EXPECT_EQ(actualResult.GetNumTests(), CalculateNumTests(expectedResult.GetTestSuites()));
|
||||
EXPECT_EQ(actualResult.GetNumEnabledTests(), CalculateNumEnabledTests(expectedResult.GetTestSuites()));
|
||||
EXPECT_EQ(actualResult.GetNumDisabledTests(), CalculateNumDisabledTests(expectedResult.GetTestSuites()));
|
||||
}
|
||||
|
||||
// Validates that the specified test enumeration cache matches the expected output
|
||||
void ValidateTestEnumerationCache(const AZ::IO::Path& cacheFile, const TestImpact::TestEnumeration& expectedEnumeration)
|
||||
{
|
||||
// Cache file must exist
|
||||
const auto fileSize = AZ::IO::SystemFile::Length(cacheFile.c_str());
|
||||
EXPECT_GT(fileSize, 0);
|
||||
|
||||
// Read raw byte data from cache
|
||||
AZStd::vector<char> buffer(fileSize + 1);
|
||||
buffer[fileSize] = 0;
|
||||
EXPECT_TRUE(AZ::IO::SystemFile::Read(cacheFile.c_str(), buffer.data()));
|
||||
|
||||
// Transform raw byte data to raw string data and attempt to construct the test enumeration
|
||||
AZStd::string rawEnum(buffer.begin(), buffer.end());
|
||||
TestImpact::TestEnumeration actualEnumeration = TestImpact::DeserializeTestEnumeration(rawEnum);
|
||||
|
||||
// Check that the constructed test enumeration matches the expected enumeration
|
||||
ValidateTestTargetEnumeration(actualEnumeration, expectedEnumeration);
|
||||
}
|
||||
|
||||
// Validates that the specified cache file does not exist
|
||||
void ValidateInvalidTestEnumerationCache(const AZ::IO::Path& cacheFile)
|
||||
{
|
||||
EXPECT_FALSE(AZ::IO::SystemFile::Exists(cacheFile.c_str()));
|
||||
}
|
||||
|
||||
TEST_P(
|
||||
TestEnumeratorFixtureWithConcurrencyAndJobExceptionParams, InvalidCommandArgument_ExpectJobResulFailedToExecuteeOrTestJobException)
|
||||
{
|
||||
// Given a test enumerator with no client callback or enumeration timeout or enumerator timeout
|
||||
m_testEnumerator = AZStd::make_unique<TestImpact::TestEnumerator>(
|
||||
AZStd::nullopt,
|
||||
m_maxConcurrency,
|
||||
AZStd::nullopt,
|
||||
AZStd::nullopt);
|
||||
|
||||
// Given a mixture of test enumeration jobs with valid and invalid command arguments
|
||||
for (size_t jobId = 0; jobId < m_testTargetJobArgs.size(); jobId++)
|
||||
{
|
||||
const AZStd::string args = (jobId % 2) ? InvalidProcessPath : m_testTargetJobArgs[jobId];
|
||||
JobData jobData(m_testTargetPaths[jobId].second, AZStd::nullopt);
|
||||
m_jobInfos.emplace_back(JobInfo({jobId}, args, AZStd::move(jobData)));
|
||||
}
|
||||
|
||||
try
|
||||
{
|
||||
// When the test enumeration jobs are executed with different exception policies
|
||||
const auto enumerationJobs = m_testEnumerator->Enumerate(m_jobInfos, CacheExceptionPolicy::Never, m_jobExceptionPolicy);
|
||||
|
||||
// Expect this statement to be reachable only if no exception policy for launch failures
|
||||
EXPECT_FALSE(::IsFlagSet(m_jobExceptionPolicy, JobExceptionPolicy::OnFailedToExecute));
|
||||
|
||||
for (const auto& job : enumerationJobs)
|
||||
{
|
||||
const JobInfo::IdType jobId = job.GetJobInfo().GetId().m_value;
|
||||
if (jobId % 2)
|
||||
{
|
||||
// Expect invalid jobs have a job result of FailedToExecute
|
||||
ValidateJobFailedToExecute(job);
|
||||
}
|
||||
else
|
||||
{
|
||||
// Expect the valid jobs job to successfully result in a test enumeration that matches the expected test enumeration data
|
||||
ValidateJobExecutedSuccessfully(job);
|
||||
ValidateTestTargetEnumeration(job.GetPayload().value(), m_expectedTestTargetEnumerations[jobId]);
|
||||
}
|
||||
}
|
||||
}
|
||||
catch ([[maybe_unused]] const TestImpact::TestJobException& e)
|
||||
{
|
||||
// Expect this statement to be reachable only if there is an exception policy for launch failures
|
||||
EXPECT_TRUE(::IsFlagSet(m_jobExceptionPolicy, JobExceptionPolicy::OnFailedToExecute));
|
||||
}
|
||||
catch (...)
|
||||
{
|
||||
// Do not expect any other exceptions
|
||||
FAIL();
|
||||
}
|
||||
}
|
||||
|
||||
TEST_P(
|
||||
TestEnumeratorFixtureWithConcurrencyAndJobExceptionParams, ErroneousReturnCode_ExpectJobResultExecutedWithFailureOrTestJobException)
|
||||
{
|
||||
// Given a test enumerator with no client callback or enumeration timeout or enumerator timeout
|
||||
m_testEnumerator = AZStd::make_unique<TestImpact::TestEnumerator>(
|
||||
AZStd::nullopt,
|
||||
m_maxConcurrency,
|
||||
AZStd::nullopt,
|
||||
AZStd::nullopt);
|
||||
|
||||
// Given a mixture of test enumeration jobs that execute and return either successfully or with failure
|
||||
for (size_t jobId = 0; jobId < m_testTargetJobArgs.size(); jobId++)
|
||||
{
|
||||
JobData jobData(m_testTargetPaths[jobId].second, AZStd::nullopt);
|
||||
const AZStd::string args = (jobId % 2)
|
||||
? AZStd::string::format("%s %s", ValidProcessPath, ConstructTestProcessArgs(jobId, AZStd::chrono::milliseconds(0)).c_str())
|
||||
: m_testTargetJobArgs[jobId];
|
||||
m_jobInfos.emplace_back(JobInfo({jobId}, args, AZStd::move(jobData)));
|
||||
}
|
||||
|
||||
try
|
||||
{
|
||||
// When the test enumeration jobs are executed with different exception policies
|
||||
const auto enumerationJobs = m_testEnumerator->Enumerate(m_jobInfos, CacheExceptionPolicy::Never, m_jobExceptionPolicy);
|
||||
|
||||
// Expect this statement to be reachable only if no exception policy for jobs that return with error
|
||||
EXPECT_FALSE(::IsFlagSet(m_jobExceptionPolicy, JobExceptionPolicy::OnExecutedWithFailure));
|
||||
|
||||
for (const auto& job : enumerationJobs)
|
||||
{
|
||||
const JobInfo::IdType jobId = job.GetJobInfo().GetId().m_value;
|
||||
if (jobId % 2)
|
||||
{
|
||||
// Expect failed jobs to have job result ExecutedWithFailure and a non-zero return code
|
||||
ValidateJobExecutedWithFailure(job);
|
||||
}
|
||||
else
|
||||
{
|
||||
// Expect the valid jobs job to successfully result in a test enumeration that matches the expected test enumeration data
|
||||
ValidateJobExecutedSuccessfully(job);
|
||||
ValidateTestTargetEnumeration(job.GetPayload().value(), m_expectedTestTargetEnumerations[jobId]);
|
||||
}
|
||||
}
|
||||
}
|
||||
catch ([[maybe_unused]] const TestImpact::TestJobException& e)
|
||||
{
|
||||
// Expect this statement to be reachable only if there is an exception policy for jobs that return with error
|
||||
EXPECT_TRUE(::IsFlagSet(m_jobExceptionPolicy, JobExceptionPolicy::OnExecutedWithFailure));
|
||||
}
|
||||
catch (...)
|
||||
{
|
||||
// Do not expect any other exceptions
|
||||
FAIL();
|
||||
}
|
||||
}
|
||||
|
||||
TEST_P(TestEnumeratorFixtureWithConcurrencyAndCacheExceptionParams, EmptyCacheRead_NoCacheDataButEnumerationsMatchTestSuitesInTarget)
|
||||
{
|
||||
// Given a test enumerator with no client callback, enumeration timeout or enumerator timeout
|
||||
m_testEnumerator = AZStd::make_unique<TestImpact::TestEnumerator>(
|
||||
AZStd::nullopt,
|
||||
m_maxConcurrency,
|
||||
AZStd::nullopt,
|
||||
AZStd::nullopt);
|
||||
|
||||
// Given an test enumeration job for each test target that reads from an enumeration caching
|
||||
for (size_t jobId = 0; jobId < m_testTargetJobArgs.size(); jobId++)
|
||||
{
|
||||
JobData jobData(m_testTargetPaths[jobId].second, JobData::Cache{JobData::CachePolicy::Read, m_cacheFiles[jobId]});
|
||||
m_jobInfos.emplace_back(JobInfo({jobId}, m_testTargetJobArgs[jobId], AZStd::move(jobData)));
|
||||
}
|
||||
|
||||
try
|
||||
{
|
||||
// When the test enumeration jobs are executed with different exception policies
|
||||
const auto enumerationJobs = m_testEnumerator->Enumerate(m_jobInfos, m_cacheExceptionPolicy, JobExceptionPolicy::Never);
|
||||
|
||||
// Expect this statement to be reachable only if no exception policy for read attempts of non-existent caches
|
||||
EXPECT_FALSE(::IsFlagSet(m_cacheExceptionPolicy, CacheExceptionPolicy::OnCacheNotExist));
|
||||
|
||||
// Expect each job to successfully result in a test enumeration that matches the expected test enumeration data for that test target
|
||||
// even though the cache files could not be read
|
||||
for (const auto& job : enumerationJobs)
|
||||
{
|
||||
const JobInfo::IdType jobId = job.GetJobInfo().GetId().m_value;
|
||||
ValidateJobExecutedSuccessfully(job);
|
||||
ValidateTestTargetEnumeration(job.GetPayload().value(), m_expectedTestTargetEnumerations[jobId]);
|
||||
ValidateInvalidTestEnumerationCache(job.GetJobInfo().GetCache()->m_file);
|
||||
}
|
||||
}
|
||||
catch ([[maybe_unused]] const TestImpact::TestEnumerationException& e)
|
||||
{
|
||||
// Expect this statement to be reachable only if there is an exception policy for read attempts of non-existent caches
|
||||
EXPECT_TRUE(::IsFlagSet(m_cacheExceptionPolicy, CacheExceptionPolicy::OnCacheNotExist));
|
||||
}
|
||||
catch (...)
|
||||
{
|
||||
// Do not expect any other exceptions
|
||||
FAIL();
|
||||
}
|
||||
}
|
||||
|
||||
// Note: this test only cues up one test for enumeration but still runs the permutations for max concurrency so there is duplicated work
|
||||
TEST_P(
|
||||
TestEnumeratorFixtureWithConcurrencyAndCacheExceptionParams,
|
||||
EmptyCacheDataRead_ExpectEnumerationsMatchTestSuitesInTargetOrTestEnumerationException)
|
||||
{
|
||||
// Given an enumeration cache for Test Target A with invalid JSON data
|
||||
WriteTextToFile("", m_cacheFiles[TestTargetA]);
|
||||
|
||||
// Given a test enumerator with no client callback, enumeration timeout or enumerator timeout
|
||||
m_testEnumerator = AZStd::make_unique<TestImpact::TestEnumerator>(
|
||||
AZStd::nullopt,
|
||||
m_maxConcurrency,
|
||||
AZStd::nullopt,
|
||||
AZStd::nullopt);
|
||||
|
||||
// Given an test enumeration job that will attempt to read an invalid enumeration cache
|
||||
JobData jobData(m_testTargetPaths[TestTargetA].second, JobData::Cache{JobData::CachePolicy::Read, m_cacheFiles[TestTargetA]});
|
||||
m_jobInfos.emplace_back(JobInfo({TestTargetA}, m_testTargetJobArgs[TestTargetA], AZStd::move(jobData)));
|
||||
|
||||
try
|
||||
{
|
||||
// When the test enumeration jobs are executed with different exception policies
|
||||
const auto enumerationJobs = m_testEnumerator->Enumerate(m_jobInfos, m_cacheExceptionPolicy, JobExceptionPolicy::Never);
|
||||
|
||||
// Expect this statement to be reachable only if no exception policy for cache reads that fail
|
||||
EXPECT_FALSE(::IsFlagSet(m_cacheExceptionPolicy, CacheExceptionPolicy::OnCacheReadFailure));
|
||||
|
||||
for (const auto& job : enumerationJobs)
|
||||
{
|
||||
const JobInfo::IdType jobId = job.GetJobInfo().GetId().m_value;
|
||||
|
||||
// Expect the valid jobs job to successfully result in a test enumeration that matches the expected test enumeration data
|
||||
ValidateJobExecutedSuccessfully(job);
|
||||
ValidateTestTargetEnumeration(job.GetPayload().value(), m_expectedTestTargetEnumerations[jobId]);
|
||||
}
|
||||
}
|
||||
catch ([[maybe_unused]] const TestImpact::TestEnumerationException& e)
|
||||
{
|
||||
// Expect this statement to be reachable only if there is an exception policy for cache reads that fail
|
||||
EXPECT_TRUE(::IsFlagSet(m_cacheExceptionPolicy, CacheExceptionPolicy::OnCacheReadFailure));
|
||||
}
|
||||
catch (...)
|
||||
{
|
||||
// Do not expect any other exceptions
|
||||
FAIL();
|
||||
}
|
||||
}
|
||||
|
||||
TEST_P(
|
||||
TestEnumeratorFixtureWithConcurrencyAndCacheExceptionParams,
|
||||
InvalidCacheWrite_ExpectEnumerationsMatchTestSuitesInTargetOrTestEnumerationException)
|
||||
{
|
||||
// Given a test enumerator with no client callback,, enumeration timeout or enumerator timeout
|
||||
m_testEnumerator = AZStd::make_unique<TestImpact::TestEnumerator>(
|
||||
AZStd::nullopt,
|
||||
m_maxConcurrency,
|
||||
AZStd::nullopt,
|
||||
AZStd::nullopt);
|
||||
|
||||
// Given an test enumeration job that will attempt to write to an invalid enumeration cache
|
||||
JobData jobData(m_testTargetPaths[TestTargetA].second, JobData::Cache{JobData::CachePolicy::Write, InvalidProcessPath});
|
||||
m_jobInfos.emplace_back(JobInfo({TestTargetA}, m_testTargetJobArgs[TestTargetA], AZStd::move(jobData)));
|
||||
|
||||
try
|
||||
{
|
||||
// When the test enumeration job is executed
|
||||
const auto enumerationJobs = m_testEnumerator->Enumerate(m_jobInfos, m_cacheExceptionPolicy, JobExceptionPolicy::Never);
|
||||
|
||||
// Expect this statement to be reachable only if no exception policy for cache writes that fail
|
||||
EXPECT_FALSE(::IsFlagSet(m_cacheExceptionPolicy, CacheExceptionPolicy::OnCacheWriteFailure));
|
||||
|
||||
for (const auto& job : enumerationJobs)
|
||||
{
|
||||
const JobInfo::IdType jobId = job.GetJobInfo().GetId().m_value;
|
||||
|
||||
// Expect the valid jobs job to successfully result in a test enumeration that matches the expected test enumeration data
|
||||
ValidateJobExecutedSuccessfully(job);
|
||||
ValidateTestTargetEnumeration(job.GetPayload().value(), m_expectedTestTargetEnumerations[jobId]);
|
||||
}
|
||||
}
|
||||
catch ([[maybe_unused]] const TestImpact::TestEnumerationException& e)
|
||||
{
|
||||
// Expect this statement to be reachable only if there is an exception policy for cache writes that fail
|
||||
EXPECT_TRUE(::IsFlagSet(m_cacheExceptionPolicy, CacheExceptionPolicy::OnCacheWriteFailure));
|
||||
}
|
||||
catch (...)
|
||||
{
|
||||
// Do not expect any other exceptions
|
||||
FAIL();
|
||||
}
|
||||
}
|
||||
|
||||
TEST_P(TestEnumeratorFixtureWithConcurrencyParams, ValidAndInvalidCacheRead_CachedEnumerationsMatchTestSuitesInTarget)
|
||||
{
|
||||
// Given the cache file written for only test target B
|
||||
WriteTextToFile(TestImpact::SerializeTestEnumeration(m_expectedTestTargetEnumerations[TestTargetB]), m_cacheFiles[TestTargetB]);
|
||||
|
||||
// Given a test enumerator with no client callback, enumeration timeout or enumerator timeout
|
||||
m_testEnumerator = AZStd::make_unique<TestImpact::TestEnumerator>(
|
||||
AZStd::nullopt,
|
||||
m_maxConcurrency,
|
||||
AZStd::nullopt,
|
||||
AZStd::nullopt);
|
||||
|
||||
// Given test enumeration jobs test target A and D with no enumeration caching
|
||||
m_jobInfos.emplace_back(
|
||||
JobInfo({TestTargetA}, m_testTargetJobArgs[TestTargetA], JobData{m_testTargetPaths[TestTargetA].second, AZStd::nullopt}));
|
||||
m_jobInfos.emplace_back(
|
||||
JobInfo({TestTargetD}, m_testTargetJobArgs[TestTargetD], JobData{m_testTargetPaths[TestTargetD].second, AZStd::nullopt}));
|
||||
|
||||
// Given test target B with enumeration cache reading and a valid cache file
|
||||
m_jobInfos.emplace_back(JobInfo(
|
||||
{TestTargetB}, m_testTargetJobArgs[TestTargetB],
|
||||
JobData{m_testTargetPaths[TestTargetB].second, JobInfo::Cache{JobData::CachePolicy::Read, m_cacheFiles[TestTargetB]}}));
|
||||
|
||||
// Given test target C with enumeration cache reading and an invalid cache file
|
||||
m_jobInfos.emplace_back(JobInfo(
|
||||
{TestTargetC}, m_testTargetJobArgs[TestTargetC],
|
||||
JobData{m_testTargetPaths[TestTargetC].second, JobInfo::Cache{JobData::CachePolicy::Read, "nothing"}}));
|
||||
|
||||
// When the test enumeration jobs are executed
|
||||
const auto enumerationJobs = m_testEnumerator->Enumerate(m_jobInfos, CacheExceptionPolicy::Never, JobExceptionPolicy::Never);
|
||||
|
||||
// Expect each job to successfully result in a test enumeration that matches the expected test enumeration data for that test target
|
||||
for (const auto& job : enumerationJobs)
|
||||
{
|
||||
const JobInfo::IdType jobId = job.GetJobInfo().GetId().m_value;
|
||||
|
||||
switch (jobId)
|
||||
{
|
||||
case TestTargetA: // No cache read
|
||||
case TestTargetC: // Cache read, but invalid cache, so re-enumerate anyway
|
||||
case TestTargetD: // No cache read
|
||||
{
|
||||
ValidateJobExecutedSuccessfully(job);
|
||||
break;
|
||||
}
|
||||
case TestTargetB: // Cache read, successful cache read, so job not executed
|
||||
{
|
||||
ValidateJobSuccessfulCacheRead(job);
|
||||
break;
|
||||
}
|
||||
default:
|
||||
{
|
||||
FAIL();
|
||||
}
|
||||
}
|
||||
|
||||
// Regardless of cache policy and cache failures all targets should still produce the expected test enumerations
|
||||
ValidateTestTargetEnumeration(job.GetPayload().value(), m_expectedTestTargetEnumerations[jobId]);
|
||||
}
|
||||
}
|
||||
|
||||
TEST_F(TestEnumeratorFixture, InvalidCacheDataRead_TestEnumerationException)
|
||||
{
|
||||
// Given an enumeration cache for Test Target A with invalid JSON data
|
||||
WriteTextToFile("There is no valid cache data here", m_cacheFiles[TestTargetA]);
|
||||
|
||||
// Given a test enumerator with no client callback, concurrency, enumeration timeout or enumerator timeout
|
||||
m_testEnumerator = AZStd::make_unique<TestImpact::TestEnumerator>(
|
||||
AZStd::nullopt,
|
||||
OneConcurrentProcess,
|
||||
AZStd::nullopt,
|
||||
AZStd::nullopt);
|
||||
|
||||
// Given an test enumeration job that will attempt to read an invalid enumeration cache
|
||||
JobData jobData(m_testTargetPaths[TestTargetA].second, JobData::Cache{JobData::CachePolicy::Read, m_cacheFiles[TestTargetA]});
|
||||
m_jobInfos.emplace_back(JobInfo({TestTargetA}, m_testTargetJobArgs[TestTargetA], AZStd::move(jobData)));
|
||||
|
||||
try
|
||||
{
|
||||
// When the test enumeration jobs are executed with different exception policies
|
||||
const auto enumerationJobs = m_testEnumerator->Enumerate(m_jobInfos, CacheExceptionPolicy::Never, JobExceptionPolicy::Never);
|
||||
}
|
||||
catch ([[maybe_unused]] const TestImpact::TestEnumerationException& e)
|
||||
{
|
||||
// Expect this statement to be reachable only if there is an exception policy for jobs that return with error
|
||||
SUCCEED();
|
||||
}
|
||||
catch (...)
|
||||
{
|
||||
// Do not expect any other exceptions
|
||||
FAIL();
|
||||
}
|
||||
}
|
||||
|
||||
TEST_P(TestEnumeratorFixtureWithConcurrencyParams, ValidCacheWrite_CachedEnumerationsMatchTestSuitesInTarget)
|
||||
{
|
||||
// Given a test enumerator with no client callback, enumeration timeout or enumerator timeout
|
||||
m_testEnumerator = AZStd::make_unique<TestImpact::TestEnumerator>(
|
||||
AZStd::nullopt,
|
||||
m_maxConcurrency,
|
||||
AZStd::nullopt,
|
||||
AZStd::nullopt);
|
||||
|
||||
// Given an test enumeration job for each test target with write enumeration caching
|
||||
for (size_t jobId = 0; jobId < m_testTargetJobArgs.size(); jobId++)
|
||||
{
|
||||
JobData jobData(m_testTargetPaths[jobId].second, JobData::Cache{JobData::CachePolicy::Write, m_cacheFiles[jobId]});
|
||||
m_jobInfos.emplace_back(JobInfo({jobId}, m_testTargetJobArgs[jobId], AZStd::move(jobData)));
|
||||
}
|
||||
|
||||
// When the test enumeration jobs are executed
|
||||
const auto enumerationJobs = m_testEnumerator->Enumerate(m_jobInfos, CacheExceptionPolicy::Never, JobExceptionPolicy::Never);
|
||||
|
||||
// Expect each job to successfully result in a test enumeration and cache that matches the expected test enumeration data for that
|
||||
// test target
|
||||
for (const auto& job : enumerationJobs)
|
||||
{
|
||||
const JobInfo::IdType jobId = job.GetJobInfo().GetId().m_value;
|
||||
ValidateJobExecutedSuccessfully(job);
|
||||
ValidateTestTargetEnumeration(job.GetPayload().value(), m_expectedTestTargetEnumerations[jobId]);
|
||||
ValidateTestEnumerationCache(job.GetJobInfo().GetCache()->m_file, m_expectedTestTargetEnumerations[jobId]);
|
||||
}
|
||||
}
|
||||
|
||||
TEST_F(TestEnumeratorFixture, EmptyArtifact_ExpectTestEnumerationException)
|
||||
{
|
||||
// Given a test enumerator with no client callback, concurrency, enumeration timeout or enumerator timeout
|
||||
m_testEnumerator = AZStd::make_unique<TestImpact::TestEnumerator>(
|
||||
AZStd::nullopt,
|
||||
OneConcurrentProcess,
|
||||
AZStd::nullopt,
|
||||
AZStd::nullopt);
|
||||
|
||||
// Given an test enumeration job that will return successfully but with an empty artifact string
|
||||
m_jobInfos.emplace_back(JobInfo({0}, m_testTargetJobArgs[0], JobData("", AZStd::nullopt)));
|
||||
|
||||
try
|
||||
{
|
||||
// When the test enumeration job is executed
|
||||
const auto enumerationJobs = m_testEnumerator->Enumerate(m_jobInfos, CacheExceptionPolicy::Never, JobExceptionPolicy::Never);
|
||||
|
||||
// Do not expect this statement to be reachable
|
||||
FAIL();
|
||||
}
|
||||
catch ([[maybe_unused]] const TestImpact::TestEnumerationException& e)
|
||||
{
|
||||
// Expect an enumeration exception
|
||||
SUCCEED();
|
||||
}
|
||||
catch (...)
|
||||
{
|
||||
// Do not expect any other exceptions
|
||||
FAIL();
|
||||
}
|
||||
}
|
||||
|
||||
TEST_F(TestEnumeratorFixture, InvalidArtifact_ExpectTestEnumerationException)
|
||||
{
|
||||
// Given a test enumerator with no client callback, concurrency, enumeration timeout or enumerator timeout
|
||||
m_testEnumerator = AZStd::make_unique<TestImpact::TestEnumerator>(
|
||||
AZStd::nullopt,
|
||||
OneConcurrentProcess,
|
||||
AZStd::nullopt,
|
||||
AZStd::nullopt);
|
||||
|
||||
// Given an test enumeration job that will return successfully but not produce an artifact
|
||||
m_jobInfos.emplace_back(JobInfo(
|
||||
{0}, AZStd::string::format("%s %s", ValidProcessPath, ConstructTestProcessArgs(0, AZStd::chrono::milliseconds(0)).c_str()),
|
||||
JobData("", AZStd::nullopt)));
|
||||
|
||||
try
|
||||
{
|
||||
// When the test enumeration job is executed
|
||||
const auto enumerationJobs = m_testEnumerator->Enumerate(m_jobInfos, CacheExceptionPolicy::Never, JobExceptionPolicy::Never);
|
||||
|
||||
// Do not expect this statement to be reachable
|
||||
FAIL();
|
||||
}
|
||||
catch ([[maybe_unused]] const TestImpact::TestEnumerationException& e)
|
||||
{
|
||||
// Expect an enumeration exception
|
||||
SUCCEED();
|
||||
}
|
||||
catch (...)
|
||||
{
|
||||
// Do not expect any other exceptions
|
||||
FAIL();
|
||||
}
|
||||
}
|
||||
|
||||
TEST_P(TestEnumeratorFixtureWithConcurrencyParams, EnumerateTestTargets_EnumerationsMatchTestSuitesInTarget)
|
||||
{
|
||||
// Given a test enumerator with no client callback, enumeration timeout or enumerator timeout
|
||||
m_testEnumerator = AZStd::make_unique<TestImpact::TestEnumerator>(
|
||||
AZStd::nullopt,
|
||||
m_maxConcurrency,
|
||||
AZStd::nullopt,
|
||||
AZStd::nullopt);
|
||||
|
||||
// Given an test enumeration job for each test target with no enumeration caching
|
||||
for (size_t jobId = 0; jobId < m_testTargetJobArgs.size(); jobId++)
|
||||
{
|
||||
JobData jobData(m_testTargetPaths[jobId].second, AZStd::nullopt);
|
||||
m_jobInfos.emplace_back(JobInfo({jobId}, m_testTargetJobArgs[jobId], AZStd::move(jobData)));
|
||||
}
|
||||
|
||||
// When the test enumeration jobs are executed
|
||||
const auto enumerationJobs = m_testEnumerator->Enumerate(m_jobInfos, CacheExceptionPolicy::Never, JobExceptionPolicy::Never);
|
||||
|
||||
// Expect each job to successfully result in a test enumeration that matches the expected test enumeration data for that test target
|
||||
for (const auto& job : enumerationJobs)
|
||||
{
|
||||
const JobInfo::IdType jobId = job.GetJobInfo().GetId().m_value;
|
||||
ValidateJobExecutedSuccessfully(job);
|
||||
ValidateTestTargetEnumeration(job.GetPayload().value(), m_expectedTestTargetEnumerations[jobId]);
|
||||
}
|
||||
}
|
||||
|
||||
TEST_P(TestEnumeratorFixtureWithConcurrencyParams, EnumerateTestTargetsWithArbitraryJobIds_EnumerationsMatchTestSuitesInTarget)
|
||||
{
|
||||
// Given a set of arbitrary job ids to be used for the test target jobs
|
||||
enum
|
||||
{
|
||||
ArbitraryA = 36,
|
||||
ArbitraryB = 890,
|
||||
ArbitraryC = 19,
|
||||
ArbitraryD = 1
|
||||
};
|
||||
|
||||
const AZStd::unordered_map<JobInfo::IdType, JobInfo::IdType> sequentialToArbitrary =
|
||||
{
|
||||
{TestTargetA, ArbitraryA},
|
||||
{TestTargetB, ArbitraryB},
|
||||
{TestTargetC, ArbitraryC},
|
||||
{TestTargetD, ArbitraryD},
|
||||
};
|
||||
|
||||
const AZStd::unordered_map<JobInfo::IdType, JobInfo::IdType> arbitraryToSequential =
|
||||
{
|
||||
{ArbitraryA, TestTargetA},
|
||||
{ArbitraryB, TestTargetB},
|
||||
{ArbitraryC, TestTargetC},
|
||||
{ArbitraryD, TestTargetD},
|
||||
};
|
||||
|
||||
// Given a test enumerator with no client callback, enumeration timeout or enumerator timeout
|
||||
m_testEnumerator = AZStd::make_unique<TestImpact::TestEnumerator>(
|
||||
AZStd::nullopt,
|
||||
m_maxConcurrency,
|
||||
AZStd::nullopt,
|
||||
AZStd::nullopt);
|
||||
|
||||
// Given an test enumeration job for each test target with no enumeration caching
|
||||
for (size_t jobId = 0; jobId < m_testTargetJobArgs.size(); jobId++)
|
||||
{
|
||||
JobData jobData(m_testTargetPaths[jobId].second, AZStd::nullopt);
|
||||
m_jobInfos.emplace_back(JobInfo({sequentialToArbitrary.at(jobId)}, m_testTargetJobArgs[jobId], AZStd::move(jobData)));
|
||||
}
|
||||
|
||||
// When the test enumeration jobs are executed
|
||||
const auto enumerationJobs = m_testEnumerator->Enumerate(m_jobInfos, CacheExceptionPolicy::Never, JobExceptionPolicy::Never);
|
||||
|
||||
// Expect each job to successfully result in a test enumeration that matches the expected test enumeration data for that test target
|
||||
for (const auto& job : enumerationJobs)
|
||||
{
|
||||
const JobInfo::IdType jobId = arbitraryToSequential.at(job.GetJobInfo().GetId().m_value);
|
||||
ValidateJobExecutedSuccessfully(job);
|
||||
ValidateTestTargetEnumeration(job.GetPayload().value(), m_expectedTestTargetEnumerations[jobId]);
|
||||
}
|
||||
}
|
||||
|
||||
TEST_P(TestEnumeratorFixtureWithConcurrencyParams, EnumerateTestTargetsWithCallback_EnumerationsMatchTestSuitesInTarget)
|
||||
{
|
||||
// Given a client callback function that tracks the number of successful enumerations
|
||||
size_t numSuccesses = 0;
|
||||
const auto jobCallback = [&numSuccesses]([[maybe_unused]] const TestImpact::TestEnumerator::JobInfo& jobInfo, const TestImpact::JobMeta& meta)
|
||||
{
|
||||
if (meta.m_result == TestImpact::JobResult::ExecutedWithSuccess)
|
||||
{
|
||||
numSuccesses++;
|
||||
}
|
||||
};
|
||||
|
||||
// Given a test enumerator with no enumeration timeout or enumerator timeout
|
||||
m_testEnumerator = AZStd::make_unique<TestImpact::TestEnumerator>(
|
||||
jobCallback,
|
||||
m_maxConcurrency,
|
||||
AZStd::nullopt,
|
||||
AZStd::nullopt);
|
||||
|
||||
// Given an test enumeration job for each test target with no enumeration caching
|
||||
for (size_t jobId = 0; jobId < m_testTargetJobArgs.size(); jobId++)
|
||||
{
|
||||
JobData jobData(m_testTargetPaths[jobId].second, AZStd::nullopt);
|
||||
m_jobInfos.emplace_back(JobInfo({jobId}, m_testTargetJobArgs[jobId], AZStd::move(jobData)));
|
||||
}
|
||||
|
||||
// When the test enumeration jobs are executed
|
||||
const auto enumerationJobs = m_testEnumerator->Enumerate(m_jobInfos, CacheExceptionPolicy::Never, JobExceptionPolicy::Never);
|
||||
|
||||
// Expect the number of successful enumerations tracked in the callback to match the number of test targets enumerated
|
||||
EXPECT_EQ(numSuccesses, enumerationJobs.size());
|
||||
|
||||
// Expect each job to successfully result in a test enumeration that matches the expected test enumeration data for that test target
|
||||
for (const auto& job : enumerationJobs)
|
||||
{
|
||||
const JobInfo::IdType jobId = job.GetJobInfo().GetId().m_value;
|
||||
ValidateJobExecutedSuccessfully(job);
|
||||
ValidateTestTargetEnumeration(job.GetPayload().value(), m_expectedTestTargetEnumerations[jobId]);
|
||||
}
|
||||
}
|
||||
|
||||
TEST_P(TestEnumeratorFixtureWithConcurrencyParams, JobRunnerTimeout_InFlightJobsTimeoutAndQueuedJobsUnlaunched)
|
||||
{
|
||||
// Given a test enumerator with no client callback or enumerator timeout and 500ms enumeration timeout
|
||||
m_testEnumerator = AZStd::make_unique<TestImpact::TestEnumerator>(
|
||||
AZStd::nullopt,
|
||||
m_maxConcurrency,
|
||||
AZStd::chrono::milliseconds(500),
|
||||
AZStd::nullopt);
|
||||
|
||||
// Given an test enumeration job for each test target with no enumeration caching where half will sleep indefinitely
|
||||
for (size_t jobId = 0; jobId < m_testTargetJobArgs.size(); jobId++)
|
||||
{
|
||||
JobData jobData(m_testTargetPaths[jobId].second, AZStd::nullopt);
|
||||
const AZStd::string args = (jobId % 2)
|
||||
? AZStd::string::format("%s %s", ValidProcessPath, ConstructTestProcessArgs(jobId, LongSleep).c_str())
|
||||
: m_testTargetJobArgs[jobId];
|
||||
m_jobInfos.emplace_back(JobInfo({jobId}, args, AZStd::move(jobData)));
|
||||
}
|
||||
|
||||
// When the test enumeration jobs are executed
|
||||
const auto enumerationJobs = m_testEnumerator->Enumerate(m_jobInfos, CacheExceptionPolicy::Never, JobExceptionPolicy::Never);
|
||||
|
||||
// Expect half the jobs to successfully result in a test enumeration that matches the expected test enumeration data for that test
|
||||
// target with the other half having timed out
|
||||
for (const auto& job : enumerationJobs)
|
||||
{
|
||||
const JobInfo::IdType jobId = job.GetJobInfo().GetId().m_value;
|
||||
if (jobId % 2)
|
||||
{
|
||||
ValidateJobTimeout(job);
|
||||
}
|
||||
else
|
||||
{
|
||||
ValidateJobExecutedSuccessfully(job);
|
||||
ValidateTestTargetEnumeration(job.GetPayload().value(), m_expectedTestTargetEnumerations[jobId]);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
TEST_F(TestEnumeratorFixture, JobTimeout_InFlightJobTimeoutAndQueuedJobsUnlaunched)
|
||||
{
|
||||
// Given a test enumerator with no client callback or enumeration timeout and a 5 second enumerator timeout
|
||||
m_testEnumerator = AZStd::make_unique<TestImpact::TestEnumerator>(
|
||||
AZStd::nullopt,
|
||||
FourConcurrentProcesses,
|
||||
AZStd::nullopt,
|
||||
AZStd::chrono::milliseconds(5000));
|
||||
|
||||
// Given an test enumeration job for each test target with no enumeration caching where half will sleep indefinitely
|
||||
for (size_t jobId = 0; jobId < m_testTargetJobArgs.size(); jobId++)
|
||||
{
|
||||
JobData jobData(m_testTargetPaths[jobId].second, AZStd::nullopt);
|
||||
const AZStd::string args = (jobId % 2)
|
||||
? AZStd::string::format("%s %s", ValidProcessPath, ConstructTestProcessArgs(jobId, LongSleep).c_str())
|
||||
: m_testTargetJobArgs[jobId];
|
||||
m_jobInfos.emplace_back(JobInfo({jobId}, args, AZStd::move(jobData)));
|
||||
}
|
||||
|
||||
// When the test enumeration jobs are executed
|
||||
const auto enumerationJobs = m_testEnumerator->Enumerate(m_jobInfos, CacheExceptionPolicy::Never, JobExceptionPolicy::Never);
|
||||
|
||||
// Expect half the jobs to successfully result in a test enumeration that matches the expected test enumeration data for that test
|
||||
// target with the other half having timed out
|
||||
for (const auto& job : enumerationJobs)
|
||||
{
|
||||
const JobInfo::IdType jobId = job.GetJobInfo().GetId().m_value;
|
||||
if (jobId % 2)
|
||||
{
|
||||
ValidateJobTimeout(job);
|
||||
}
|
||||
else
|
||||
{
|
||||
ValidateJobExecutedSuccessfully(job);
|
||||
ValidateTestTargetEnumeration(job.GetPayload().value(), m_expectedTestTargetEnumerations[jobId]);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
INSTANTIATE_TEST_CASE_P(
|
||||
,
|
||||
TestEnumeratorFixtureWithConcurrencyAndJobExceptionParams,
|
||||
::testing::Combine(
|
||||
::testing::ValuesIn(MaxConcurrentEnumerations),
|
||||
::testing::ValuesIn(JobExceptionPolicies))
|
||||
);
|
||||
|
||||
INSTANTIATE_TEST_CASE_P(
|
||||
,
|
||||
TestEnumeratorFixtureWithConcurrencyAndCacheExceptionParams,
|
||||
::testing::Combine(
|
||||
::testing::ValuesIn(MaxConcurrentEnumerations),
|
||||
::testing::ValuesIn(CacheExceptionPolicies))
|
||||
);
|
||||
|
||||
INSTANTIATE_TEST_CASE_P(
|
||||
,
|
||||
TestEnumeratorFixtureWithConcurrencyParams,
|
||||
::testing::ValuesIn(MaxConcurrentEnumerations)
|
||||
);
|
||||
} // namespace UnitTest
|
||||
-99
@@ -1,99 +0,0 @@
|
||||
/*
|
||||
* All or portions of this file Copyright (c) Amazon.com, Inc. or its affiliates or
|
||||
* its licensors.
|
||||
*
|
||||
* For complete copyright and license terms please see the LICENSE at the root of this
|
||||
* distribution (the "License"). All use of this software is governed by the License,
|
||||
* or, if provided, by the license below or the license accompanying this file. Do not
|
||||
* remove or modify any license notices. This file is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
*
|
||||
*/
|
||||
|
||||
#include <TestImpactTestUtils.h>
|
||||
|
||||
#include <Test/Enumeration/TestImpactTestEnumeration.h>
|
||||
#include <Test/Enumeration/TestImpactTestEnumerationSerializer.h>
|
||||
|
||||
#include <AzCore/UnitTest/TestTypes.h>
|
||||
#include <AzTest/AzTest.h>
|
||||
|
||||
namespace UnitTest
|
||||
{
|
||||
const TestImpact::TestEnumeration EmptyTestEnum(AZStd::vector<TestImpact::TestEnumerationSuite>{});
|
||||
|
||||
TEST(TestEnumerationSerializer, EmptyTestEnumerationSuites_ExpectTemptyTestEnumeration)
|
||||
{
|
||||
// Given an empty set of test enumeration suites
|
||||
// When the test enumeration is serialized and deserialized back again
|
||||
const AZStd::string serializedString = TestImpact::SerializeTestEnumeration(EmptyTestEnum);
|
||||
const TestImpact::TestEnumeration actualEnumeration = TestImpact::DeserializeTestEnumeration(serializedString);
|
||||
|
||||
// Expect the actual test enumeration to match the expected test enumeration
|
||||
EXPECT_TRUE(actualEnumeration == EmptyTestEnum);
|
||||
}
|
||||
|
||||
TEST(TestEnumerationSerializer, SerializeAndDeserializeSuitesForTestTargetA_ActualSuiteDataMatchesExpectedSuiteData)
|
||||
{
|
||||
// Given the test of test enumeration suites for Test target A
|
||||
const TestImpact::TestEnumeration expectedEnumeration = TestImpact::TestEnumeration(GetTestTargetATestEnumerationSuites());
|
||||
|
||||
// When the test enumeration is serialized and deserialized back again
|
||||
const AZStd::string serializedString = TestImpact::SerializeTestEnumeration(expectedEnumeration);
|
||||
const TestImpact::TestEnumeration actualEnumeration = TestImpact::DeserializeTestEnumeration(serializedString);
|
||||
|
||||
// Expect the actual test enumeration to match the expected test enumeration
|
||||
EXPECT_TRUE(actualEnumeration == expectedEnumeration);
|
||||
|
||||
// Do not expect the actual test enumeration to match the empty test enumeration
|
||||
EXPECT_FALSE(actualEnumeration == EmptyTestEnum);
|
||||
}
|
||||
|
||||
TEST(TestEnumerationSerializer, SerializeAndDeserializeSuitesForTestTargetB_ActualSuiteDataMatchesExpectedSuiteData)
|
||||
{
|
||||
// Given the test of test enumeration suites for Test target B
|
||||
const TestImpact::TestEnumeration expectedEnumeration = TestImpact::TestEnumeration(GetTestTargetBTestEnumerationSuites());
|
||||
|
||||
// When the test enumeration is serialized and deserialized back again
|
||||
const AZStd::string serializedString = TestImpact::SerializeTestEnumeration(expectedEnumeration);
|
||||
const TestImpact::TestEnumeration actualEnumeration = TestImpact::DeserializeTestEnumeration(serializedString);
|
||||
|
||||
// Expect the actual test enumeration to match the expected test enumeration
|
||||
EXPECT_TRUE(actualEnumeration == expectedEnumeration);
|
||||
|
||||
// Do not expect the actual test enumeration to match the empty test enumeration
|
||||
EXPECT_FALSE(actualEnumeration == EmptyTestEnum);
|
||||
}
|
||||
|
||||
TEST(TestEnumerationSerializer, SerializeAndDeserializeSuitesForTestTargetC_ActualSuiteDataMatchesExpectedSuiteData)
|
||||
{
|
||||
// Given the test of test enumeration suites for Test target B
|
||||
const TestImpact::TestEnumeration expectedEnumeration = TestImpact::TestEnumeration(GetTestTargetCTestEnumerationSuites());
|
||||
|
||||
// When the test enumeration is serialized and deserialized back again
|
||||
const AZStd::string serializedString = TestImpact::SerializeTestEnumeration(expectedEnumeration);
|
||||
const TestImpact::TestEnumeration actualEnumeration = TestImpact::DeserializeTestEnumeration(serializedString);
|
||||
|
||||
// Expect the actual test enumeration to match the expected test enumeration
|
||||
EXPECT_TRUE(actualEnumeration == expectedEnumeration);
|
||||
|
||||
// Do not expect the actual test enumeration to match the empty test enumeration
|
||||
EXPECT_FALSE(actualEnumeration == EmptyTestEnum);
|
||||
}
|
||||
|
||||
TEST(TestEnumerationSerializer, SerializeAndDeserializeSuitesForTestTargetD_ActualSuiteDataMatchesExpectedSuiteData)
|
||||
{
|
||||
// Given the test of test enumeration suites for Test target B
|
||||
const TestImpact::TestEnumeration expectedEnumeration = TestImpact::TestEnumeration(GetTestTargetDTestEnumerationSuites());
|
||||
|
||||
// When the test enumeration is serialized and deserialized back again
|
||||
const AZStd::string serializedString = TestImpact::SerializeTestEnumeration(expectedEnumeration);
|
||||
const TestImpact::TestEnumeration actualEnumeration = TestImpact::DeserializeTestEnumeration(serializedString);
|
||||
|
||||
// Expect the actual test enumeration to match the expected test enumeration
|
||||
EXPECT_TRUE(actualEnumeration == expectedEnumeration);
|
||||
|
||||
// Do not expect the actual test enumeration to match the empty test enumeration
|
||||
EXPECT_FALSE(actualEnumeration == EmptyTestEnum);
|
||||
}
|
||||
} // namespace UnitTest
|
||||
-99
@@ -1,99 +0,0 @@
|
||||
/*
|
||||
* All or portions of this file Copyright (c) Amazon.com, Inc. or its affiliates or
|
||||
* its licensors.
|
||||
*
|
||||
* For complete copyright and license terms please see the LICENSE at the root of this
|
||||
* distribution (the "License"). All use of this software is governed by the License,
|
||||
* or, if provided, by the license below or the license accompanying this file. Do not
|
||||
* remove or modify any license notices. This file is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
*
|
||||
*/
|
||||
|
||||
#include <TestImpactTestUtils.h>
|
||||
|
||||
#include <Test/Run/TestImpactTestRun.h>
|
||||
#include <Test/Run/TestImpactTestRunSerializer.h>
|
||||
|
||||
#include <AzCore/UnitTest/TestTypes.h>
|
||||
#include <AzTest/AzTest.h>
|
||||
|
||||
namespace UnitTest
|
||||
{
|
||||
const TestImpact::TestRun EmptyTestRun(AZStd::vector<TestImpact::TestRunSuite>{}, AZStd::chrono::milliseconds{0});
|
||||
|
||||
TEST(TestRunSerializer, EmptyTestRunSuites_ExpectTemptyTestRun)
|
||||
{
|
||||
// Given an empty set of test run suites
|
||||
// When the test run is serialized and deserialized back again
|
||||
const auto serializedString = TestImpact::SerializeTestRun(EmptyTestRun);
|
||||
const auto actualRun = TestImpact::DeserializeTestRun(serializedString);
|
||||
|
||||
// Expect the actual test run to match the expected test run
|
||||
EXPECT_TRUE(actualRun == EmptyTestRun);
|
||||
}
|
||||
|
||||
TEST(TestRunSerializer, SerializeAndDeserializeSuitesForTestTargetA_ActualSuiteDataMatchesExpectedSuiteData)
|
||||
{
|
||||
// Given the test of test run suites for Test target A
|
||||
const auto expectedRun = TestImpact::TestRun(GetTestTargetATestRunSuites(), AZStd::chrono::milliseconds{500});
|
||||
|
||||
// When the test run is serialized and deserialized back again
|
||||
const auto serializedString = TestImpact::SerializeTestRun(expectedRun);
|
||||
const auto actualRun = TestImpact::DeserializeTestRun(serializedString);
|
||||
|
||||
// Expect the actual test run to match the expected test run
|
||||
EXPECT_TRUE(actualRun == expectedRun);
|
||||
|
||||
// Do not expect the actual test run to match the empty test run
|
||||
EXPECT_FALSE(actualRun == EmptyTestRun);
|
||||
}
|
||||
|
||||
TEST(TestRunSerializer, SerializeAndDeserializeSuitesForTestTargetB_ActualSuiteDataMatchesExpectedSuiteData)
|
||||
{
|
||||
// Given the test of test run suites for Test target B
|
||||
const auto expectedRun = TestImpact::TestRun(GetTestTargetBTestRunSuites(), AZStd::chrono::milliseconds{500});
|
||||
|
||||
// When the test run is serialized and deserialized back again
|
||||
const auto serializedString = TestImpact::SerializeTestRun(expectedRun);
|
||||
const auto actualRun = TestImpact::DeserializeTestRun(serializedString);
|
||||
|
||||
// Expect the actual test run to match the expected test run
|
||||
EXPECT_TRUE(actualRun == expectedRun);
|
||||
|
||||
// Do not expect the actual test run to match the empty test run
|
||||
EXPECT_FALSE(actualRun == EmptyTestRun);
|
||||
}
|
||||
|
||||
TEST(TestRunSerializer, SerializeAndDeserializeSuitesForTestTargetC_ActualSuiteDataMatchesExpectedSuiteData)
|
||||
{
|
||||
// Given the test of test run suites for Test target B
|
||||
const auto expectedRun = TestImpact::TestRun(GetTestTargetCTestRunSuites(), AZStd::chrono::milliseconds{500});
|
||||
|
||||
// When the test run is serialized and deserialized back again
|
||||
const auto serializedString = TestImpact::SerializeTestRun(expectedRun);
|
||||
const auto actualRun = TestImpact::DeserializeTestRun(serializedString);
|
||||
|
||||
// Expect the actual test run to match the expected test run
|
||||
EXPECT_TRUE(actualRun == expectedRun);
|
||||
|
||||
// Do not expect the actual test run to match the empty test run
|
||||
EXPECT_FALSE(actualRun == EmptyTestRun);
|
||||
}
|
||||
|
||||
TEST(TestRunSerializer, SerializeAndDeserializeSuitesForTestTargetD_ActualSuiteDataMatchesExpectedSuiteData)
|
||||
{
|
||||
// Given the test of test run suites for Test target B
|
||||
const auto expectedRun = TestImpact::TestRun(GetTestTargetDTestRunSuites(), AZStd::chrono::milliseconds{500});
|
||||
|
||||
// When the test run is serialized and deserialized back again
|
||||
const auto serializedString = TestImpact::SerializeTestRun(expectedRun);
|
||||
const auto actualRun = TestImpact::DeserializeTestRun(serializedString);
|
||||
|
||||
// Expect the actual test run to match the expected test run
|
||||
EXPECT_TRUE(actualRun == expectedRun);
|
||||
|
||||
// Do not expect the actual test run to match the empty test run
|
||||
EXPECT_FALSE(actualRun == EmptyTestRun);
|
||||
}
|
||||
} // namespace UnitTest
|
||||
@@ -1,516 +0,0 @@
|
||||
/*
|
||||
* All or portions of this file Copyright (c) Amazon.com, Inc. or its affiliates or
|
||||
* its licensors.
|
||||
*
|
||||
* For complete copyright and license terms please see the LICENSE at the root of this
|
||||
* distribution (the "License"). All use of this software is governed by the License,
|
||||
* or, if provided, by the license below or the license accompanying this file. Do not
|
||||
* remove or modify any license notices. This file is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
*
|
||||
*/
|
||||
|
||||
#include <TestImpactTestJobRunnerCommon.h>
|
||||
#include <TestImpactTestUtils.h>
|
||||
|
||||
#include <Artifact/TestImpactArtifactException.h>
|
||||
#include <Test/Run/TestImpactTestRunException.h>
|
||||
#include <Test/Run/TestImpactTestRunner.h>
|
||||
|
||||
#include <AzCore/IO/Path/Path.h>
|
||||
#include <AzCore/UnitTest/TestTypes.h>
|
||||
#include <AzCore/std/containers/array.h>
|
||||
#include <AzCore/std/containers/vector.h>
|
||||
#include <AzCore/std/optional.h>
|
||||
#include <AzCore/std/smart_ptr/unique_ptr.h>
|
||||
#include <AzCore/std/string/string.h>
|
||||
#include <AzTest/AzTest.h>
|
||||
|
||||
namespace UnitTest
|
||||
{
|
||||
using JobExceptionPolicy = TestImpact::TestRunner::JobExceptionPolicy;
|
||||
|
||||
AZStd::string GetRunCommandForTarget(AZStd::pair<AZ::IO::Path, AZ::IO::Path> testTarget)
|
||||
{
|
||||
return AZStd::string::format(
|
||||
"%s %s AzRunUnitTests --gtest_output=xml:%s", LY_TEST_IMPACT_AZ_TESTRUNNER_BIN, testTarget.first.c_str(),
|
||||
testTarget.second.c_str());
|
||||
}
|
||||
|
||||
class TestRunnerFixture
|
||||
: public AllocatorsTestFixture
|
||||
{
|
||||
public:
|
||||
void SetUp() override;
|
||||
|
||||
protected:
|
||||
using JobInfo = TestImpact::TestRunner::JobInfo;
|
||||
using JobData = TestImpact::TestRunner::JobData;
|
||||
|
||||
AZStd::vector<JobInfo> m_jobInfos;
|
||||
AZStd::unique_ptr<TestImpact::TestRunner> m_testRunner;
|
||||
AZStd::vector<AZStd::string> m_testTargetJobArgs;
|
||||
AZStd::vector<AZStd::pair<AZ::IO::Path, AZ::IO::Path>> m_testTargetPaths;
|
||||
AZStd::vector<TestImpact::TestRun> m_expectedTestTargetRuns;
|
||||
AZStd::vector<TestImpact::TestRunResult> m_expectedTestTargetResult;
|
||||
size_t m_maxConcurrency = 0;
|
||||
};
|
||||
|
||||
void TestRunnerFixture::SetUp()
|
||||
{
|
||||
UnitTest::AllocatorsTestFixture::SetUp();
|
||||
|
||||
DeleteFiles(LY_TEST_IMPACT_TEST_TARGET_RESULTS_DIR, "*.xml");
|
||||
|
||||
// first: path to test target bin
|
||||
// second: path to test target gtest results file in XML format
|
||||
const AZStd::string runPath = AZStd::string(LY_TEST_IMPACT_TEST_TARGET_RESULTS_DIR) + "/%s.Run.xml";
|
||||
m_testTargetPaths.emplace_back(
|
||||
LY_TEST_IMPACT_TEST_TARGET_A_BIN, AZStd::string::format(runPath.c_str(), LY_TEST_IMPACT_TEST_TARGET_A_BASE_NAME));
|
||||
m_testTargetPaths.emplace_back(
|
||||
LY_TEST_IMPACT_TEST_TARGET_B_BIN, AZStd::string::format(runPath.c_str(), LY_TEST_IMPACT_TEST_TARGET_B_BASE_NAME));
|
||||
m_testTargetPaths.emplace_back(
|
||||
LY_TEST_IMPACT_TEST_TARGET_C_BIN, AZStd::string::format(runPath.c_str(), LY_TEST_IMPACT_TEST_TARGET_C_BASE_NAME));
|
||||
m_testTargetPaths.emplace_back(
|
||||
LY_TEST_IMPACT_TEST_TARGET_D_BIN, AZStd::string::format(runPath.c_str(), LY_TEST_IMPACT_TEST_TARGET_D_BASE_NAME));
|
||||
|
||||
m_expectedTestTargetRuns.emplace_back(GetTestTargetATestRunSuites(), AZStd::chrono::milliseconds{500});
|
||||
m_expectedTestTargetRuns.emplace_back(GetTestTargetBTestRunSuites(), AZStd::chrono::milliseconds{500});
|
||||
m_expectedTestTargetRuns.emplace_back(GetTestTargetCTestRunSuites(), AZStd::chrono::milliseconds{500});
|
||||
m_expectedTestTargetRuns.emplace_back(GetTestTargetDTestRunSuites(), AZStd::chrono::milliseconds{500});
|
||||
|
||||
m_expectedTestTargetResult.emplace_back(TestImpact::TestRunResult::Failed);
|
||||
m_expectedTestTargetResult.emplace_back(TestImpact::TestRunResult::Passed);
|
||||
m_expectedTestTargetResult.emplace_back(TestImpact::TestRunResult::Passed);
|
||||
m_expectedTestTargetResult.emplace_back(TestImpact::TestRunResult::Passed);
|
||||
|
||||
for (const auto& testTarget : m_testTargetPaths)
|
||||
{
|
||||
m_testTargetJobArgs.emplace_back(GetRunCommandForTarget(testTarget));
|
||||
}
|
||||
}
|
||||
|
||||
// Fixture parameterized for different max number of concurrent jobs
|
||||
class TestRunnerFixtureWithConcurrencyParams
|
||||
: public TestRunnerFixture
|
||||
, public ::testing::WithParamInterface<size_t>
|
||||
{
|
||||
public:
|
||||
void SetUp() override
|
||||
{
|
||||
TestRunnerFixture::SetUp();
|
||||
m_maxConcurrency = GetParam();
|
||||
}
|
||||
};
|
||||
|
||||
using ConcurrencyAndJobExceptionPermutation = AZStd::tuple
|
||||
<
|
||||
size_t, // Max number of concurrent processes
|
||||
JobExceptionPolicy // Test job exception policy
|
||||
>;
|
||||
|
||||
// Fixture parameterized for different max number of concurrent jobs and different job exception policies
|
||||
class TestRunnerFixtureWithConcurrencyAndJobExceptionParams
|
||||
: public TestRunnerFixture
|
||||
, public ::testing::WithParamInterface<ConcurrencyAndJobExceptionPermutation>
|
||||
{
|
||||
public:
|
||||
void SetUp() override
|
||||
{
|
||||
TestRunnerFixture::SetUp();
|
||||
const auto& [maxConcurrency, jobExceptionPolicy] = GetParam();
|
||||
m_maxConcurrency = maxConcurrency;
|
||||
m_jobExceptionPolicy = jobExceptionPolicy;
|
||||
}
|
||||
|
||||
protected:
|
||||
JobExceptionPolicy m_jobExceptionPolicy = JobExceptionPolicy::Never;
|
||||
};
|
||||
|
||||
class TestRunnerFixtureWithConcurrencyAndFailedToLaunchExceptionParams
|
||||
: public TestRunnerFixtureWithConcurrencyAndJobExceptionParams
|
||||
{
|
||||
};
|
||||
|
||||
class TestRunnerFixtureWithConcurrencyAndExecutedWithFailureExceptionParams
|
||||
: public TestRunnerFixtureWithConcurrencyAndJobExceptionParams
|
||||
{
|
||||
};
|
||||
|
||||
namespace
|
||||
{
|
||||
AZStd::array<size_t, 4> MaxConcurrentRuns = {1, 2, 3, 4};
|
||||
|
||||
AZStd::array<JobExceptionPolicy, 2> FailedToLaunchExceptionPolicies = {
|
||||
JobExceptionPolicy::Never, JobExceptionPolicy::OnFailedToExecute};
|
||||
|
||||
AZStd::array<JobExceptionPolicy, 2> ExecutedWithFailureExceptionPolicies = {
|
||||
JobExceptionPolicy::Never, JobExceptionPolicy::OnExecutedWithFailure};
|
||||
} // namespace
|
||||
|
||||
TEST_P(
|
||||
TestRunnerFixtureWithConcurrencyAndFailedToLaunchExceptionParams,
|
||||
InvalidCommandArgument_ExpectJobResulFailedToExecuteeOrTestJobException)
|
||||
{
|
||||
// Given a test runner with no client callback or run timeout or runner timeout
|
||||
m_testRunner = AZStd::make_unique<TestImpact::TestRunner>(AZStd::nullopt, m_maxConcurrency, AZStd::nullopt, AZStd::nullopt);
|
||||
|
||||
// Given a mixture of test run jobs with valid and invalid command arguments
|
||||
for (size_t jobId = 0; jobId < m_testTargetJobArgs.size(); jobId++)
|
||||
{
|
||||
const AZStd::string args = (jobId % 2) ? InvalidProcessPath : m_testTargetJobArgs[jobId];
|
||||
JobData jobData(m_testTargetPaths[jobId].second);
|
||||
m_jobInfos.emplace_back(JobInfo({jobId}, args, AZStd::move(jobData)));
|
||||
}
|
||||
|
||||
try
|
||||
{
|
||||
// When the test run jobs are executed with different exception policies
|
||||
const auto runnerJobs = m_testRunner->RunTests(m_jobInfos, m_jobExceptionPolicy);
|
||||
|
||||
// Expect this statement to be reachable only if no exception policy for launch failures
|
||||
EXPECT_FALSE(::IsFlagSet(m_jobExceptionPolicy, JobExceptionPolicy::OnFailedToExecute));
|
||||
|
||||
for (const auto& job : runnerJobs)
|
||||
{
|
||||
const JobInfo::IdType jobId = job.GetJobInfo().GetId().m_value;
|
||||
if (jobId % 2)
|
||||
{
|
||||
// Expect invalid jobs have a job result of FailedToExecute
|
||||
ValidateJobFailedToExecute(job);
|
||||
}
|
||||
else
|
||||
{
|
||||
// Expect the valid jobs to successfully result in a test run that matches the expected test run data
|
||||
ValidateTestRunCompleted(job, m_expectedTestTargetResult[jobId]);
|
||||
ValidateTestTargetRun(job.GetPayload().value(), m_expectedTestTargetRuns[jobId]);
|
||||
}
|
||||
}
|
||||
}
|
||||
catch ([[maybe_unused]] const TestImpact::TestJobException& e)
|
||||
{
|
||||
// Expect this statement to be reachable only if there is an exception policy for launch failures
|
||||
EXPECT_TRUE(::IsFlagSet(m_jobExceptionPolicy, JobExceptionPolicy::OnFailedToExecute));
|
||||
}
|
||||
catch (...)
|
||||
{
|
||||
// Do not expect any other exceptions
|
||||
FAIL();
|
||||
}
|
||||
}
|
||||
|
||||
TEST_P(
|
||||
TestRunnerFixtureWithConcurrencyAndExecutedWithFailureExceptionParams,
|
||||
ErroneousReturnCode_ExpectJobResultExecutedWithFailureOrTestJobException)
|
||||
{
|
||||
// Given a test runner with no client callback or run timeout or runner timeout
|
||||
m_testRunner = AZStd::make_unique<TestImpact::TestRunner>(AZStd::nullopt, m_maxConcurrency, AZStd::nullopt, AZStd::nullopt);
|
||||
|
||||
// Given a mixture of test run jobs that execute and return either successfully or with failure
|
||||
for (size_t jobId = 0; jobId < m_testTargetJobArgs.size(); jobId++)
|
||||
{
|
||||
JobData jobData(m_testTargetPaths[jobId].second);
|
||||
m_jobInfos.emplace_back(JobInfo({jobId}, m_testTargetJobArgs[jobId], AZStd::move(jobData)));
|
||||
}
|
||||
|
||||
try
|
||||
{
|
||||
// When the test run jobs are executed with different exception policies
|
||||
const auto runnerJobs = m_testRunner->RunTests(m_jobInfos, m_jobExceptionPolicy);
|
||||
|
||||
// Expect this statement to be reachable only if no exception policy for jobs that return with error
|
||||
EXPECT_FALSE(::IsFlagSet(m_jobExceptionPolicy, JobExceptionPolicy::OnExecutedWithFailure));
|
||||
|
||||
for (const auto& job : runnerJobs)
|
||||
{
|
||||
const JobInfo::IdType jobId = job.GetJobInfo().GetId().m_value;
|
||||
|
||||
// Expect the valid jobs to successfully result in a test run that matches the expected test run data
|
||||
ValidateTestRunCompleted(job, m_expectedTestTargetResult[jobId]);
|
||||
ValidateTestTargetRun(job.GetPayload().value(), m_expectedTestTargetRuns[jobId]);
|
||||
}
|
||||
}
|
||||
catch ([[maybe_unused]] const TestImpact::TestJobException& e)
|
||||
{
|
||||
// Expect this statement to be reachable only if there is an exception policy for jobs that return with error
|
||||
EXPECT_TRUE(::IsFlagSet(m_jobExceptionPolicy, JobExceptionPolicy::OnExecutedWithFailure));
|
||||
}
|
||||
catch ([[maybe_unused]] const TestImpact::Exception& e)
|
||||
{
|
||||
FAIL();
|
||||
}
|
||||
catch (...)
|
||||
{
|
||||
// Do not expect any other exceptions
|
||||
FAIL();
|
||||
}
|
||||
}
|
||||
|
||||
TEST_F(TestRunnerFixture, EmptyArtifact_ExpectTestRunnerException)
|
||||
{
|
||||
// Given a test runner with no client callback, concurrency, run timeout or runner timeout
|
||||
m_testRunner = AZStd::make_unique<TestImpact::TestRunner>(AZStd::nullopt, OneConcurrentProcess, AZStd::nullopt, AZStd::nullopt);
|
||||
|
||||
// Given an test runner job that will return successfully but with an empty artifact string
|
||||
m_jobInfos.emplace_back(JobInfo({0}, m_testTargetJobArgs[0], JobData("")));
|
||||
|
||||
try
|
||||
{
|
||||
// When the test runner job is executed
|
||||
const auto runnerJobs = m_testRunner->RunTests(m_jobInfos, JobExceptionPolicy::Never);
|
||||
|
||||
// Do not expect this statement to be reachable
|
||||
FAIL();
|
||||
}
|
||||
catch ([[maybe_unused]] const TestImpact::TestRunException& e)
|
||||
{
|
||||
// Expect an runner exception
|
||||
SUCCEED();
|
||||
}
|
||||
catch (...)
|
||||
{
|
||||
// Do not expect any other exceptions
|
||||
FAIL();
|
||||
}
|
||||
}
|
||||
|
||||
TEST_F(TestRunnerFixture, InvalidRunArtifact_ExpectArtifactException)
|
||||
{
|
||||
// Given a test run artifact with invalid contents
|
||||
WriteTextToFile("There is nothing valid here", m_testTargetPaths[TestTargetA].second);
|
||||
|
||||
// Given a job command that will write the test run artifact to a different location that what we will read from
|
||||
auto invalidRunArtifact = m_testTargetPaths[TestTargetA];
|
||||
invalidRunArtifact.second /= ".xml";
|
||||
const AZStd::string args = GetRunCommandForTarget(invalidRunArtifact);
|
||||
|
||||
// Given a test runner with no client callback, concurrency, run timeout or runner timeout
|
||||
m_testRunner = AZStd::make_unique<TestImpact::TestRunner>(AZStd::nullopt, OneConcurrentProcess, AZStd::nullopt, AZStd::nullopt);
|
||||
|
||||
// Given an test runner job that will return successfully but not produce an artifact
|
||||
JobData jobData(m_testTargetPaths[TestTargetA].second);
|
||||
m_jobInfos.emplace_back(JobInfo({TestTargetA}, args, AZStd::move(jobData)));
|
||||
|
||||
try
|
||||
{
|
||||
// When the test runner job is executed
|
||||
const auto runnerJobs = m_testRunner->RunTests(m_jobInfos, JobExceptionPolicy::Never);
|
||||
|
||||
// Do not expect this statement to be reachable
|
||||
FAIL();
|
||||
}
|
||||
catch ([[maybe_unused]] const TestImpact::ArtifactException& e)
|
||||
{
|
||||
// Expect an runner exception
|
||||
SUCCEED();
|
||||
}
|
||||
catch (const TestImpact::Exception& e)
|
||||
{
|
||||
std::cout << e.what();
|
||||
FAIL();
|
||||
}
|
||||
catch (...)
|
||||
{
|
||||
// Do not expect any other exceptions
|
||||
FAIL();
|
||||
}
|
||||
}
|
||||
|
||||
TEST_P(TestRunnerFixtureWithConcurrencyParams, RunTestTargets_RunsMatchTestSuitesInTarget)
|
||||
{
|
||||
// Given a test runner with no client callback, runner timeout or runner timeout
|
||||
m_testRunner = AZStd::make_unique<TestImpact::TestRunner>(AZStd::nullopt, m_maxConcurrency, AZStd::nullopt, AZStd::nullopt);
|
||||
|
||||
// Given an test runner job for each test target with no runner caching
|
||||
for (size_t jobId = 0; jobId < m_testTargetJobArgs.size(); jobId++)
|
||||
{
|
||||
JobData jobData(m_testTargetPaths[jobId].second);
|
||||
m_jobInfos.emplace_back(JobInfo({jobId}, m_testTargetJobArgs[jobId], AZStd::move(jobData)));
|
||||
}
|
||||
|
||||
// When the test runner jobs are executed
|
||||
const auto runnerJobs = m_testRunner->RunTests(m_jobInfos, JobExceptionPolicy::Never);
|
||||
|
||||
// Expect each job to successfully result in a test runner that matches the expected test runner data for that test target
|
||||
for (const auto& job : runnerJobs)
|
||||
{
|
||||
const JobInfo::IdType jobId = job.GetJobInfo().GetId().m_value;
|
||||
ValidateTestRunCompleted(job, m_expectedTestTargetResult[jobId]);
|
||||
ValidateTestTargetRun(job.GetPayload().value(), m_expectedTestTargetRuns[jobId]);
|
||||
}
|
||||
}
|
||||
|
||||
TEST_P(TestRunnerFixtureWithConcurrencyParams, RunTestTargetsWithArbitraryJobIds_RunsMatchTestSuitesInTarget)
|
||||
{
|
||||
// Given a set of arbitrary job ids to be used for the test target jobs
|
||||
enum
|
||||
{
|
||||
ArbitraryA = 36,
|
||||
ArbitraryB = 890,
|
||||
ArbitraryC = 19,
|
||||
ArbitraryD = 1
|
||||
};
|
||||
|
||||
const AZStd::unordered_map<JobInfo::IdType, JobInfo::IdType> sequentialToArbitrary =
|
||||
{
|
||||
{TestTargetA, ArbitraryA},
|
||||
{TestTargetB, ArbitraryB},
|
||||
{TestTargetC, ArbitraryC},
|
||||
{TestTargetD, ArbitraryD},
|
||||
};
|
||||
|
||||
const AZStd::unordered_map<JobInfo::IdType, JobInfo::IdType> arbitraryToSequential =
|
||||
{
|
||||
{ArbitraryA, TestTargetA},
|
||||
{ArbitraryB, TestTargetB},
|
||||
{ArbitraryC, TestTargetC},
|
||||
{ArbitraryD, TestTargetD},
|
||||
};
|
||||
|
||||
// Given a test runner with no client callback, run timeout or runner timeout
|
||||
m_testRunner = AZStd::make_unique<TestImpact::TestRunner>(AZStd::nullopt, m_maxConcurrency, AZStd::nullopt, AZStd::nullopt);
|
||||
|
||||
// Given an test run job for each test target
|
||||
for (size_t jobId = 0; jobId < m_testTargetJobArgs.size(); jobId++)
|
||||
{
|
||||
JobData jobData(m_testTargetPaths[jobId].second);
|
||||
m_jobInfos.emplace_back(JobInfo({sequentialToArbitrary.at(jobId)}, m_testTargetJobArgs[jobId], AZStd::move(jobData)));
|
||||
}
|
||||
|
||||
// When the test run jobs are executed
|
||||
const auto runnerJobs = m_testRunner->RunTests(m_jobInfos, JobExceptionPolicy::Never);
|
||||
|
||||
// Expect each job to successfully result in a test run that matches the expected test run data for that test target
|
||||
for (const auto& job : runnerJobs)
|
||||
{
|
||||
const JobInfo::IdType jobId = arbitraryToSequential.at(job.GetJobInfo().GetId().m_value);
|
||||
ValidateTestRunCompleted(job, m_expectedTestTargetResult[jobId]);
|
||||
ValidateTestTargetRun(job.GetPayload().value(), m_expectedTestTargetRuns[jobId]);
|
||||
}
|
||||
}
|
||||
|
||||
TEST_P(TestRunnerFixtureWithConcurrencyParams, RunTestTargetsWithCallback_RunsMatchTestSuitesInTarget)
|
||||
{
|
||||
// Given a client callback function that tracks the number of successful runs
|
||||
size_t numSuccesses = 0;
|
||||
const auto jobCallback =
|
||||
[&numSuccesses]([[maybe_unused]] const TestImpact::TestRunner::JobInfo& jobInfo, const TestImpact::JobMeta& meta) {
|
||||
if (meta.m_result == TestImpact::JobResult::ExecutedWithSuccess)
|
||||
{
|
||||
numSuccesses++;
|
||||
}
|
||||
};
|
||||
|
||||
// Given a test runner with no run timeout or runner timeout
|
||||
m_testRunner = AZStd::make_unique<TestImpact::TestRunner>(jobCallback, m_maxConcurrency, AZStd::nullopt, AZStd::nullopt);
|
||||
|
||||
// Given an test run job for each test target
|
||||
for (size_t jobId = 0; jobId < m_testTargetJobArgs.size(); jobId++)
|
||||
{
|
||||
JobData jobData(m_testTargetPaths[jobId].second);
|
||||
m_jobInfos.emplace_back(JobInfo({jobId}, m_testTargetJobArgs[jobId], AZStd::move(jobData)));
|
||||
}
|
||||
|
||||
// When the test run jobs are executed
|
||||
const auto runnerJobs = m_testRunner->RunTests(m_jobInfos, JobExceptionPolicy::Never);
|
||||
|
||||
// Expect the number of successful runs tracked in the callback to match the number of test targets run with no failures
|
||||
EXPECT_EQ(
|
||||
numSuccesses,
|
||||
AZStd::count_if(m_expectedTestTargetResult.begin(), m_expectedTestTargetResult.end(), [](TestImpact::TestRunResult result) {
|
||||
return result == TestImpact::TestRunResult::Passed;
|
||||
}));
|
||||
|
||||
// Expect each job to successfully result in a test run that matches the expected test run data for that test target
|
||||
for (const auto& job : runnerJobs)
|
||||
{
|
||||
const JobInfo::IdType jobId = job.GetJobInfo().GetId().m_value;
|
||||
ValidateTestRunCompleted(job, m_expectedTestTargetResult[jobId]);
|
||||
ValidateTestTargetRun(job.GetPayload().value(), m_expectedTestTargetRuns[jobId]);
|
||||
}
|
||||
}
|
||||
|
||||
TEST_P(TestRunnerFixtureWithConcurrencyParams, JobRunnerTimeout_InFlightJobsTimeoutAndQueuedJobsUnlaunched)
|
||||
{
|
||||
// Given a test runner with no client callback or runner timeout and 500ms run timeout
|
||||
m_testRunner =
|
||||
AZStd::make_unique<TestImpact::TestRunner>(AZStd::nullopt, m_maxConcurrency, AZStd::chrono::milliseconds(500), AZStd::nullopt);
|
||||
|
||||
// Given an test run job for each test target where half will sleep indefinitely
|
||||
for (size_t jobId = 0; jobId < m_testTargetJobArgs.size(); jobId++)
|
||||
{
|
||||
JobData jobData(m_testTargetPaths[jobId].second);
|
||||
const AZStd::string args = (jobId % 2)
|
||||
? AZStd::string::format("%s %s", ValidProcessPath, ConstructTestProcessArgs(jobId, LongSleep).c_str())
|
||||
: m_testTargetJobArgs[jobId];
|
||||
m_jobInfos.emplace_back(JobInfo({jobId}, args, AZStd::move(jobData)));
|
||||
}
|
||||
|
||||
// When the test run jobs are executed
|
||||
const auto runnerJobs = m_testRunner->RunTests(m_jobInfos, JobExceptionPolicy::Never);
|
||||
|
||||
// Expect half the jobs to successfully result in a test run that matches the expected test run data for that test target
|
||||
// with the other half having timed out
|
||||
for (const auto& job : runnerJobs)
|
||||
{
|
||||
const JobInfo::IdType jobId = job.GetJobInfo().GetId().m_value;
|
||||
if (jobId % 2)
|
||||
{
|
||||
ValidateJobTimeout(job);
|
||||
}
|
||||
else
|
||||
{
|
||||
ValidateTestRunCompleted(job, m_expectedTestTargetResult[jobId]);
|
||||
ValidateTestTargetRun(job.GetPayload().value(), m_expectedTestTargetRuns[jobId]);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
TEST_F(TestRunnerFixture, JobTimeout_InFlightJobTimeoutAndQueuedJobsUnlaunched)
|
||||
{
|
||||
// Given a test runner with no client callback or run timeout and a 5 second runner timeout
|
||||
m_testRunner = AZStd::make_unique<TestImpact::TestRunner>(
|
||||
AZStd::nullopt, FourConcurrentProcesses, AZStd::nullopt, AZStd::chrono::milliseconds(5000));
|
||||
|
||||
// Given an test run job for each test target where half will sleep indefinitely
|
||||
for (size_t jobId = 0; jobId < m_testTargetJobArgs.size(); jobId++)
|
||||
{
|
||||
JobData jobData(m_testTargetPaths[jobId].second);
|
||||
const AZStd::string args = (jobId % 2)
|
||||
? AZStd::string::format("%s %s", ValidProcessPath, ConstructTestProcessArgs(jobId, LongSleep).c_str())
|
||||
: m_testTargetJobArgs[jobId];
|
||||
m_jobInfos.emplace_back(JobInfo({jobId}, args, AZStd::move(jobData)));
|
||||
}
|
||||
|
||||
// When the test run jobs are executed
|
||||
const auto runnerJobs = m_testRunner->RunTests(m_jobInfos, JobExceptionPolicy::Never);
|
||||
|
||||
// Expect half the jobs to successfully result in a test run that matches the expected test run data for that test target
|
||||
// with the other half having timed out
|
||||
for (const auto& job : runnerJobs)
|
||||
{
|
||||
const JobInfo::IdType jobId = job.GetJobInfo().GetId().m_value;
|
||||
if (jobId % 2)
|
||||
{
|
||||
ValidateJobTimeout(job);
|
||||
}
|
||||
else
|
||||
{
|
||||
ValidateTestRunCompleted(job, m_expectedTestTargetResult[jobId]);
|
||||
ValidateTestTargetRun(job.GetPayload().value(), m_expectedTestTargetRuns[jobId]);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
INSTANTIATE_TEST_CASE_P(
|
||||
,
|
||||
TestRunnerFixtureWithConcurrencyAndFailedToLaunchExceptionParams,
|
||||
::testing::Combine(::testing::ValuesIn(MaxConcurrentRuns), ::testing::ValuesIn(FailedToLaunchExceptionPolicies)));
|
||||
|
||||
INSTANTIATE_TEST_CASE_P(
|
||||
,
|
||||
TestRunnerFixtureWithConcurrencyAndExecutedWithFailureExceptionParams,
|
||||
::testing::Combine(::testing::ValuesIn(MaxConcurrentRuns), ::testing::ValuesIn(ExecutedWithFailureExceptionPolicies)));
|
||||
|
||||
INSTANTIATE_TEST_CASE_P(, TestRunnerFixtureWithConcurrencyParams, ::testing::ValuesIn(MaxConcurrentRuns));
|
||||
} // namespace UnitTest
|
||||
@@ -1,137 +0,0 @@
|
||||
/*
|
||||
* All or portions of this file Copyright (c) Amazon.com, Inc. or its affiliates or
|
||||
* its licensors.
|
||||
*
|
||||
* For complete copyright and license terms please see the LICENSE at the root of this
|
||||
* distribution (the "License"). All use of this software is governed by the License,
|
||||
* or, if provided, by the license below or the license accompanying this file. Do not
|
||||
* remove or modify any license notices. This file is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
*
|
||||
*/
|
||||
|
||||
#include <AzCore/UnitTest/TestTypes.h>
|
||||
#include <AzTest/AzTest.h>
|
||||
#include <TestImpactFramework/TestImpactFrameworkPath.h>
|
||||
|
||||
namespace UnitTest
|
||||
{
|
||||
class FrameworkPathTestFixture
|
||||
: public AllocatorsTestFixture
|
||||
{
|
||||
public:
|
||||
void SetUp() override
|
||||
{
|
||||
UnitTest::AllocatorsTestFixture::SetUp();
|
||||
|
||||
m_parentPathAbs = AZStd::string(AZStd::string("parent") + AZ_TRAIT_OS_PATH_SEPARATOR + "path");
|
||||
m_childPathRel = AZStd::string(AZStd::string("child") + AZ_TRAIT_OS_PATH_SEPARATOR + "path");
|
||||
m_childPathAbs = AZStd::string(m_parentPathAbs + AZ_TRAIT_OS_PATH_SEPARATOR + m_childPathRel);
|
||||
|
||||
m_pathComponentA = AZStd::string("DirA");
|
||||
m_pathComponentB = AZStd::string("DirB");
|
||||
m_pathComponentC = AZStd::string("DirC");
|
||||
|
||||
m_posixPath = m_pathComponentA + AZ::IO::PosixPathSeparator + m_pathComponentB + AZ::IO::PosixPathSeparator + m_pathComponentC;
|
||||
|
||||
m_windowsPath =
|
||||
m_pathComponentA + AZ::IO::WindowsPathSeparator + m_pathComponentB + AZ::IO::WindowsPathSeparator + m_pathComponentC;
|
||||
|
||||
m_mixedPath =
|
||||
m_pathComponentA + AZ::IO::WindowsPathSeparator + m_pathComponentB + AZ::IO::PosixPathSeparator + m_pathComponentC;
|
||||
|
||||
m_referredPath =
|
||||
m_pathComponentA + AZ_TRAIT_OS_PATH_SEPARATOR + m_pathComponentB + AZ_TRAIT_OS_PATH_SEPARATOR + m_pathComponentC;
|
||||
}
|
||||
|
||||
void TearDown() override
|
||||
{
|
||||
UnitTest::AllocatorsTestFixture::TearDown();
|
||||
}
|
||||
|
||||
AZStd::string m_parentPathAbs;
|
||||
AZStd::string m_childPathRel;
|
||||
AZStd::string m_childPathAbs;
|
||||
|
||||
AZStd::string m_pathComponentA;
|
||||
AZStd::string m_pathComponentB;
|
||||
AZStd::string m_pathComponentC;
|
||||
|
||||
AZ::IO::Path m_posixPath;
|
||||
AZ::IO::Path m_windowsPath;
|
||||
AZ::IO::Path m_mixedPath;
|
||||
AZ::IO::Path m_referredPath;
|
||||
};
|
||||
|
||||
TEST_F(FrameworkPathTestFixture, DefaultConstructor_HasEmptyAbsAndRelPaths)
|
||||
{
|
||||
// Given an empty framework path
|
||||
TestImpact::FrameworkPath path;
|
||||
|
||||
// Expect the absolute path to be empty
|
||||
EXPECT_TRUE(path.Absolute().empty());
|
||||
|
||||
// Expect the relative path to be empty
|
||||
EXPECT_TRUE(path.Relative().empty());
|
||||
}
|
||||
|
||||
TEST_F(FrameworkPathTestFixture, OrphanConstructor_HasAbsAndEmptyRelPaths)
|
||||
{
|
||||
// Given an orhpan framework path
|
||||
TestImpact::FrameworkPath path(m_parentPathAbs);
|
||||
|
||||
// Expect the absolute path to be equal to the specified path
|
||||
EXPECT_EQ(path.Absolute().String(), m_parentPathAbs);
|
||||
|
||||
// Expect the relative path to be current directory symbol
|
||||
EXPECT_STREQ(path.Relative().c_str(), ".");
|
||||
}
|
||||
|
||||
TEST_F(FrameworkPathTestFixture, ParentConstructor_HasAbsAndRelPaths)
|
||||
{
|
||||
// Given a child framework path
|
||||
TestImpact::FrameworkPath path(m_childPathAbs, TestImpact::FrameworkPath(m_parentPathAbs));
|
||||
|
||||
// Expect the absolute path to be equal to the concatenation of the parent and child path
|
||||
EXPECT_EQ(path.Absolute().String(), m_childPathAbs);
|
||||
|
||||
// Expect the relative path to equal to the specified path
|
||||
EXPECT_EQ(path.Relative().String(), m_childPathRel);
|
||||
}
|
||||
|
||||
TEST_F(FrameworkPathTestFixture, PosixSeperators_HasUniformPreferredSeperators)
|
||||
{
|
||||
// Given an orhpan framework path with Posix separators
|
||||
TestImpact::FrameworkPath path(m_posixPath);
|
||||
|
||||
// Expect the absolute path to be equal to the specified path with preferred separators
|
||||
EXPECT_EQ(path.Absolute(), m_referredPath);
|
||||
|
||||
// Expect the relative path to be current directory symbol
|
||||
EXPECT_STREQ(path.Relative().c_str(), ".");
|
||||
}
|
||||
|
||||
TEST_F(FrameworkPathTestFixture, WindowsSeperators_HasUniformPreferredSeperators)
|
||||
{
|
||||
// Given an orhpan framework path with Windows separators
|
||||
TestImpact::FrameworkPath path(m_windowsPath);
|
||||
|
||||
// Expect the absolute path to be equal to the specified path with preferred separators
|
||||
EXPECT_EQ(path.Absolute(), m_referredPath);
|
||||
|
||||
// Expect the relative path to be current directory symbol
|
||||
EXPECT_STREQ(path.Relative().c_str(), ".");
|
||||
}
|
||||
|
||||
TEST_F(FrameworkPathTestFixture, MixedSeperators_HasUniformPreferredSeperators)
|
||||
{
|
||||
// Given an orhpan framework path with mixed separators
|
||||
TestImpact::FrameworkPath path(m_windowsPath);
|
||||
|
||||
// Expect the absolute path to be equal to the specified path with preferred separators
|
||||
EXPECT_EQ(path.Absolute(), m_referredPath);
|
||||
|
||||
// Expect the relative path to be current directory symbol
|
||||
EXPECT_STREQ(path.Relative().c_str(), ".");
|
||||
}
|
||||
} // namespace UnitTest
|
||||
@@ -1,33 +0,0 @@
|
||||
/*
|
||||
* All or portions of this file Copyright (c) Amazon.com, Inc. or its affiliates or
|
||||
* its licensors.
|
||||
*
|
||||
* For complete copyright and license terms please see the LICENSE at the root of this
|
||||
* distribution (the "License"). All use of this software is governed by the License,
|
||||
* or, if provided, by the license below or the license accompanying this file. Do not
|
||||
* remove or modify any license notices. This file is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
*
|
||||
*/
|
||||
|
||||
#include <AzCore/Memory/OSAllocator.h>
|
||||
#include <AzCore/Memory/SystemAllocator.h>
|
||||
#include <AzTest/AzTest.h>
|
||||
|
||||
class TestImpactTestEnvironment : public AZ::Test::ITestEnvironment
|
||||
{
|
||||
protected:
|
||||
void SetupEnvironment() override
|
||||
{
|
||||
AZ::AllocatorInstance<AZ::OSAllocator>::Create();
|
||||
AZ::AllocatorInstance<AZ::SystemAllocator>::Create();
|
||||
}
|
||||
|
||||
void TeardownEnvironment() override
|
||||
{
|
||||
AZ::AllocatorInstance<AZ::OSAllocator>::Destroy();
|
||||
AZ::AllocatorInstance<AZ::SystemAllocator>::Destroy();
|
||||
}
|
||||
};
|
||||
|
||||
AZ_UNIT_TEST_HOOK(new TestImpactTestEnvironment);
|
||||
Reference in New Issue
Block a user