birkeh/o3de
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Merge pull request #67 from aws-lumberyard-dev/TIF/Jenkins_Console

Browse Source 
Tif/jenkins console
This commit is contained in:
jonawals
2021-06-01 15:56:41 +01:00
committed by GitHub
parent f3eb267a5e 9fe91ec705
commit b72733b156
7 changed files with 361 additions and 17 deletions
Download Patch File Download Diff File Expand all files Collapse all files
Code/Tools/TestImpactFramework/Frontend/Console/Executable/Code/Source/TestImpactConsole.cpp
+29
View File
@@ -0,0 +1,29 @@
/*
* All or portions of this file Copyright (c) Amazon.com, Inc. or its affiliates or
* its licensors.
*
* For complete copyright and license terms please see the LICENSE at the root of this
* distribution (the "License"). All use of this software is governed by the License,
* or, if provided, by the license below or the license accompanying this file. Do not
* remove or modify any license notices. This file is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
*
*/
#include <TestImpactFramework/TestImpactConsoleMain.h>
#include <AzCore/Memory/OSAllocator.h>
#include <AzCore/Memory/SystemAllocator.h>
int main(int argc, char** argv)
{
AZ::AllocatorInstance<AZ::OSAllocator>::Create();
AZ::AllocatorInstance<AZ::SystemAllocator>::Create();
TestImpact::Console::ReturnCode returnCode = TestImpact::Console::Main(argc, argv);
AZ::AllocatorInstance<AZ::SystemAllocator>::Destroy();
AZ::AllocatorInstance<AZ::OSAllocator>::Destroy();
return static_cast<int>(returnCode);
}
Code/Tools/TestImpactFramework/Frontend/Console/Static/Code/Include/TestImpactFramework/TestImpactConsoleMain.h
+35
View File
@@ -0,0 +1,35 @@
/*
* All or portions of this file Copyright (c) Amazon.com, Inc. or its affiliates or
* its licensors.
*
* For complete copyright and license terms please see the LICENSE at the root of this
* distribution (the "License"). All use of this software is governed by the License,
* or, if provided, by the license below or the license accompanying this file. Do not
* remove or modify any license notices. This file is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
*
*/
#pragma once
namespace TestImpact
{
namespace Console
{
enum class ReturnCode : int
{
Success = 0, //!< The instigation operation(s) returned without error.
InvalidArgs, //!< The specified command line arguments were incorrect.
InvalidUnifiedDiff, //!< The specified unified diff could not be transformed into a valid change list.
InvalidConfiguration, //!< The runtime configuration is malformed.
RuntimeError, //!< The runtime encountered an error that it could not recover from.
UnhandledError, //!< The framework encountered an error that it anticipated but did not handle and could not recover from.
UnknownError, //!< An error of unknown origin was encountered that the console or runtime could not recover from.
TestFailure, //!< The test sequence had one or more test failures.
Timeout //!< The test sequence runtime exceeded the global timeout value.
};
//! Entry point for the console front end application.
[[nodiscard]] ReturnCode Main(int argc, char** argv);
} // namespace Console
} // namespace TestImpact
Code/Tools/TestImpactFramework/Frontend/Console/Static/Code/Source/TestImpactConsoleMain.cpp
+279
View File
@@ -0,0 +1,279 @@
/*
* All or portions of this file Copyright (c) Amazon.com, Inc. or its affiliates or
* its licensors.
*
* For complete copyright and license terms please see the LICENSE at the root of this
* distribution (the "License"). All use of this software is governed by the License,
* or, if provided, by the license below or the license accompanying this file. Do not
* remove or modify any license notices. This file is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
*
*/
#include <TestImpactFramework/TestImpactException.h>
#include <TestImpactFramework/TestImpactChangeListException.h>
#include <TestImpactFramework/TestImpactConfigurationException.h>
#include <TestImpactFramework/TestImpactRuntimeException.h>
#include <TestImpactFramework/TestImpactConsoleMain.h>
#include <TestImpactFramework/TestImpactChangeListSerializer.h>
#include <TestImpactFramework/TestImpactChangeList.h>
#include <TestImpactFramework/TestImpactRuntime.h>
#include <TestImpactFramework/TestImpactFileUtils.h>
#include <TestImpactFramework/TestImpactClientTestSelection.h>
#include <TestImpactFramework/TestImpactRuntime.h>
#include <TestImpactConsoleTestSequenceEventHandler.h>
#include <TestImpactCommandLineOptions.h>
#include <TestImpactConfigurationFactory.h>
#include <TestImpactCommandLineOptionsException.h>
#include <AzCore/IO/SystemFile.h>
#include <AzCore/std/containers/vector.h>
#include <AzCore/std/string/string.h>
#include <iostream>
namespace TestImpact
{
namespace Console
{
//! Generates a string to be used for printing to the console for the specified change list.
AZStd::string GenerateChangeListString(const ChangeList& changeList)
{
AZStd::string output;
const auto& outputFiles = [&output](const AZStd::vector<RepoPath>& files)
{
for (const auto& file : files)
{
output += AZStd::string::format("\t%s\n", file.c_str());
}
};
output += AZStd::string::format("Created files (%u):\n", changeList.m_createdFiles.size());
outputFiles(changeList.m_createdFiles);
output += AZStd::string::format("Updated files (%u):\n", changeList.m_updatedFiles.size());
outputFiles(changeList.m_updatedFiles);
output += AZStd::string::format("Deleted files (%u):\n", changeList.m_deletedFiles.size());
outputFiles(changeList.m_deletedFiles);
return output;
}
//! Gets the appropriate console return code for the specified test sequence result.
ReturnCode GetReturnCodeForTestSequenceResult(TestSequenceResult result)
{
switch (result)
{
case TestSequenceResult::Success:
return ReturnCode::Success;
case TestSequenceResult::Failure:
return ReturnCode::TestFailure;
case TestSequenceResult::Timeout:
return ReturnCode::Timeout;
default:
std::cout << "Unexpected TestSequenceResult value: " << aznumeric_cast<size_t>(result) << std::endl;
return ReturnCode::UnknownError;
}
}
//! Wrapper around impact analysis sequences to handle the case where the safe mode option is active.
ReturnCode WrappedImpactAnalysisTestSequence(
TestSequenceEventHandler& sequenceEventHandler,
const CommandLineOptions& options,
Runtime& runtime,
const AZStd::optional<ChangeList>& changeList)
{
AZ_TestImpact_Eval(
changeList.has_value(),
CommandLineOptionsException,
"Expected a change list for impact analysis but none was provided");
TestSequenceResult result = TestSequenceResult::Failure;
if (options.HasSafeMode())
{
auto [selectedResult, discardedResult] = runtime.SafeImpactAnalysisTestSequence(
changeList.value(),
options.GetSuitesFilter(),
options.GetTestPrioritizationPolicy(),
options.GetTestTargetTimeout(),
options.GetGlobalTimeout(),
AZStd::ref(sequenceEventHandler),
AZStd::ref(sequenceEventHandler),
AZStd::ref(sequenceEventHandler));
// Handling the possible timeout and failure permutations of the selected and discarded test results is splitting hairs
// so apply the following, admittedly arbitrary, rules to determine what the composite test sequence result should be
if (selectedResult == TestSequenceResult::Success && discardedResult == TestSequenceResult::Success)
{
// Trivial case: both sequences succeeded
result = TestSequenceResult::Success;
}
else if (selectedResult == TestSequenceResult::Failure || discardedResult == TestSequenceResult::Failure)
{
// One sequence failed whilst the other sequence either succeeded or timed out
result = TestSequenceResult::Failure;
}
else
{
// One sequence timed out whilst the other sequence succeeded or both sequences timed out
result = TestSequenceResult::Timeout;
}
}
else
{
result = runtime.ImpactAnalysisTestSequence(
changeList.value(),
options.GetTestPrioritizationPolicy(),
options.GetTestTargetTimeout(),
options.GetGlobalTimeout(),
AZStd::ref(sequenceEventHandler),
AZStd::ref(sequenceEventHandler),
AZStd::ref(sequenceEventHandler));
}
return GetReturnCodeForTestSequenceResult(result);
};
//! Entry point for the test impact analysis framework console front end application.
ReturnCode Main(int argc, char** argv)
{
try
{
CommandLineOptions options(argc, argv);
AZStd::optional<ChangeList> changeList;
// If we have a change list, check to see whether or not the client has requested the printing of said change list
if (options.HasChangeListFile())
{
changeList = DeserializeChangeList(ReadFileContents<CommandLineOptionsException>(*options.GetChangeListFile()));
if (options.HasOutputChangeList())
{
std::cout << "Change List:\n";
std::cout << GenerateChangeListString(*changeList).c_str();
if (options.GetTestSequenceType() == TestSequenceType::None)
{
return ReturnCode::Success;
}
}
}
// As of now, there are no other non-test operations other than printing a change list so getting this far is considered an error
AZ_TestImpact_Eval(options.GetTestSequenceType() != TestSequenceType::None, CommandLineOptionsException, "No action specified");
std::cout << "Constructing in-memory model of source tree and test coverage, this may take a moment...\n";
Runtime runtime(
ConfigurationFactory(ReadFileContents<CommandLineOptionsException>(options.GetConfigurationFile())),
options.GetExecutionFailurePolicy(),
options.GetExecutionFailureDraftingPolicy(),
options.GetTestFailurePolicy(),
options.GetIntegrityFailurePolicy(),
options.GetTestShardingPolicy(),
options.GetTargetOutputCapture(),
options.GetMaxConcurrency());
if (runtime.HasImpactAnalysisData())
{
std::cout << "Test impact analysis data for this repository was found.\n";
}
else
{
std::cout << "Test impact analysis data for this repository was not found, seed or regular sequence fallbacks will be used.\n";
}
TestSequenceEventHandler sequenceEventHandler(&options.GetSuitesFilter());
switch (const auto type = options.GetTestSequenceType())
{
case TestSequenceType::Regular:
{
const auto result = runtime.RegularTestSequence(
options.GetSuitesFilter(),
options.GetTestTargetTimeout(),
options.GetGlobalTimeout(),
AZStd::ref(sequenceEventHandler),
AZStd::ref(sequenceEventHandler),
AZStd::ref(sequenceEventHandler));
return GetReturnCodeForTestSequenceResult(result);
}
case TestSequenceType::Seed:
{
const auto result = runtime.SeededTestSequence(
options.GetTestTargetTimeout(),
options.GetGlobalTimeout(),
AZStd::ref(sequenceEventHandler),
AZStd::ref(sequenceEventHandler),
AZStd::ref(sequenceEventHandler));
return GetReturnCodeForTestSequenceResult(result);
}
case TestSequenceType::ImpactAnalysis:
{
return WrappedImpactAnalysisTestSequence(sequenceEventHandler, options, runtime, changeList);
}
case TestSequenceType::ImpactAnalysisOrSeed:
{
if (runtime.HasImpactAnalysisData())
{
return WrappedImpactAnalysisTestSequence(sequenceEventHandler, options, runtime, changeList);
}
else
{
const auto result = runtime.SeededTestSequence(
options.GetTestTargetTimeout(),
options.GetGlobalTimeout(),
AZStd::ref(sequenceEventHandler),
AZStd::ref(sequenceEventHandler),
AZStd::ref(sequenceEventHandler));
return GetReturnCodeForTestSequenceResult(result);
}
}
default:
std::cout << "Unexpected TestSequenceType value: " << static_cast<size_t>(type) << std::endl;
return ReturnCode::UnknownError;
}
}
catch (const CommandLineOptionsException& e)
{
std::cout << e.what() << std::endl;
std::cout << CommandLineOptions::GetCommandLineUsageString().c_str() << std::endl;
return ReturnCode::InvalidArgs;
}
catch (const ChangeListException& e)
{
std::cout << e.what() << std::endl;
return ReturnCode::InvalidUnifiedDiff;
}
catch (const ConfigurationException& e)
{
std::cout << e.what() << std::endl;
return ReturnCode::InvalidConfiguration;
}
catch (const RuntimeException& e)
{
std::cout << e.what() << std::endl;
return ReturnCode::RuntimeError;
}
catch (const Exception& e)
{
std::cout << e.what() << std::endl;
return ReturnCode::UnhandledError;
}
catch (const std::exception& e)
{
std::cout << e.what() << std::endl;
return ReturnCode::UnknownError;
}
catch (...)
{
std::cout << "An unknown error occurred" << std::endl;
return ReturnCode::UnknownError;
}
}
} // namespace Console
} // namespace TestImpact
Code/Tools/TestImpactFramework/Runtime/Code/Source/Platform/Windows/platform_windows_files.cmake
+1
View File
@@ -17,4 +17,5 @@ set(FILES
Process/TestImpactWin32_Pipe.cpp
Process/TestImpactWin32_Pipe.h
TestEngine/JobRunner/TestImpactWin32_TestTargetExtension.cpp
TestEngine/TestImpactWin32_TestEngineJobFailure.cpp
)
Code/Tools/TestImpactFramework/Runtime/Code/Source/TestImpactRepoPath.cpp
+11 -11
View File
@@ -32,70 +32,70 @@ namespace TestImpact
return *this;
}
inline RepoPath operator/(const RepoPath& lhs, const AZ::IO::PathView& rhs)
RepoPath operator/(const RepoPath& lhs, const AZ::IO::PathView& rhs)
{
RepoPath result(lhs);
result.m_path /= RepoPath(rhs).m_path;
return result;
}
inline RepoPath operator/(const RepoPath& lhs, AZStd::string_view rhs)
RepoPath operator/(const RepoPath& lhs, AZStd::string_view rhs)
{
RepoPath result(lhs);
result.m_path /= RepoPath(rhs).m_path;
return result;
}
inline RepoPath operator/(const RepoPath& lhs, const RepoPath::value_type* rhs)
RepoPath operator/(const RepoPath& lhs, const RepoPath::value_type* rhs)
{
RepoPath result(lhs);
result.m_path /= RepoPath(rhs).m_path;
return result;
}
inline RepoPath operator/(const RepoPath& lhs, const RepoPath& rhs)
RepoPath operator/(const RepoPath& lhs, const RepoPath& rhs)
{
RepoPath result(lhs);
result.m_path /= rhs.m_path;
return result;
}
inline RepoPath& RepoPath::operator/=(const AZ::IO::PathView& rhs)
RepoPath& RepoPath::operator/=(const AZ::IO::PathView& rhs)
{
m_path /= RepoPath(rhs).m_path;
return *this;
}
inline RepoPath& RepoPath::operator/=(AZStd::string_view rhs)
RepoPath& RepoPath::operator/=(AZStd::string_view rhs)
{
m_path /= RepoPath(rhs).m_path;
return *this;
}
inline RepoPath& RepoPath::operator/=(const RepoPath::value_type* rhs)
RepoPath& RepoPath::operator/=(const RepoPath::value_type* rhs)
{
m_path /= RepoPath(rhs).m_path;
return *this;
}
inline RepoPath& RepoPath::operator/=(const RepoPath& rhs)
RepoPath& RepoPath::operator/=(const RepoPath& rhs)
{
m_path /= rhs.m_path;
return *this;
}
inline bool operator==(const RepoPath& lhs, const RepoPath& rhs) noexcept
bool operator==(const RepoPath& lhs, const RepoPath& rhs) noexcept
{
return lhs.m_path.Compare(rhs.m_path) == 0;
}
inline bool operator!=(const RepoPath& lhs, const RepoPath& rhs) noexcept
bool operator!=(const RepoPath& lhs, const RepoPath& rhs) noexcept
{
return lhs.m_path.Compare(rhs.m_path) != 0;
}
inline bool operator<([[maybe_unused]] const RepoPath& lhs, [[maybe_unused]] const RepoPath& rhs) noexcept
bool operator<([[maybe_unused]] const RepoPath& lhs, [[maybe_unused]] const RepoPath& rhs) noexcept
{
return lhs.m_path.String() < rhs.m_path.String();
}
Code/Tools/TestImpactFramework/Runtime/Code/Source/TestImpactRuntime.cpp
+5 -5
View File
@@ -309,7 +309,7 @@ namespace TestImpact
if (testSequenceEndCallback.has_value())
{
(*testSequenceEndCallback)(CreateSequenceFailureReport(testJobs), timer.Elapsed());
(*testSequenceEndCallback)(GenerateSequenceFailureReport(testJobs), timer.Elapsed());
}
return result;
@@ -353,7 +353,7 @@ namespace TestImpact
if (testSequenceEndCallback.has_value())
{
(*testSequenceEndCallback)(CreateSequenceFailureReport(testJobs), timer.Elapsed());
(*testSequenceEndCallback)(GenerateSequenceFailureReport(testJobs), timer.Elapsed());
}
return result;
@@ -419,8 +419,8 @@ namespace TestImpact
if (testSequenceEndCallback.has_value())
{
(*testSequenceEndCallback)(
CreateSequenceFailureReport(selectedTestJobs),
CreateSequenceFailureReport(discardedTestJobs),
GenerateSequenceFailureReport(selectedTestJobs),
GenerateSequenceFailureReport(discardedTestJobs),
timer.Elapsed());
}
@@ -471,7 +471,7 @@ namespace TestImpact
if (testSequenceEndCallback.has_value())
{
(*testSequenceEndCallback)(CreateSequenceFailureReport(testJobs), timer.Elapsed());
(*testSequenceEndCallback)(GenerateSequenceFailureReport(testJobs), timer.Elapsed());
}
return result;
Code/Tools/TestImpactFramework/Runtime/Code/Source/TestImpactRuntimeUtils.h
+1 -1
View File
@@ -114,7 +114,7 @@ namespace TestImpact
}
case Client::TestRunResult::TestFailures:
{
testRunFailures.push_back(ExtractTestRunFailure(testJob));
testRunFailures.push_back(GenerateTestRunFailure(testJob));
break;
}
default: