birkeh/o3de
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Merge branch 'TIF/Jenkins' into TIF/Jenkins_Test

Browse Source
This commit is contained in:
jonawals
2021-06-08 17:00:16 +01:00
parent f6f38a12d8 f78ad7f377
commit ff2cdb3f94
3 changed files with 168 additions and 95 deletions
Download Patch File Download Diff File Expand all files Collapse all files
Code/Tools/TestImpactFramework/Frontend/Console/Static/Code/Source/TestImpactCommandLineOptions.cpp
+33 -33
View File
@@ -22,22 +22,22 @@ namespace TestImpact
enum
{
// Options
Config,
ChangeList,
OutputChangeList,
Sequence,
TestPrioritizationPolicy,
ExecutionFailurePolicy,
FailedTestCoveragePolicy,
TestFailurePolicy,
IntegrityFailurePolicy,
TestShardingPolicy,
TargetOutputCapture,
MaxConcurrency,
TestTargetTimeout,
GlobalTimeout,
SuiteFilter,
SafeMode,
ConfigKey,
ChangeListKey,
OutputChangeListKey,
SequenceKey,
TestPrioritizationPolicyKey,
ExecutionFailurePolicyKey,
FailedTestCoveragePolicyKey,
TestFailurePolicyKey,
IntegrityFailurePolicyKey,
TestShardingPolicyKey,
TargetOutputCaptureKey,
MaxConcurrencyKey,
TestTargetTimeoutKey,
GlobalTimeoutKey,
SuiteFilterKey,
SafeModeKey,
// Values
None,
Seed,
@@ -93,17 +93,17 @@ namespace TestImpact
RepoPath ParseConfigurationFile(const AZ::CommandLine& cmd)
{
return ParsePathOption(OptionKeys[Config], cmd).value_or(LY_TEST_IMPACT_DEFAULT_CONFIG_FILE);
return ParsePathOption(OptionKeys[ConfigKey], cmd).value_or(LY_TEST_IMPACT_DEFAULT_CONFIG_FILE);
}
AZStd::optional<RepoPath> ParseChangeListFile(const AZ::CommandLine& cmd)
{
return ParsePathOption(OptionKeys[ChangeList], cmd);
return ParsePathOption(OptionKeys[ChangeListKey], cmd);
}
bool ParseOutputChangeList(const AZ::CommandLine& cmd)
{
return ParseOnOffOption(OptionKeys[OutputChangeList], BinaryStateValue<bool>{ false, true }, cmd).value_or(false);
return ParseOnOffOption(OptionKeys[OutputChangeListKey], BinaryStateValue<bool>{ false, true }, cmd).value_or(false);
}
TestSequenceType ParseTestSequenceType(const AZ::CommandLine& cmd)
@@ -118,7 +118,7 @@ namespace TestImpact
{OptionKeys[ImpactAnalysisOrSeed], TestSequenceType::ImpactAnalysisOrSeed}
};
return ParseMultiStateOption(OptionKeys[Sequence], states, cmd).value_or(TestSequenceType::None);
return ParseMultiStateOption(OptionKeys[SequenceKey], states, cmd).value_or(TestSequenceType::None);
}
Policy::TestPrioritization ParseTestPrioritizationPolicy(const AZ::CommandLine& cmd)
@@ -129,7 +129,7 @@ namespace TestImpact
{OptionKeys[Locality], Policy::TestPrioritization::DependencyLocality}
};
return ParseBinaryStateOption(OptionKeys[TestPrioritizationPolicy], states, cmd).value_or(Policy::TestPrioritization::None);
return ParseBinaryStateOption(OptionKeys[TestPrioritizationPolicyKey], states, cmd).value_or(Policy::TestPrioritization::None);
}
Policy::ExecutionFailure ParseExecutionFailurePolicy(const AZ::CommandLine& cmd)
@@ -140,7 +140,7 @@ namespace TestImpact
{OptionKeys[Continue], Policy::ExecutionFailure::Continue},
{OptionKeys[Ignore], Policy::ExecutionFailure::Ignore}
};
return ParseMultiStateOption(OptionKeys[ExecutionFailurePolicy], states, cmd).value_or(Policy::ExecutionFailure::Continue);
return ParseMultiStateOption(OptionKeys[ExecutionFailurePolicyKey], states, cmd).value_or(Policy::ExecutionFailure::Continue);
}
Policy::FailedTestCoverage ParseFailedTestCoveragePolicy(const AZ::CommandLine& cmd)
@@ -151,7 +151,7 @@ namespace TestImpact
{OptionKeys[Keep], Policy::FailedTestCoverage::Keep}
};
return ParseMultiStateOption(OptionKeys[FailedTestCoveragePolicy], states, cmd).value_or(Policy::FailedTestCoverage::Keep);
return ParseMultiStateOption(OptionKeys[FailedTestCoveragePolicyKey], states, cmd).value_or(Policy::FailedTestCoverage::Keep);
}
Policy::TestFailure ParseTestFailurePolicy(const AZ::CommandLine& cmd)
@@ -162,7 +162,7 @@ namespace TestImpact
Policy::TestFailure::Continue
};
return ParseAbortContinueOption(OptionKeys[TestFailurePolicy], states, cmd).value_or(Policy::TestFailure::Abort);
return ParseAbortContinueOption(OptionKeys[TestFailurePolicyKey], states, cmd).value_or(Policy::TestFailure::Abort);
}
Policy::IntegrityFailure ParseIntegrityFailurePolicy(const AZ::CommandLine& cmd)
@@ -173,7 +173,7 @@ namespace TestImpact
Policy::IntegrityFailure::Continue
};
return ParseAbortContinueOption(OptionKeys[IntegrityFailurePolicy], states, cmd).value_or(Policy::IntegrityFailure::Abort);
return ParseAbortContinueOption(OptionKeys[IntegrityFailurePolicyKey], states, cmd).value_or(Policy::IntegrityFailure::Abort);
}
Policy::TestSharding ParseTestShardingPolicy(const AZ::CommandLine& cmd)
@@ -184,12 +184,12 @@ namespace TestImpact
Policy::TestSharding::Always
};
return ParseOnOffOption("shard", states, cmd).value_or(Policy::TestSharding::Never);
return ParseOnOffOption(OptionKeys[TestShardingPolicyKey], states, cmd).value_or(Policy::TestSharding::Never);
}
Policy::TargetOutputCapture ParseTargetOutputCapture(const AZ::CommandLine& cmd)
{
if (const auto numSwitchValues = cmd.GetNumSwitchValues(OptionKeys[TargetOutputCapture]);
if (const auto numSwitchValues = cmd.GetNumSwitchValues(OptionKeys[TargetOutputCaptureKey]);
numSwitchValues)
{
AZ_TestImpact_Eval(
@@ -198,7 +198,7 @@ namespace TestImpact
Policy::TargetOutputCapture targetOutputCapture = Policy::TargetOutputCapture::None;
for (auto i = 0; i < numSwitchValues; i++)
{
const auto option = cmd.GetSwitchValue(OptionKeys[TargetOutputCapture], i);
const auto option = cmd.GetSwitchValue(OptionKeys[TargetOutputCaptureKey], i);
if (option == OptionKeys[StdOut])
{
if (targetOutputCapture == Policy::TargetOutputCapture::File)
@@ -236,23 +236,23 @@ namespace TestImpact
AZStd::optional<size_t> ParseMaxConcurrency(const AZ::CommandLine& cmd)
{
return ParseUnsignedIntegerOption(OptionKeys[MaxConcurrency], cmd);
return ParseUnsignedIntegerOption(OptionKeys[MaxConcurrencyKey], cmd);
}
AZStd::optional<AZStd::chrono::milliseconds> ParseTestTargetTimeout(const AZ::CommandLine& cmd)
{
return ParseSecondsOption(OptionKeys[TestTargetTimeout], cmd);
return ParseSecondsOption(OptionKeys[TestTargetTimeoutKey], cmd);
}
AZStd::optional<AZStd::chrono::milliseconds> ParseGlobalTimeout(const AZ::CommandLine& cmd)
{
return ParseSecondsOption(OptionKeys[GlobalTimeout], cmd);
return ParseSecondsOption(OptionKeys[GlobalTimeoutKey], cmd);
}
bool ParseSafeMode(const AZ::CommandLine& cmd)
{
const BinaryStateValue<bool> states = { false, true };
return ParseOnOffOption(OptionKeys[SafeMode], states, cmd).value_or(false);
return ParseOnOffOption(OptionKeys[SafeModeKey], states, cmd).value_or(false);
}
SuiteType ParseSuiteFilter(const AZ::CommandLine& cmd)
@@ -264,7 +264,7 @@ namespace TestImpact
{GetSuiteTypeName(SuiteType::Sandbox), SuiteType::Sandbox}
};
return ParseMultiStateOption(OptionKeys[SuiteFilter], states, cmd).value_or(SuiteType::Main);
return ParseMultiStateOption(OptionKeys[SuiteFilterKey], states, cmd).value_or(SuiteType::Main);
}
}
scripts/build/TestImpactAnalysis/tiaf.py
+127 -39
View File
@@ -23,33 +23,40 @@ def is_child_path(parent_path, child_path):
child_path = os.path.abspath(child_path)
return os.path.commonpath([os.path.abspath(parent_path)]) == os.path.commonpath([os.path.abspath(parent_path), os.path.abspath(child_path)])
# Enumerations for test sequence types
class SequenceType(Enum):
# Regular sequence as-per the tiaf regular sequence
REGULAR = 1
# TIA sequence as-per the tiaf read-only impact analysis sequence
TEST_IMPACT_ANALYSIS = 2
# Seed sequence as-per the tiaf seed sequence
SEED = 3
class TestImpact:
def __init__(self, config_file, dst_commit):
def __init__(self, config_file, pipeline, dst_commit):
self.__pipeline = pipeline
self.__parse_config_file(config_file)
self.__init_repo(dst_commit)
self.__generate_change_list()
if self.__use_test_impact_analysis and not self.__is_pipeline_of_truth:
self.__generate_change_list()
# Parse the configuration file and retrieve the data needed for launching the test impact analysis runtime
def __parse_config_file(self, config_file):
print(f"Attempting to parse configuration file '{config_file}'...")
with open(config_file, "r") as config_data:
config = json.load(config_data)
# Repository
self.__repo_dir = config["repo"]["root"]
# Jenkins
self.__use_test_impact_analysis = config["jenkins"]["use_test_impact_analysis"]
self.__pipeline_of_truth = config["jenkins"]["pipeline_of_truth"]
print(f"Pipeline of truth: '{self.__pipeline_of_truth}'.")
print(f"This pipeline: '{self.__pipeline}'.")
if self.__pipeline in self.__pipeline_of_truth:
self.__is_pipeline_of_truth = True
else:
self.__is_pipeline_of_truth = False
print(f"Is pipeline of truth: '{self.__is_pipeline_of_truth}'.")
# TIAF binary
self.__tiaf_bin = config["repo"]["tiaf_bin"]
if not os.path.isfile(self.__tiaf_bin):
if self.__use_test_impact_analysis and not os.path.isfile(self.__tiaf_bin):
raise FileNotFoundError("Could not find tiaf binary")
# Workspaces
self.__active_workspace = config["workspace"]["active"]["root"]
self.__historic_workspace = config["workspace"]["historic"]["root"]
self.__temp_workspace = config["workspace"]["temp"]["root"]
# Last commit hash
last_commit_hash_path_rel = config["workspace"]["historic"]["relative_paths"]["last_run_hash_file"]
self.__last_commit_hash_path = os.path.join(self.__historic_workspace, last_commit_hash_path_rel)
print("The configuration file was parsed successfully.")
@@ -71,7 +78,7 @@ class TestImpact:
def __read_last_run_hash(self):
self.__has_src_commit = False
if os.path.isfile(self.__last_commit_hash_path):
print(f"Previous commit hash found at '{self.__last_commit_hash_path}'")
print(f"Previous commit hash found at '{self.__last_commit_hash_path}'.")
with open(self.__last_commit_hash_path) as file:
self.__src_commit = file.read()
self.__has_src_commit = True
@@ -144,43 +151,79 @@ class TestImpact:
return
# Runs the specified test sequence
def run(self, sequence_type, safe_mode, test_timeout, global_timeout):
def run(self, suite, safe_mode, test_timeout, global_timeout):
args = []
print("Please note: test impact analysis sequences will be run in read-only mode (seed sequences are unaffected).")
if sequence_type == SequenceType.REGULAR:
print("Sequence type: regular.")
args.append("--sequence=regular")
args.append("--fpolicy=abort")
elif sequence_type == SequenceType.SEED:
print("Sequence type: seed.")
args.append("--sequence=seed")
args.append("--fpolicy=continue")
elif sequence_type == SequenceType.TEST_IMPACT_ANALYSIS:
print("Sequence type: test impact analysis (no write).")
args.append("--fpolicy=abort")
if self.__has_change_list:
args.append(f"-changelist={self.__change_list_path}")
args.append("--sequence=tianowrite")
else:
print(f"No change list was generated, falling back to a regular sequence.")
print("Sequence type: Regular.")
args.append("--sequence=regular")
else:
raise ValueError(sequence_type)
# Suite
args.append(f"--suite={suite}")
print(f"Test suite is set to '{suite}'.")
# Timeouts
if test_timeout != None:
args.append(f"--ttimeout={test_timeout}")
print(f"Test target timeout is set to {test_timeout} seconds.")
if global_timeout != None:
args.append(f"--gtimeout={global_timeout}")
print(f"Global sequence timeout is set to {test_timeout} seconds.")
# If test impact analysis is enabled:
# -> Pipleine of truth will perform a seed that will continue until the sequence is complete regardless of test failues
# -> Non pipline of truth will attempt to perform an impact analysis sequence and exit early upon the fist test failure
# If test impact analysis is disabled:
# -> Pipleine of truth will perform a regular sequence that will continue until the sequence is complete regardless of test failues
# -> Non pipline of truth will perform a regular sequence and exit early upon the fist test failure
if self.__use_test_impact_analysis:
print("Test impact analysis ie enabled.")
# Pipeline of truth sequence
if self.__is_pipeline_of_truth:
# Sequence type
args.append("--sequence=seed")
print("Sequence type is set to 'seed'.")
# Test failure policy
args.append("--fpolicy=continue")
print("Test failure policy is set to 'continue'.")
# Non pipeline of truth sequence
else:
if self.__has_change_list:
# Change list
args.append(f"--changelist={self.__change_list_path}")
print(f"Change list is set to '{self.__change_list_path}'.")
# Sequence type
args.append("--sequence=tianowrite")
print("Sequence type is set to 'tianowrite'.")
# Safe mode
if safe_mode:
args.append("--safemode=on")
print("Safe mode set to 'on'.")
else:
args.append("--safemode=off")
print("Safe mode set to 'off'.")
else:
args.append("--sequence=regular")
print("Sequence type is set to 'regular'.")
# Test failure policy
args.append("--fpolicy=abort")
print("Test failure policy is set to 'abort'.")
else:
print("Test impact analysis ie disabled.")
# Sequence type
args.append("--sequence=regular")
print("Sequence type is set to 'seed'.")
# Pipeline of truth sequence
if self.__is_pipeline_of_truth:
# Test failure policy
args.append("--fpolicy=continue")
print("Test failure policy is set to 'continue'.")
# Non pipeline of truth sequence
else:
# Test failure policy
args.append("--fpolicy=abort")
print("Test failure policy is set to 'abort'.")
print("Args: ", end='')
print(*args)
result = subprocess.run([self.__tiaf_bin] + args)
if result.returncode == 0:
# If the sequence completed 9with or without failures) we will update the historical meta-data
if result.returncode == 0 or result.returncode == 7:
print("Test impact analysis runtime returned successfully.")
if sequence_type == SequenceType.SEED:
if self.__is_pipeline_of_truth:
print("Writing historical meta-data...")
self.__write_last_run_hash(self.__dst_commit)
print("Complete!")
@@ -188,3 +231,48 @@ class TestImpact:
print(f"The test impact analysis runtime returned with error: '{result.returncode}'.")
return result.returncode
#args = []
#print("Please note: test impact analysis sequences will be run in read-only mode (seed sequences are unaffected).")
#if sequence_type == SequenceType.REGULAR:
# print("Sequence type: regular.")
# args.append("--sequence=regular")
# args.append("--fpolicy=abort")
#elif sequence_type == SequenceType.SEED:
# print("Sequence type: seed.")
# args.append("--sequence=seed")
# args.append("--fpolicy=continue")
#elif sequence_type == SequenceType.TEST_IMPACT_ANALYSIS:
# print("Sequence type: test impact analysis (no write).")
# args.append("--fpolicy=abort")
# if self.__has_change_list:
# args.append(f"-changelist={self.__change_list_path}")
# args.append("--sequence=tianowrite")
# else:
# print(f"No change list was generated, falling back to a regular sequence.")
# print("Sequence type: Regular.")
# args.append("--sequence=regular")
#else:
# raise ValueError(sequence_type)
#
#if test_timeout != None:
# args.append(f"--ttimeout={test_timeout}")
# print(f"Test target timeout is set to {test_timeout} seconds.")
#if global_timeout != None:
# args.append(f"--gtimeout={global_timeout}")
# print(f"Global sequence timeout is set to {test_timeout} seconds.")
#
#print("Args: ", end='')
#print(*args)
#result = subprocess.run([self.__tiaf_bin] + args)
#if result.returncode == 0:
# print("Test impact analysis runtime returned successfully.")
# if sequence_type == SequenceType.SEED:
# print("Writing historical meta-data...")
# self.__write_last_run_hash(self.__dst_commit)
# print("Complete!")
#else:
# print(f"The test impact analysis runtime returned with error: '{result.returncode}'.")
#return result.returncode
scripts/build/TestImpactAnalysis/tiaf_driver.py
+8 -23
View File
@@ -11,7 +11,6 @@
import argparse
from tiaf import TestImpact
from tiaf import SequenceType
import sys
import os
@@ -26,16 +25,6 @@ def parse_args():
else:
raise FileNotFoundError(value)
def sequence_type(value):
if value == "regular":
return SequenceType.REGULAR
elif value == "tia":
return SequenceType.TEST_IMPACT_ANALYSIS
elif value == "seed":
return SequenceType.SEED
else:
raise ValueError(value)
def timout_type(value):
value = int(value)
if value <= 0:
@@ -44,24 +33,20 @@ def parse_args():
parser = argparse.ArgumentParser()
parser.add_argument('--config', dest="config", type=file_path, help="Path to the test impact analysis framework configuration file", required=True)
parser.add_argument('--destCommit', dest="dst_commit", help="Commit to run test impact analysis on (not required for seed)")
parser.add_argument('--sequenceType', dest="sequence_type", type=sequence_type, help="Test sequence type to run ('regular', 'seed' or 'tia')", required=True)
parser.add_argument('--suites', dest="suites", nargs='*', help="Suites to include for regular tes sequences (use '*' for all suites)")
parser.add_argument('--testTimeout', dest="test_timeout", type=timout_type, help="Maximum flight time (in seconds) of any test target before being terminated", required=False)
parser.add_argument('--globalTimeout', dest="global_timeout", type=timout_type, help="Maximum tun time of the sequence before being terminated", required=False)
parser.set_defaults(suites="*")
parser.set_defaults(safe_mode=False)
parser.add_argument('--pipeline', dest="pipeline", help="Pipeline the test impact analysis framework is running on", required=True)
parser.add_argument('--destCommit', dest="dst_commit", help="Commit to run test impact analysis on (ignored when seeding)", required=True)
parser.add_argument('--suite', dest="suite", help="Test suite to run", required=True)
parser.add_argument('--safeMode', dest="safe_mode", action='store_true', help="Run impact analysis tests in safe mode (ignored when seeding)")
parser.add_argument('--testTimeout', dest="test_timeout", type=timout_type, help="Maximum run time (in seconds) of any test target before being terminated", required=False)
parser.add_argument('--globalTimeout', dest="global_timeout", type=timout_type, help="Maximum run time of the sequence before being terminated", required=False)
parser.set_defaults(test_timeout=None)
parser.set_defaults(global_timeout=None)
args = parser.parse_args()
if args.sequence_type == SequenceType.TEST_IMPACT_ANALYSIS and args.dst_commit == None:
raise ValueError("Test impact analysis sequence must have a change list")
return args
if __name__ == "__main__":
args = parse_args()
tiaf = TestImpact(args.config, args.dst_commit)
return_code = tiaf.run(args.sequence_type, args.test_timeout, args.global_timeout)
tiaf = TestImpact(args.config, args.pipeline, args.dst_commit)
return_code = tiaf.run(args.suite, args.safe_mode, args.test_timeout, args.global_timeout)
sys.exit(return_code)