birkeh/o3de
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Further subdivide TIAF data by suite. (#3128)

Browse Source 
* Further subdivide TIAF data by suite.
* Key fix typo.

Signed-off-by: John <jonawals@amazon.com>
This commit is contained in:
jonawals
2021-08-16 14:15:02 +01:00
committed by GitHub
parent f7e536bfb1
commit 5925bd22f6
9 changed files with 42 additions and 51 deletions
Download Patch File Download Diff File Expand all files Collapse all files
Code/Tools/TestImpactFramework/Frontend/Console/Code/Source/TestImpactRuntimeConfigurationFactory.cpp
+6 -19
View File
@@ -27,7 +27,7 @@ namespace TestImpact
"relative_paths",
"artifact_dir",
"enumeration_cache_dir",
"test_impact_data_files",
"test_impact_data_file",
"temp",
"active",
"target_sources",
@@ -72,7 +72,7 @@ namespace TestImpact
RelativePaths,
ArtifactDir,
EnumerationCacheDir,
TestImpactDataFiles,
TestImpactDataFile,
TempWorkspace,
ActiveWorkspace,
TargetSources,
@@ -138,31 +138,18 @@ namespace TestImpact
tempWorkspaceConfig.m_artifactDirectory =
GetAbsPathFromRelPath(
tempWorkspaceConfig.m_root, tempWorkspace[Config::Keys[Config::RelativePaths]][Config::Keys[Config::ArtifactDir]].GetString());
tempWorkspaceConfig.m_enumerationCacheDirectory = GetAbsPathFromRelPath(
tempWorkspaceConfig.m_root,
tempWorkspace[Config::Keys[Config::RelativePaths]][Config::Keys[Config::EnumerationCacheDir]].GetString());
return tempWorkspaceConfig;
}
AZStd::array<RepoPath, 3> ParseTestImpactAnalysisDataFiles(const RepoPath& root, const rapidjson::Value& sparTiaFile)
{
AZStd::array<RepoPath, 3> sparTiaFiles;
sparTiaFiles[static_cast<size_t>(SuiteType::Main)] =
GetAbsPathFromRelPath(root, sparTiaFile[SuiteTypeAsString(SuiteType::Main).c_str()].GetString());
sparTiaFiles[static_cast<size_t>(SuiteType::Periodic)] =
GetAbsPathFromRelPath(root, sparTiaFile[SuiteTypeAsString(SuiteType::Periodic).c_str()].GetString());
sparTiaFiles[static_cast<size_t>(SuiteType::Sandbox)] =
GetAbsPathFromRelPath(root, sparTiaFile[SuiteTypeAsString(SuiteType::Sandbox).c_str()].GetString());
return sparTiaFiles;
}
WorkspaceConfig::Active ParseActiveWorkspaceConfig(const rapidjson::Value& activeWorkspace)
{
WorkspaceConfig::Active activeWorkspaceConfig;
const auto& relativePaths = activeWorkspace[Config::Keys[Config::RelativePaths]];
activeWorkspaceConfig.m_root = activeWorkspace[Config::Keys[Config::Root]].GetString();
activeWorkspaceConfig.m_enumerationCacheDirectory
= GetAbsPathFromRelPath(activeWorkspaceConfig.m_root, relativePaths[Config::Keys[Config::EnumerationCacheDir]].GetString());
activeWorkspaceConfig.m_sparTiaFiles =
ParseTestImpactAnalysisDataFiles(activeWorkspaceConfig.m_root, relativePaths[Config::Keys[Config::TestImpactDataFiles]]);
activeWorkspaceConfig.m_sparTiaFile = relativePaths[Config::Keys[Config::TestImpactDataFile]].GetString();
return activeWorkspaceConfig;
}
Code/Tools/TestImpactFramework/Runtime/Code/Include/TestImpactFramework/TestImpactClientSequenceReport.h
+1 -1
View File
@@ -530,7 +530,7 @@ namespace TestImpact
size_t GetTotalNumTimedOutTestRuns() const override;
size_t GetTotalNumUnexecutedTestRuns() const override;
//! Returns the report for the discarded test runs.
// ImpactAnalysisSequenceReport overrides ...
const TestRunSelection GetDiscardedTestRuns() const;
//! Returns the report for the discarded test runs.
Code/Tools/TestImpactFramework/Runtime/Code/Include/TestImpactFramework/TestImpactConfiguration.h
+2 -2
View File
@@ -37,14 +37,14 @@ namespace TestImpact
{
RepoPath m_root; //!< Path to the temporary workspace (cleaned prior to use).
RepoPath m_artifactDirectory; //!< Path to read and write runtime artifacts to and from.
RepoPath m_enumerationCacheDirectory; //!< Path to the test enumerations cache.
};
//! Active persistent data workspace configuration.
struct Active
{
RepoPath m_root; //!< Path to the persistent workspace tracked by the repository.
RepoPath m_enumerationCacheDirectory; //!< Path to the test enumerations cache.
AZStd::array<RepoPath, 3> m_sparTiaFiles; //!< Paths to the test impact analysis data files for each test suite.
RepoPath m_sparTiaFile; //!< Paths to the test impact analysis data file.
};
Temp m_temp;
Code/Tools/TestImpactFramework/Runtime/Code/Source/TestImpactRuntime.cpp
+3 -2
View File
@@ -275,7 +275,7 @@ namespace TestImpact
m_testEngine = AZStd::make_unique<TestEngine>(
m_config.m_repo.m_root,
m_config.m_target.m_outputDirectory,
m_config.m_workspace.m_active.m_enumerationCacheDirectory,
m_config.m_workspace.m_temp.m_enumerationCacheDirectory,
m_config.m_workspace.m_temp.m_artifactDirectory,
m_config.m_testEngine.m_testRunner.m_binary,
m_config.m_testEngine.m_instrumentation.m_binary,
@@ -289,7 +289,8 @@ namespace TestImpact
}
else
{
m_sparTiaFile = m_config.m_workspace.m_active.m_sparTiaFiles[static_cast<size_t>(m_suiteFilter)].String();
m_sparTiaFile =
m_config.m_workspace.m_active.m_root / RepoPath(SuiteTypeAsString(m_suiteFilter)) / m_config.m_workspace.m_active.m_sparTiaFile;
}
// Populate the dynamic dependency map with the existing source coverage data (if any)
cmake/TestImpactFramework/ConsoleFrontendConfig.in
+3 -8
View File
@@ -15,19 +15,14 @@
"temp": {
"root": "${temp_dir}",
"relative_paths": {
"artifact_dir": "RuntimeArtifact"
"artifact_dir": "RuntimeArtifact",
"enumeration_cache_dir": "EnumerationCache"
}
},
"active": {
"root": "${active_dir}",
"relative_paths": {
"test_impact_data_files": {
"main": "TestImpactData.main.spartia",
"periodic": "TestImpactData.periodic.spartia",
"sandbox": "TestImpactData.sandbox.spartia"
},
"enumeration_cache_dir": "EnumerationCache",
"last_build_target_list_file": "LastRunBuildTargets.json"
"test_impact_data_file": "TestImpactData.spartia"
}
},
"historic": {
scripts/build/TestImpactAnalysis/tiaf.py
+1 -2
View File
@@ -230,7 +230,7 @@ class TestImpact:
# Flag for corner case where:
# 1. TIAF was already run previously for this commit.
# 2. There was no last commit hash when TIAF last ran on this commit (due to no coverage data existing get for this branch)
# 2. There was no last commit hash when TIAF last ran on this commit (due to no coverage data existing yet for this branch)
# 3. TIAF has not been run on any other commits between the run for this commit and the last run for this commit.
# The above results in TIAF being stuck in a state of generating an empty change list (and thus doing no work until another
# commit comes in) which is problematic if the commit needs to be re-run for whatever reason so in these conditions we revert
@@ -323,7 +323,6 @@ class TestImpact:
logger.info(f"Args: {unpacked_args}")
runtime_result = subprocess.run([str(self._tiaf_bin)] + args)
report = None
# If the sequence completed (with or without failures) we will update the historical meta-data
if runtime_result.returncode == 0 or runtime_result.returncode == 7:
logger.info("Test impact analysis runtime returned successfully.")
scripts/build/TestImpactAnalysis/tiaf_persistent_storage.py
+21 -14
View File
@@ -17,11 +17,11 @@ logger = get_logger(__file__)
class PersistentStorage(ABC):
WORKSPACE_KEY = "workspace"
LAST_RUNS_KEY = "last_runs"
HISTORIC_SEQUENCES_KEY = "historic_sequences"
ACTIVE_KEY = "active"
ROOT_KEY = "root"
RELATIVE_PATHS_KEY = "relative_paths"
TEST_IMPACT_DATA_FILES_KEY = "test_impact_data_files"
TEST_IMPACT_DATA_FILE_KEY = "test_impact_data_file"
LAST_COMMIT_HASH_KEY = "last_commit_hash"
COVERAGE_DATA_KEY = "coverage_data"
@@ -35,19 +35,21 @@ class PersistentStorage(ABC):
"""
# Work on the assumption that there is no historic meta-data (a valid state to be in, should none exist)
self._suite = suite
self._last_commit_hash = None
self._has_historic_data = False
self._has_previous_last_commit_hash = False
self._this_commit_hash = commit
self._this_commit_hash_last_commit_hash = None
self._historic_data = None
logger.info(f"Attempting to access persistent storage for the commit {self._this_commit_hash}")
logger.info(f"Attempting to access persistent storage for the commit '{self._this_commit_hash}' for suite '{self._suite}'")
try:
# The runtime expects the coverage data to be in the location specified in the config file (unless overridden with
# the --datafile command line argument, which the TIAF scripts do not do)
self._active_workspace = pathlib.Path(config[self.WORKSPACE_KEY][self.ACTIVE_KEY][self.ROOT_KEY])
unpacked_coverage_data_file = config[self.WORKSPACE_KEY][self.ACTIVE_KEY][self.RELATIVE_PATHS_KEY][self.TEST_IMPACT_DATA_FILES_KEY][suite]
self._active_workspace = self._active_workspace.joinpath(pathlib.Path(self._suite))
unpacked_coverage_data_file = config[self.WORKSPACE_KEY][self.ACTIVE_KEY][self.RELATIVE_PATHS_KEY][self.TEST_IMPACT_DATA_FILE_KEY]
except KeyError as e:
raise SystemError(f"The config does not contain the key {str(e)}.")
@@ -70,25 +72,27 @@ class PersistentStorage(ABC):
self._last_commit_hash = self._historic_data[self.LAST_COMMIT_HASH_KEY]
logger.info(f"Last commit hash '{self._last_commit_hash}' found.")
if self.LAST_RUNS_KEY in self._historic_data:
# Last commit hash for the sequence that was run for this commit previously (if any)
if self._this_commit_hash in self._historic_data[self.LAST_RUNS_KEY]:
# Last commit hash for the sequence that was run for this commit previously (if any)
if self.HISTORIC_SEQUENCES_KEY in self._historic_data:
if self._this_commit_hash in self._historic_data[self.HISTORIC_SEQUENCES_KEY]:
# 'None' is a valid value for the previously used last commit hash if there was no coverage data at that time
self._this_commit_hash_last_commit_hash = self._historic_data[self.LAST_RUNS_KEY][self._this_commit_hash]
self._this_commit_hash_last_commit_hash = self._historic_data[self.HISTORIC_SEQUENCES_KEY][self._this_commit_hash]
self._has_previous_last_commit_hash = self._this_commit_hash_last_commit_hash is not None
if self._has_previous_last_commit_hash:
logger.info(f"Last commit hash '{self._this_commit_hash_last_commit_hash}' was used previously for this commit.")
else:
logger.info(f"Prior sequence data found for this commit but it is empty (there was no coverage data vailable at that time).")
logger.info(f"Prior sequence data found for this commit but it is empty (there was no coverage data available at that time).")
else:
logger.info(f"No prior sequence data found for commit '{self._this_commit_hash}', this is the first sequence for this commit.")
else:
logger.info(f"No prior sequence data found for any commits.")
# Create the active workspace directory where the coverage data file will be placed and unpack the coverage data so
# it is accessible by the runtime
# Create the active workspace directory for the unpacked historic data files so they are accessible by the runtime
self._active_workspace.mkdir(exist_ok=True)
# Coverage file
logger.info(f"Writing coverage data to '{self._unpacked_coverage_data_file}'.")
with open(self._unpacked_coverage_data_file, "w", newline='\n') as coverage_data:
coverage_data.write(self._historic_data[self.COVERAGE_DATA_KEY])
@@ -117,9 +121,12 @@ class PersistentStorage(ABC):
self._historic_data[self.LAST_COMMIT_HASH_KEY] = self._this_commit_hash
# Last commit hash for this commit
if not self.LAST_RUNS_KEY in self._historic_data:
self._historic_data[self.LAST_RUNS_KEY] = {}
self._historic_data[self.LAST_RUNS_KEY][self._this_commit_hash] = self._last_commit_hash
if not self.HISTORIC_SEQUENCES_KEY in self._historic_data:
self._historic_data[self.HISTORIC_SEQUENCES_KEY] = {}
self._historic_data[self.HISTORIC_SEQUENCES_KEY][self._this_commit_hash] = self._last_commit_hash
# Test runs for this completed sequence
self._historic_data[self.PREVIOUS_TEST_RUNS_KEY] = test_runs
# Coverage data for this branch
with open(self._unpacked_coverage_data_file, "r") as coverage_data:
scripts/build/TestImpactAnalysis/tiaf_persistent_storage_local.py
+2
View File
@@ -32,10 +32,12 @@ class PersistentStorageLocal(PersistentStorage):
try:
# Attempt to obtain the local persistent data location specified in the runtime config file
self._historic_workspace = pathlib.Path(config[self.WORKSPACE_KEY][self.HISTORIC_KEY][self.ROOT_KEY])
self._historic_workspace = self._historic_workspace.joinpath(pathlib.Path(self._suite))
historic_data_file = pathlib.Path(config[self.WORKSPACE_KEY][self.HISTORIC_KEY][self.RELATIVE_PATHS_KEY][self.DATA_KEY])
# Attempt to unpack the local historic data file
self._historic_data_file = self._historic_workspace.joinpath(historic_data_file)
logger.info(f"Attempting to retrieve historic data at location '{self._historic_data_file}'...")
if self._historic_data_file.is_file():
with open(self._historic_data_file, "r") as historic_data_raw:
historic_data_json = historic_data_raw.read()
scripts/build/TestImpactAnalysis/tiaf_persistent_storage_s3.py
+3 -3
View File
@@ -43,9 +43,9 @@ class PersistentStorageS3(PersistentStorage):
# historic_data.json.zip is the file containing the coverage and meta-data of the last TIAF sequence run
historic_data_file = f"historic_data.{object_extension}"
# The location of the data is in the form <root_dir>/<branch>/<config> so the build config of each branch gets its own historic data
self._historic_data_dir = f'{root_dir}/{branch}/{config[self.META_KEY][self.BUILD_CONFIG_KEY]}'
self._historic_data_key = f'{self._historic_data_dir}/{historic_data_file}'
# The location of the data is in the form <root_dir>/<branch>/<config>/<suite> so the build config of each branch gets its own historic data
self._historic_data_dir = f"{root_dir}/{branch}/{config[self.META_KEY][self.BUILD_CONFIG_KEY]}/{self._suite}"
self._historic_data_key = f"{self._historic_data_dir}/{historic_data_file}"
logger.info(f"Attempting to retrieve historic data for branch '{branch}' at location '{self._historic_data_key}' on bucket '{s3_bucket}'...")
self._s3 = boto3.resource("s3")