From 6eec5e1a21b75aaabede378649964a037cfc44f4 Mon Sep 17 00:00:00 2001 From: Mike Chang Date: Wed, 26 Jan 2022 13:00:46 -0800 Subject: [PATCH] EBS snapshot script (#6978) Adding snapshot script, modify Jenkinsfile and build_config.json for each platform to use the snapshot tag Signed-off-by: Mike Chang --- scripts/build/Jenkins/Jenkinsfile | 32 ++- .../build/Platform/Android/build_config.json | 9 +- scripts/build/Platform/Android/pipeline.json | 6 +- .../build/Platform/Linux/build_config.json | 3 +- scripts/build/Platform/Linux/pipeline.json | 4 + .../build/Platform/Windows/build_config.json | 9 +- scripts/build/Platform/Windows/pipeline.json | 6 +- scripts/build/tools/ebs_snapshot.py | 197 ++++++++++++++++++ 8 files changed, 255 insertions(+), 11 deletions(-) create mode 100644 scripts/build/tools/ebs_snapshot.py diff --git a/scripts/build/Jenkins/Jenkinsfile b/scripts/build/Jenkins/Jenkinsfile index b55689b871..e4957992a5 100644 --- a/scripts/build/Jenkins/Jenkinsfile +++ b/scripts/build/Jenkins/Jenkinsfile @@ -9,6 +9,7 @@ import groovy.json.JsonOutput PIPELINE_CONFIG_FILE = 'scripts/build/Jenkins/lumberyard.json' INCREMENTAL_BUILD_SCRIPT_PATH = 'scripts/build/bootstrap/incremental_build_util.py' +EBS_SNAPSHOT_SCRIPT_PATH = 'scripts/build/tools/ebs_snapshot.py' PIPELINE_RETRY_ATTEMPTS = 3 EMPTY_JSON = readJSON text: '{}' @@ -206,7 +207,8 @@ def CheckoutBootstrapScripts(String branchName) { [$class: 'SparseCheckoutPaths', sparseCheckoutPaths: [ [ $class: 'SparseCheckoutPath', path: 'scripts/build/Jenkins/' ], [ $class: 'SparseCheckoutPath', path: 'scripts/build/bootstrap/' ], - [ $class: 'SparseCheckoutPath', path: 'scripts/build/Platform' ] + [ $class: 'SparseCheckoutPath', path: 'scripts/build/Platform' ], + [ $class: 'SparseCheckoutPath', path: 'scripts/build/tools/' ] ]], // Shallow checkouts break changelog computation. Do not enable. [$class: 'CloneOption', noTags: false, reference: '', shallow: false] @@ -495,6 +497,19 @@ def PostBuildCommonSteps(String workspace, Map params, boolean mount = true) { } } +def HandleDriveSnapshots(String repositoryName, String projectName, String pipeline, String branchName, String platform, String buildType) { + unstash name: 'ebs_snapshot_script' + + catchError(message: "Error snapshotting volume (this won't fail the build)", buildResult: 'UNSTABLE', stageResult: 'FAILURE') { + def pythonCmd = 'python3 -u ' + + mountName = "Name:${repositoryName}_${projectName}_${pipeline}_${branchName}_${platform}_${buildType}" + mountName = mountName.replace('/', '_').replace('\\', '_') + palSh("${pythonCmd} ${EBS_SNAPSHOT_SCRIPT_PATH} --action create --tags ${mountName} --execute", "Starting volume snapshots", true) + palSh("${pythonCmd} ${EBS_SNAPSHOT_SCRIPT_PATH} --action delete --tags ${mountName} --retention ${env.SNAP_RETENTION} --execute", "Cleaning up old snapshots", true) + } +} + def CreateSetupStage(Map pipelineConfig, String snapshot, String repositoryName, String projectName, String pipelineName, String branchName, String platformName, String jobName, Map environmentVars, boolean onlyMountEBSVolume = false) { return { stage('Setup') { @@ -564,6 +579,14 @@ def CreateTeardownStage(Map environmentVars, Map params) { } } +def CreateSnapshotStage(String repositoryName, String projectName, String pipelineName, String branchName, String platformName, String buildType, String jobName) { + return{ + stage("${jobName}_snapshot_ebs_volume") { + HandleDriveSnapshots(repositoryName, projectName, pipelineName, branchName, platformName, buildType) + } + } +} + def CreateSingleNode(Map pipelineConfig, def platform, def build_job, Map envVars, String branchName, String pipelineName, String repositoryName, String projectName, boolean onlyMountEBSVolume = false) { def nodeLabel = envVars['NODE_LABEL'] return { @@ -632,6 +655,9 @@ def CreateSingleNode(Map pipelineConfig, def platform, def build_job, Map envVar CreateExportTestScreenshotsStage(pipelineConfig, branchName, platform.key, build_job_name, envVars, params).call() } CreateTeardownStage(envVars, params).call() + if (envVars['CREATE_SNAPSHOT']?.toBoolean()) { + CreateSnapshotStage(repositoryName, projectName, pipelineName, branchName, platform.key, build_job.key, build_job_name).call() + } } } } @@ -816,9 +842,11 @@ try { pipelineProperties.add(parameters(pipelineParameters.unique())) properties(pipelineProperties) - // Stash the INCREMENTAL_BUILD_SCRIPT_PATH since all nodes will use it + // Stash the INCREMENTAL_BUILD_SCRIPT_PATH and EBS_SNAPSHOT_SCRIPT_PATH since all nodes will use it stash name: 'incremental_build_script', includes: INCREMENTAL_BUILD_SCRIPT_PATH + stash name: 'ebs_snapshot_script', + includes: EBS_SNAPSHOT_SCRIPT_PATH } } } diff --git a/scripts/build/Platform/Android/build_config.json b/scripts/build/Platform/Android/build_config.json index da538b22ec..e31aa4d5a0 100644 --- a/scripts/build/Platform/Android/build_config.json +++ b/scripts/build/Platform/Android/build_config.json @@ -9,7 +9,8 @@ }, "profile_pipe": { "TAGS": [ - "default" + "default", + "snapshot" ], "steps": [ "profile" @@ -77,7 +78,8 @@ "default", "weekly-build-metrics", "nightly-incremental", - "nightly-clean" + "nightly-clean", + "snapshot" ], "COMMAND":"../Windows/build_asset_windows.cmd", "PARAMETERS": { @@ -127,7 +129,8 @@ "gradle": { "TAGS":[ "default", - "weekly-build-metrics" + "weekly-build-metrics", + "snapshot" ], "COMMAND":"gradle_windows.cmd", "PARAMETERS": { diff --git a/scripts/build/Platform/Android/pipeline.json b/scripts/build/Platform/Android/pipeline.json index 2e426c7891..ca3a9f998e 100644 --- a/scripts/build/Platform/Android/pipeline.json +++ b/scripts/build/Platform/Android/pipeline.json @@ -14,6 +14,10 @@ }, "nightly-clean": { "CLEAN_WORKSPACE": true + }, + "snapshot": { + "CLEAN_WORKSPACE": true, + "CREATE_SNAPSHOT": true } } -} \ No newline at end of file +} diff --git a/scripts/build/Platform/Linux/build_config.json b/scripts/build/Platform/Linux/build_config.json index c8f8752609..c7c20dca28 100644 --- a/scripts/build/Platform/Linux/build_config.json +++ b/scripts/build/Platform/Linux/build_config.json @@ -9,7 +9,8 @@ }, "profile_nounity_pipe": { "TAGS": [ - "default" + "default", + "snapshot" ], "steps": [ "profile_nounity", diff --git a/scripts/build/Platform/Linux/pipeline.json b/scripts/build/Platform/Linux/pipeline.json index 605adf1837..e12d41294b 100644 --- a/scripts/build/Platform/Linux/pipeline.json +++ b/scripts/build/Platform/Linux/pipeline.json @@ -12,6 +12,10 @@ }, "nightly-clean": { "CLEAN_WORKSPACE": true + }, + "snapshot": { + "CLEAN_WORKSPACE": true, + "CREATE_SNAPSHOT": true } }, "PIPELINE_JENKINS_PARAMETERS": { diff --git a/scripts/build/Platform/Windows/build_config.json b/scripts/build/Platform/Windows/build_config.json index e3d5a4a3fc..017f56bb68 100644 --- a/scripts/build/Platform/Windows/build_config.json +++ b/scripts/build/Platform/Windows/build_config.json @@ -9,7 +9,8 @@ }, "validation_pipe": { "TAGS": [ - "default" + "default", + "snapshot" ], "steps": [ "validation" @@ -27,7 +28,8 @@ }, "profile_pipe": { "TAGS": [ - "default" + "default", + "snapshot" ], "steps": [ "profile", @@ -288,7 +290,8 @@ "default", "nightly-incremental", "nightly-clean", - "weekly-build-metrics" + "weekly-build-metrics", + "snapshot" ], "COMMAND": "build_windows.cmd", "PARAMETERS": { diff --git a/scripts/build/Platform/Windows/pipeline.json b/scripts/build/Platform/Windows/pipeline.json index 47e6af2d10..3b93da9713 100644 --- a/scripts/build/Platform/Windows/pipeline.json +++ b/scripts/build/Platform/Windows/pipeline.json @@ -12,6 +12,10 @@ }, "nightly-clean": { "CLEAN_WORKSPACE": true + }, + "snapshot": { + "CLEAN_WORKSPACE": true, + "CREATE_SNAPSHOT": true } }, "PIPELINE_JENKINS_PARAMETERS": { @@ -62,4 +66,4 @@ } ] } -} \ No newline at end of file +} diff --git a/scripts/build/tools/ebs_snapshot.py b/scripts/build/tools/ebs_snapshot.py new file mode 100644 index 0000000000..483bceb988 --- /dev/null +++ b/scripts/build/tools/ebs_snapshot.py @@ -0,0 +1,197 @@ +# +# Copyright (c) Contributors to the Open 3D Engine Project. +# For complete copyright and license terms please see the LICENSE at the root of this distribution. +# +# SPDX-License-Identifier: Apache-2.0 OR MIT +# +# + +import argparse +import boto3 +import logging +import sys +from botocore.config import Config + +log = logging.getLogger(__name__) +log.setLevel(logging.INFO) +log.addHandler(logging.StreamHandler()) + +DEFAULT_REGION = 'us-west-2' +DEFAULT_SNAPSHOT_RETAIN = 2 +DEFAULT_SNAPSHOT_DESCRIPTION = 'Created for Build Artifact Snapshots' +DEFAULT_DRYRUN = True + +def _kv_to_dict(kv_string): + """ + Simple splitting of a key value string to dictionary in "Name: , Values: []" form + + :param kv_string: String in the form of "key:value" + :return Dictionary of values + """ + dict = {} + + if ":" not in kv_string: + log.error(f'Keyvalue parameter not in the form of "key:value"') + raise ValueError + + kv = kv_string.split(':') + dict['Name'] = f'tag:{kv[0]}' + dict['Values'] = [kv[1]] + + return dict + +def _format_tags(tag_keyvalue): + """ + Format tags in list form + + :param tag_keyvalue: String of comma separated key value pairs + :return List of dictionary values + """ + tag_filter = [] + + for keyvalue in tag_keyvalue: + tag_filter.append(_kv_to_dict(keyvalue)) + + return tag_filter + +def get_ec2_resource(): + """ + Get the AWS EC2 resource object, with appropriate region + + :return The EC2 resource object + """ + session = boto3.session.Session() + region = session.region_name + if region is None: + region = DEFAULT_REGION + + resource_config = Config( + region_name=region, + retries={ + 'mode': 'standard' + } + ) + resource = boto3.resource('ec2', config=resource_config) + return resource + +def create_snapshot(ec2_resource, tag_keyvalue, snap_description, snap_dryrun=DEFAULT_DRYRUN): + """ + Find and snapshot all EBS volumes that have a matching tag value. Injects all volume tags into the snapshot, + including the name and adds a description + + :param ec2_resource: The EC2 resource object + :param tag_keyvalue: List of Strings with tag keyvalues in the form of "key:value" + :param snap_description: String with the snapshot description to write + :param snap_dryrun: Boolean to dryrun the action. Set to true by default (always dryrun) + :return: Number of EBS volumes that are snapshotted successfully, number of EBS volumes that failed to be snapshotted + """ + success = 0 + failure = 0 + + tags = _format_tags(tag_keyvalue) + + for tag in tags: + response = ec2_resource.volumes.filter(Filters=[tag]) + log.info(f'Snapshotting EBS volumes with tags that match {tag}...') + for volume in response: + try: + log.info(f'Snapshotting volume {volume.volume_id}') + volume.create_snapshot(Description=snap_description, TagSpecifications=[{'ResourceType': 'snapshot', 'Tags': volume.tags}], DryRun=snap_dryrun) + success += 1 + except Exception as e: + log.error(f'Failed to snapshot volume {volume.volume_id}.') + log.error(e) + failure += 1 + + return success, failure + +def delete_snapshot(ec2_resource, tag_keyvalue, snap_description, snap_retention, snap_dryrun=DEFAULT_DRYRUN): + """ + Find all EBS snapshots that have a matching tag value AND description. If the number of snapshots exceeds a retention amount, + delete the oldest snapshot until retention is achived. + + :param ec2_resource: The EC2 resource object + :param tag_keyvalue: List of Strings with tag keyvalues in the form of "key:value" + :param snap_description: String with the snapshot description to search + :param snap_retention: Integer with the number of snapshots to retain + :param snap_dryrun: Boolean to dryrun the action. Set to true by default (always dryrun) + :return: Number of EBS snapshots deleted successfully, number of EBS snapshots that failed to be deleted + """ + success = 0 + failure = 0 + description_filter = {"Name": "description", "Values": [snap_description]} + + tags = _format_tags(tag_keyvalue) + + for tag in tags: + response = list(ec2_resource.snapshots.filter(Filters=[tag,description_filter])) + log.info(f'Getting snapshots with tags that match {tag}...') + num_snaps = len(response) + log.info(f'Tag {tag} has {num_snaps} snapshots') + + if num_snaps > snap_retention: + log.info(f'Deleting oldest snapshots to keep retention of {snap_retention}') + snap_list = sorted(response, key=lambda k: k.start_time) # Get a sorted list of snapshots by start time in descending order + diff_snap = num_snaps - snap_retention + for n in range(diff_snap): + try: + log.info(f'Deleting snapshot {snap_list[n].snapshot_id}') + snap_list[n].delete(DryRun=snap_dryrun) + success += 1 + except Exception as e: + log.error(f'Failed to delete snapshot {snap_list[n].snapshot_id}.') + log.error(e) + failure += 1 + + return success, failure + +def list_snapshot(ec2_resource, tag_keyvalue, snap_description): + """ + Find all EBS snapshots that have a matching tag value AND description. Prints snap id, description, tags, and start time. + + :param ec2_resource: The EC2 resource object + :param tag_keyvalue: List of Strings with tag keyvalues in the form of "key:value" + :param snap_description: String with the snapshot description to search + :return: None + """ + + description_filter = {"Name": "description", "Values": [snap_description]} + + tags = _format_tags(tag_keyvalue) + + for tag in tags: + response = ec2_resource.snapshots.filter(Filters=[tag,description_filter]) + log.info(f'Getting snapshots with tags that match {tag}...') + num_snaps = len(list(response)) + log.info(f'Tag {tag} has {num_snaps} snapshots') + snap_list = sorted(response, key=lambda k: k.start_time) + for n in range(num_snaps): + print(f'Snap ID: {snap_list[n].snapshot_id} \n Description: {snap_list[n].description} \n Tags: {snap_list[n].tags} \n Start Time: {snap_list[n].start_time}') + + return None + +def parse_args(): + parser = argparse.ArgumentParser(description='Script to manage EBS snapshots for build artifacts') + parser.add_argument('--action', '-a', type=str, help='(create|delete|list) Creates, deletes, or lists EBS snapshots based on tag. Requires --tags argument') + parser.add_argument('--tags', '-t', type=str, required=True, help='Comma separated key value tags to search for in the form of "key:value", for example, "PipelineAndBranch:default_development","PipelineAndBranch:default_development"') + parser.add_argument('--description', '-d', default=DEFAULT_SNAPSHOT_DESCRIPTION, help=f'Snapshot description to write or search for. Defaults to "{DEFAULT_SNAPSHOT_DESCRIPTION}"') + parser.add_argument('--retention', '-r', nargs="?", const=DEFAULT_SNAPSHOT_RETAIN, type=int, help=f'Integer with the number of snapshots to retain. Defaults to {DEFAULT_SNAPSHOT_RETAIN}') + parser.add_argument('--execute', '-e', action='store_false', help=f'Execute the snapshot commands. This needs to be set, otherwise it will always dryrun') + return parser.parse_args() + +def main(): + args = parse_args() + tag_list = args.tags.split(",") + ec2_resource = get_ec2_resource() + if 'create' in args.action: + ret = create_snapshot(ec2_resource, tag_list, args.description, args.execute) + log.info(f'{ret[0]} snapshots created, {ret[1]} snapshots failed') + elif 'delete' in args.action: + ret = delete_snapshot(ec2_resource, tag_list, args.description, args.retention, args.execute) + log.info(f'{ret[0]} snapshots deleted, {ret[1]} snapshot deletions failed') + elif 'list' in args.action: + ret = list_snapshot(ec2_resource, tag_list, args.description) + +if __name__ == "__main__": + sys.exit(main()) + \ No newline at end of file