Merge pull request #143 from aws-lumberyard-dev/build_strike/SPEC-6433
Add repository name to EBS volume tag
This commit is contained in:
Shirang Jia
GitHub
Vendored
+9
-6
@@ -268,7 +268,7 @@ def CheckoutRepo(boolean disableSubmodules = false) {
|
||||
palRm('commitid')
|
||||
}
|
||||
|
||||
def PreBuildCommonSteps(Map pipelineConfig, String projectName, String pipeline, String branchName, String platform, String buildType, String workspace, boolean mount = true, boolean disableSubmodules = false) {
|
||||
def PreBuildCommonSteps(Map pipelineConfig, String repositoryName, String projectName, String pipeline, String branchName, String platform, String buildType, String workspace, boolean mount = true, boolean disableSubmodules = false) {
|
||||
echo 'Starting pre-build common steps...'
|
||||
|
||||
if (mount) {
|
||||
@@ -279,10 +279,10 @@ def PreBuildCommonSteps(Map pipelineConfig, String projectName, String pipeline,
|
||||
else pythonCmd = 'python -u '
|
||||
|
||||
if(env.RECREATE_VOLUME?.toBoolean()) {
|
||||
palSh("${pythonCmd} ${INCREMENTAL_BUILD_SCRIPT_PATH} --action delete --project ${projectName} --pipeline ${pipeline} --branch ${branchName} --platform ${platform} --build_type ${buildType}", 'Deleting volume')
|
||||
palSh("${pythonCmd} ${INCREMENTAL_BUILD_SCRIPT_PATH} --action delete --repository_name ${repositoryName} --project ${projectName} --pipeline ${pipeline} --branch ${branchName} --platform ${platform} --build_type ${buildType}", 'Deleting volume', winSlashReplacement=false)
|
||||
}
|
||||
timeout(5) {
|
||||
palSh("${pythonCmd} ${INCREMENTAL_BUILD_SCRIPT_PATH} --action mount --project ${projectName} --pipeline ${pipeline} --branch ${branchName} --platform ${platform} --build_type ${buildType}", 'Mounting volume')
|
||||
palSh("${pythonCmd} ${INCREMENTAL_BUILD_SCRIPT_PATH} --action mount --repository_name ${repositoryName} --project ${projectName} --pipeline ${pipeline} --branch ${branchName} --platform ${platform} --build_type ${buildType}", 'Mounting volume', winSlashReplacement=false)
|
||||
}
|
||||
|
||||
if(env.IS_UNIX) {
|
||||
@@ -386,10 +386,10 @@ def PostBuildCommonSteps(String workspace, boolean mount = true) {
|
||||
}
|
||||
}
|
||||
|
||||
def CreateSetupStage(Map pipelineConfig, String projectName, String pipelineName, String branchName, String platformName, String jobName, Map environmentVars) {
|
||||
def CreateSetupStage(Map pipelineConfig, String repositoryName, String projectName, String pipelineName, String branchName, String platformName, String jobName, Map environmentVars) {
|
||||
return {
|
||||
stage("Setup") {
|
||||
PreBuildCommonSteps(pipelineConfig, projectName, pipelineName, branchName, platformName, jobName, environmentVars['WORKSPACE'], environmentVars['MOUNT_VOLUME'])
|
||||
PreBuildCommonSteps(pipelineConfig, repositoryName, projectName, pipelineName, branchName, platformName, jobName, environmentVars['WORKSPACE'], environmentVars['MOUNT_VOLUME'])
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -433,6 +433,9 @@ try {
|
||||
}
|
||||
withEnv(envVarList) {
|
||||
timestamps {
|
||||
repositoryUrl = scm.getUserRemoteConfigs()[0].getUrl()
|
||||
// repositoryName is the full repository name
|
||||
repositoryName = (repositoryUrl =~ /https:\/\/github.com\/(.*)\.git/)[0][1]
|
||||
(projectName, pipelineName) = GetRunningPipelineName(env.JOB_NAME) // env.JOB_NAME is the name of the job given by Jenkins
|
||||
|
||||
if(env.BRANCH_NAME) {
|
||||
@@ -500,7 +503,7 @@ try {
|
||||
try {
|
||||
def build_job_name = build_job.key
|
||||
|
||||
CreateSetupStage(pipelineConfig, projectName, pipelineName, branchName, platform.key, build_job.key, envVars).call()
|
||||
CreateSetupStage(pipelineConfig, repositoryName, projectName, pipelineName, branchName, platform.key, build_job.key, envVars).call()
|
||||
|
||||
if(build_job.value.steps) { //this is a pipe with many steps so create all the build stages
|
||||
build_job.value.steps.each { build_step ->
|
||||
|
||||
@@ -94,7 +94,8 @@ def error(message):
|
||||
def parse_args():
|
||||
parser = argparse.ArgumentParser()
|
||||
parser.add_argument('-a', '--action', dest="action", help="Action (mount|unmount|delete)")
|
||||
parser.add_argument('-proj', '--project', dest="project", help="Project")
|
||||
parser.add_argument('-repository_name', '--repository_name', dest="repository_name", help="Repository name")
|
||||
parser.add_argument('-project', '--project', dest="project", help="Project")
|
||||
parser.add_argument('-pipe', '--pipeline', dest="pipeline", help="Pipeline")
|
||||
parser.add_argument('-b', '--branch', dest="branch", help="Branch")
|
||||
parser.add_argument('-plat', '--platform', dest="platform", help="Platform")
|
||||
@@ -108,6 +109,8 @@ def parse_args():
|
||||
error('No action specified')
|
||||
args.action = args.action.lower()
|
||||
if args.action != 'unmount':
|
||||
if args.repository_name is None:
|
||||
error('No repository specified')
|
||||
if args.project is None:
|
||||
error('No project specified')
|
||||
if args.pipeline is None:
|
||||
@@ -121,8 +124,8 @@ def parse_args():
|
||||
|
||||
return args
|
||||
|
||||
def get_mount_name(project, pipeline, branch, platform, build_type):
|
||||
mount_name = "{}_{}_{}_{}_{}".format(project, pipeline, branch, platform, build_type)
|
||||
def get_mount_name(repository_name, project, pipeline, branch, platform, build_type):
|
||||
mount_name = "{}_{}_{}_{}_{}_{}".format(repository_name, project, pipeline, branch, platform, build_type)
|
||||
mount_name = mount_name.replace('/','_').replace('\\','_')
|
||||
return mount_name
|
||||
|
||||
@@ -174,8 +177,8 @@ def delete_volume(ec2_client, volume_id):
|
||||
response = ec2_client.delete_volume(VolumeId=volume_id)
|
||||
print 'Volume {} deleted'.format(volume_id)
|
||||
|
||||
def find_snapshot_id(ec2_client, project, pipeline, platform, build_type, disk_size):
|
||||
mount_name = get_mount_name(project, pipeline, 'main', platform, build_type) # we take snapshots out of main
|
||||
def find_snapshot_id(ec2_client, repository_name, project, pipeline, platform, build_type, disk_size):
|
||||
mount_name = get_mount_name(repository_name, project, pipeline, 'main', platform, build_type) # we take snapshots out of main
|
||||
response = ec2_client.describe_snapshots(Filters= [{
|
||||
'Name': 'tag:Name', 'Values': [mount_name]
|
||||
}])
|
||||
@@ -191,9 +194,9 @@ def find_snapshot_id(ec2_client, project, pipeline, platform, build_type, disk_s
|
||||
snapshot_id = snapshot['SnapshotId']
|
||||
return snapshot_id
|
||||
|
||||
def create_volume(ec2_client, availability_zone, project, pipeline, branch, platform, build_type, disk_size, disk_type):
|
||||
def create_volume(ec2_client, availability_zone, repository_name, project, pipeline, branch, platform, build_type, disk_size, disk_type):
|
||||
# The actual EBS default calculation for IOps is a floating point number, the closest approxmiation is 4x of the disk size for simplicity
|
||||
mount_name = get_mount_name(project, pipeline, branch, platform, build_type)
|
||||
mount_name = get_mount_name(repository_name, project, pipeline, branch, platform, build_type)
|
||||
pipeline_and_branch = get_pipeline_and_branch(pipeline, branch)
|
||||
parameters = dict(
|
||||
AvailabilityZone = availability_zone,
|
||||
@@ -202,6 +205,7 @@ def create_volume(ec2_client, availability_zone, project, pipeline, branch, plat
|
||||
'ResourceType': 'volume',
|
||||
'Tags': [
|
||||
{ 'Key': 'Name', 'Value': mount_name },
|
||||
{ 'Key': 'RepositoryName', 'Value': repository_name},
|
||||
{ 'Key': 'Project', 'Value': project },
|
||||
{ 'Key': 'Pipeline', 'Value': pipeline },
|
||||
{ 'Key': 'BranchName', 'Value': branch },
|
||||
@@ -214,7 +218,7 @@ def create_volume(ec2_client, availability_zone, project, pipeline, branch, plat
|
||||
if 'io1' in disk_type.lower():
|
||||
parameters['Iops'] = (4 * disk_size)
|
||||
|
||||
snapshot_id = find_snapshot_id(ec2_client, project, pipeline, platform, build_type, disk_size)
|
||||
snapshot_id = find_snapshot_id(ec2_client, repository_name, project, pipeline, platform, build_type, disk_size)
|
||||
if snapshot_id:
|
||||
parameters['SnapshotId'] = snapshot_id
|
||||
created = False
|
||||
@@ -234,8 +238,8 @@ def create_volume(ec2_client, availability_zone, project, pipeline, branch, plat
|
||||
time.sleep(1)
|
||||
response = ec2_client.describe_volumes(VolumeIds=[volume_id, ])
|
||||
|
||||
print("Volume {} created\n\tSnapshot: {}\n\tProject {}\n\tPipeline {}\n\tBranch {}\n\tPlatform: {}\n\tBuild type: {}"
|
||||
.format(volume_id, snapshot_id, project, pipeline, branch, platform, build_type))
|
||||
print("Volume {} created\n\tSnapshot: {}\n\tRepository {}\n\tProject {}\n\tPipeline {}\n\tBranch {}\n\tPlatform: {}\n\tBuild type: {}"
|
||||
.format(volume_id, snapshot_id, repository_name, project, pipeline, branch, platform, build_type))
|
||||
return volume_id, created
|
||||
|
||||
|
||||
@@ -359,7 +363,7 @@ def attach_ebs_and_create_partition_with_retry(volume, volume_id, ec2_instance_i
|
||||
mount_volume(created)
|
||||
attempt += 1
|
||||
|
||||
def mount_ebs(project, pipeline, branch, platform, build_type, disk_size, disk_type):
|
||||
def mount_ebs(repository_name, project, pipeline, branch, platform, build_type, disk_size, disk_type):
|
||||
session = boto3.session.Session()
|
||||
region = session.region_name
|
||||
if region is None:
|
||||
@@ -379,7 +383,7 @@ def mount_ebs(project, pipeline, branch, platform, build_type, disk_size, disk_t
|
||||
unmount_volume()
|
||||
detach_volume(volume, ec2_instance_id, False) # Force unmounts should not be used, as that will cause the EBS block device driver to fail the remount
|
||||
|
||||
mount_name = get_mount_name(project, pipeline, branch, platform, build_type)
|
||||
mount_name = get_mount_name(repository_name, project, pipeline, branch, platform, build_type)
|
||||
response = ec2_client.describe_volumes(Filters=[{
|
||||
'Name': 'tag:Name', 'Values': [mount_name]
|
||||
}])
|
||||
@@ -388,7 +392,7 @@ def mount_ebs(project, pipeline, branch, platform, build_type, disk_size, disk_t
|
||||
if 'Volumes' in response and not len(response['Volumes']):
|
||||
print 'Volume for {} doesn\'t exist creating it...'.format(mount_name)
|
||||
# volume doesn't exist, create it
|
||||
volume_id, created = create_volume(ec2_client, ec2_availability_zone, project, pipeline, branch, platform, build_type, disk_size, disk_type)
|
||||
volume_id, created = create_volume(ec2_client, ec2_availability_zone, repository_name, project, pipeline, branch, platform, build_type, disk_size, disk_type)
|
||||
else:
|
||||
volume = response['Volumes'][0]
|
||||
volume_id = volume['VolumeId']
|
||||
@@ -396,7 +400,7 @@ def mount_ebs(project, pipeline, branch, platform, build_type, disk_size, disk_t
|
||||
if (volume['Size'] != disk_size or volume['VolumeType'] != disk_type):
|
||||
print 'Override disk attributes does not match the existing volume, deleting {} and replacing the volume'.format(volume_id)
|
||||
delete_volume(ec2_client, volume_id)
|
||||
volume_id, created = create_volume(ec2_client, ec2_availability_zone, project, pipeline, branch, platform, build_type, disk_size, disk_type)
|
||||
volume_id, created = create_volume(ec2_client, ec2_availability_zone, repository_name, project, pipeline, branch, platform, build_type, disk_size, disk_type)
|
||||
if len(volume['Attachments']):
|
||||
# this is bad we shouldn't be attached, we should have detached at the end of a build
|
||||
attachment = volume['Attachments'][0]
|
||||
@@ -426,7 +430,7 @@ def mount_ebs(project, pipeline, branch, platform, build_type, disk_size, disk_t
|
||||
print 'Error: EBS disk size reached to the allowed maximum disk size {}MB, please contact ly-infra@ and ly-build@ to investigate.'.format(MAX_EBS_DISK_SIZE)
|
||||
exit(1)
|
||||
print 'Recreating the EBS with disk size {}'.format(new_disk_size)
|
||||
volume_id, created = create_volume(ec2_client, ec2_availability_zone, project, pipeline, branch, platform, build_type, new_disk_size, disk_type)
|
||||
volume_id, created = create_volume(ec2_client, ec2_availability_zone, repository_name, project, pipeline, branch, platform, build_type, new_disk_size, disk_type)
|
||||
volume = ec2_resource.Volume(volume_id)
|
||||
attach_ebs_and_create_partition_with_retry(volume, volume_id, ec2_instance_id, created)
|
||||
|
||||
@@ -458,7 +462,7 @@ def unmount_ebs():
|
||||
unmount_volume()
|
||||
detach_volume(volume, ec2_instance_id, False)
|
||||
|
||||
def delete_ebs(project, pipeline, branch, platform, build_type):
|
||||
def delete_ebs(repository_name, project, pipeline, branch, platform, build_type):
|
||||
unmount_ebs()
|
||||
|
||||
session = boto3.session.Session()
|
||||
@@ -470,7 +474,7 @@ def delete_ebs(project, pipeline, branch, platform, build_type):
|
||||
ec2_resource = boto3.resource('ec2', region_name=region)
|
||||
ec2_instance = ec2_resource.Instance(ec2_instance_id)
|
||||
|
||||
mount_name = get_mount_name(project, pipeline, branch, platform, build_type)
|
||||
mount_name = get_mount_name(repository_name, project, pipeline, branch, platform, build_type)
|
||||
response = ec2_client.describe_volumes(Filters=[
|
||||
{ 'Name': 'tag:Name', 'Values': [mount_name] }
|
||||
])
|
||||
@@ -481,15 +485,15 @@ def delete_ebs(project, pipeline, branch, platform, build_type):
|
||||
delete_volume(ec2_client, volume_id)
|
||||
|
||||
|
||||
def main(action, project, pipeline, branch, platform, build_type, disk_size, disk_type):
|
||||
def main(action, repository_name, project, pipeline, branch, platform, build_type, disk_size, disk_type):
|
||||
if action == 'mount':
|
||||
mount_ebs(project, pipeline, branch, platform, build_type, disk_size, disk_type)
|
||||
mount_ebs(repository_name, project, pipeline, branch, platform, build_type, disk_size, disk_type)
|
||||
elif action == 'unmount':
|
||||
unmount_ebs()
|
||||
elif action == 'delete':
|
||||
delete_ebs(project, pipeline, branch, platform, build_type)
|
||||
delete_ebs(repository_name, project, pipeline, branch, platform, build_type)
|
||||
|
||||
if __name__ == "__main__":
|
||||
args = parse_args()
|
||||
ret = main(args.action, args.project, args.pipeline, args.branch, args.platform, args.build_type, args.disk_size, args.disk_type)
|
||||
ret = main(args.action, args.repository_name, args.project, args.pipeline, args.branch, args.platform, args.build_type, args.disk_size, args.disk_type)
|
||||
sys.exit(ret)
|
||||
Reference in New Issue
Block a user