From a6cea546da84e079a77e1525c7960f55220ba74d Mon Sep 17 00:00:00 2001 From: shiranj Date: Mon, 19 Apr 2021 14:52:37 -0700 Subject: [PATCH 1/4] Add repository name to EBS volume tag --- scripts/build/Jenkins/Jenkinsfile | 15 ++++--- .../build/bootstrap/incremental_build_util.py | 44 ++++++++++--------- 2 files changed, 33 insertions(+), 26 deletions(-) diff --git a/scripts/build/Jenkins/Jenkinsfile b/scripts/build/Jenkins/Jenkinsfile index f77c139342..7f269c0012 100644 --- a/scripts/build/Jenkins/Jenkinsfile +++ b/scripts/build/Jenkins/Jenkinsfile @@ -265,7 +265,7 @@ def CheckoutRepo(boolean disableSubmodules = false) { palRm('commitid') } -def PreBuildCommonSteps(Map pipelineConfig, String projectName, String pipeline, String branchName, String platform, String buildType, String workspace, boolean mount = true, boolean disableSubmodules = false) { +def PreBuildCommonSteps(Map pipelineConfig, String repositoryName, String projectName, String pipeline, String branchName, String platform, String buildType, String workspace, boolean mount = true, boolean disableSubmodules = false) { echo 'Starting pre-build common steps...' if (mount) { @@ -276,10 +276,10 @@ def PreBuildCommonSteps(Map pipelineConfig, String projectName, String pipeline, else pythonCmd = 'python -u ' if(env.RECREATE_VOLUME?.toBoolean()) { - palSh("${pythonCmd} ${INCREMENTAL_BUILD_SCRIPT_PATH} --action delete --project ${projectName} --pipeline ${pipeline} --branch ${branchName} --platform ${platform} --build_type ${buildType}", 'Deleting volume') + palSh("${pythonCmd} ${INCREMENTAL_BUILD_SCRIPT_PATH} --action delete --repository_name ${repositoryName} --project ${projectName} --pipeline ${pipeline} --branch ${branchName} --platform ${platform} --build_type ${buildType}", 'Deleting volume') } timeout(5) { - palSh("${pythonCmd} ${INCREMENTAL_BUILD_SCRIPT_PATH} --action mount --project ${projectName} --pipeline ${pipeline} --branch ${branchName} --platform ${platform} --build_type ${buildType}", 'Mounting volume') + palSh("${pythonCmd} ${INCREMENTAL_BUILD_SCRIPT_PATH} --action mount --repository_name ${repositoryName} --project ${projectName} --pipeline ${pipeline} --branch ${branchName} --platform ${platform} --build_type ${buildType}", 'Mounting volume') } if(env.IS_UNIX) { @@ -383,10 +383,10 @@ def PostBuildCommonSteps(String workspace, boolean mount = true) { } } -def CreateSetupStage(Map pipelineConfig, String projectName, String pipelineName, String branchName, String platformName, String jobName, Map environmentVars) { +def CreateSetupStage(Map pipelineConfig, String repositoryName, String projectName, String pipelineName, String branchName, String platformName, String jobName, Map environmentVars) { return { stage("Setup") { - PreBuildCommonSteps(pipelineConfig, projectName, pipelineName, branchName, platformName, jobName, environmentVars['WORKSPACE'], environmentVars['MOUNT_VOLUME']) + PreBuildCommonSteps(pipelineConfig, repositoryName, projectName, pipelineName, branchName, platformName, jobName, environmentVars['WORKSPACE'], environmentVars['MOUNT_VOLUME']) } } } @@ -430,6 +430,9 @@ try { } withEnv(envVarList) { timestamps { + repositoryUrl = scm.getUserRemoteConfigs()[0].getUrl() + // repositoryName is the full repository name + repositoryName = (repositoryUrl =~ /https:\/\/github.com\/(.*)\.git/)[0][1] (projectName, pipelineName) = GetRunningPipelineName(env.JOB_NAME) // env.JOB_NAME is the name of the job given by Jenkins if(env.BRANCH_NAME) { @@ -493,7 +496,7 @@ try { try { def build_job_name = build_job.key - CreateSetupStage(pipelineConfig, projectName, pipelineName, branchName, platform.key, build_job.key, envVars).call() + CreateSetupStage(pipelineConfig, repositoryName, projectName, pipelineName, branchName, platform.key, build_job.key, envVars).call() if(build_job.value.steps) { //this is a pipe with many steps so create all the build stages build_job.value.steps.each { build_step -> diff --git a/scripts/build/bootstrap/incremental_build_util.py b/scripts/build/bootstrap/incremental_build_util.py index 40b4a5cb4f..fec48e2bee 100755 --- a/scripts/build/bootstrap/incremental_build_util.py +++ b/scripts/build/bootstrap/incremental_build_util.py @@ -94,6 +94,7 @@ def error(message): def parse_args(): parser = argparse.ArgumentParser() parser.add_argument('-a', '--action', dest="action", help="Action (mount|unmount|delete)") + parser.add_argument('-proj', '--repository_name', dest="repository_name", help="Repository name") parser.add_argument('-proj', '--project', dest="project", help="Project") parser.add_argument('-pipe', '--pipeline', dest="pipeline", help="Pipeline") parser.add_argument('-b', '--branch', dest="branch", help="Branch") @@ -108,6 +109,8 @@ def parse_args(): error('No action specified') args.action = args.action.lower() if args.action != 'unmount': + if args.repository_name is None: + error('No repository specified') if args.project is None: error('No project specified') if args.pipeline is None: @@ -121,8 +124,8 @@ def parse_args(): return args -def get_mount_name(project, pipeline, branch, platform, build_type): - mount_name = "{}_{}_{}_{}_{}".format(project, pipeline, branch, platform, build_type) +def get_mount_name(repository_name, project, pipeline, branch, platform, build_type): + mount_name = "{}_{}_{}_{}_{}_{}".format(repository_name, project, pipeline, branch, platform, build_type) mount_name = mount_name.replace('/','_').replace('\\','_') return mount_name @@ -174,8 +177,8 @@ def delete_volume(ec2_client, volume_id): response = ec2_client.delete_volume(VolumeId=volume_id) print 'Volume {} deleted'.format(volume_id) -def find_snapshot_id(ec2_client, project, pipeline, platform, build_type, disk_size): - mount_name = get_mount_name(project, pipeline, 'main', platform, build_type) # we take snapshots out of main +def find_snapshot_id(ec2_client, repository_name, project, pipeline, platform, build_type, disk_size): + mount_name = get_mount_name(repository_name, project, pipeline, 'main', platform, build_type) # we take snapshots out of main response = ec2_client.describe_snapshots(Filters= [{ 'Name': 'tag:Name', 'Values': [mount_name] }]) @@ -191,9 +194,9 @@ def find_snapshot_id(ec2_client, project, pipeline, platform, build_type, disk_s snapshot_id = snapshot['SnapshotId'] return snapshot_id -def create_volume(ec2_client, availability_zone, project, pipeline, branch, platform, build_type, disk_size, disk_type): +def create_volume(ec2_client, availability_zone, repository_name, project, pipeline, branch, platform, build_type, disk_size, disk_type): # The actual EBS default calculation for IOps is a floating point number, the closest approxmiation is 4x of the disk size for simplicity - mount_name = get_mount_name(project, pipeline, branch, platform, build_type) + mount_name = get_mount_name(repository_name, project, pipeline, branch, platform, build_type) pipeline_and_branch = get_pipeline_and_branch(pipeline, branch) parameters = dict( AvailabilityZone = availability_zone, @@ -202,6 +205,7 @@ def create_volume(ec2_client, availability_zone, project, pipeline, branch, plat 'ResourceType': 'volume', 'Tags': [ { 'Key': 'Name', 'Value': mount_name }, + {'Key': 'RepositoryName', 'Value': repository_name}, { 'Key': 'Project', 'Value': project }, { 'Key': 'Pipeline', 'Value': pipeline }, { 'Key': 'BranchName', 'Value': branch }, @@ -214,7 +218,7 @@ def create_volume(ec2_client, availability_zone, project, pipeline, branch, plat if 'io1' in disk_type.lower(): parameters['Iops'] = (4 * disk_size) - snapshot_id = find_snapshot_id(ec2_client, project, pipeline, platform, build_type, disk_size) + snapshot_id = find_snapshot_id(ec2_client, repository_name, project, pipeline, platform, build_type, disk_size) if snapshot_id: parameters['SnapshotId'] = snapshot_id created = False @@ -234,8 +238,8 @@ def create_volume(ec2_client, availability_zone, project, pipeline, branch, plat time.sleep(1) response = ec2_client.describe_volumes(VolumeIds=[volume_id, ]) - print("Volume {} created\n\tSnapshot: {}\n\tProject {}\n\tPipeline {}\n\tBranch {}\n\tPlatform: {}\n\tBuild type: {}" - .format(volume_id, snapshot_id, project, pipeline, branch, platform, build_type)) + print("Volume {} created\n\tSnapshot: {}\n\tRepository {}\n\tProject {}\n\tPipeline {}\n\tBranch {}\n\tPlatform: {}\n\tBuild type: {}" + .format(volume_id, snapshot_id, repository_name, project, pipeline, branch, platform, build_type)) return volume_id, created @@ -359,7 +363,7 @@ def attach_ebs_and_create_partition_with_retry(volume, volume_id, ec2_instance_i mount_volume(created) attempt += 1 -def mount_ebs(project, pipeline, branch, platform, build_type, disk_size, disk_type): +def mount_ebs(repository_name, project, pipeline, branch, platform, build_type, disk_size, disk_type): session = boto3.session.Session() region = session.region_name if region is None: @@ -379,7 +383,7 @@ def mount_ebs(project, pipeline, branch, platform, build_type, disk_size, disk_t unmount_volume() detach_volume(volume, ec2_instance_id, False) # Force unmounts should not be used, as that will cause the EBS block device driver to fail the remount - mount_name = get_mount_name(project, pipeline, branch, platform, build_type) + mount_name = get_mount_name(repository_name, project, pipeline, branch, platform, build_type) response = ec2_client.describe_volumes(Filters=[{ 'Name': 'tag:Name', 'Values': [mount_name] }]) @@ -388,7 +392,7 @@ def mount_ebs(project, pipeline, branch, platform, build_type, disk_size, disk_t if 'Volumes' in response and not len(response['Volumes']): print 'Volume for {} doesn\'t exist creating it...'.format(mount_name) # volume doesn't exist, create it - volume_id, created = create_volume(ec2_client, ec2_availability_zone, project, pipeline, branch, platform, build_type, disk_size, disk_type) + volume_id, created = create_volume(ec2_client, ec2_availability_zone, repository_name, project, pipeline, branch, platform, build_type, disk_size, disk_type) else: volume = response['Volumes'][0] volume_id = volume['VolumeId'] @@ -396,7 +400,7 @@ def mount_ebs(project, pipeline, branch, platform, build_type, disk_size, disk_t if (volume['Size'] != disk_size or volume['VolumeType'] != disk_type): print 'Override disk attributes does not match the existing volume, deleting {} and replacing the volume'.format(volume_id) delete_volume(ec2_client, volume_id) - volume_id, created = create_volume(ec2_client, ec2_availability_zone, project, pipeline, branch, platform, build_type, disk_size, disk_type) + volume_id, created = create_volume(ec2_client, ec2_availability_zone, repository_name, project, pipeline, branch, platform, build_type, disk_size, disk_type) if len(volume['Attachments']): # this is bad we shouldn't be attached, we should have detached at the end of a build attachment = volume['Attachments'][0] @@ -426,7 +430,7 @@ def mount_ebs(project, pipeline, branch, platform, build_type, disk_size, disk_t print 'Error: EBS disk size reached to the allowed maximum disk size {}MB, please contact ly-infra@ and ly-build@ to investigate.'.format(MAX_EBS_DISK_SIZE) exit(1) print 'Recreating the EBS with disk size {}'.format(new_disk_size) - volume_id, created = create_volume(ec2_client, ec2_availability_zone, project, pipeline, branch, platform, build_type, new_disk_size, disk_type) + volume_id, created = create_volume(ec2_client, ec2_availability_zone, repository_name, project, pipeline, branch, platform, build_type, new_disk_size, disk_type) volume = ec2_resource.Volume(volume_id) attach_ebs_and_create_partition_with_retry(volume, volume_id, ec2_instance_id, created) @@ -458,7 +462,7 @@ def unmount_ebs(): unmount_volume() detach_volume(volume, ec2_instance_id, False) -def delete_ebs(project, pipeline, branch, platform, build_type): +def delete_ebs(repository_name, project, pipeline, branch, platform, build_type): unmount_ebs() session = boto3.session.Session() @@ -470,7 +474,7 @@ def delete_ebs(project, pipeline, branch, platform, build_type): ec2_resource = boto3.resource('ec2', region_name=region) ec2_instance = ec2_resource.Instance(ec2_instance_id) - mount_name = get_mount_name(project, pipeline, branch, platform, build_type) + mount_name = get_mount_name(repository_name, project, pipeline, branch, platform, build_type) response = ec2_client.describe_volumes(Filters=[ { 'Name': 'tag:Name', 'Values': [mount_name] } ]) @@ -481,15 +485,15 @@ def delete_ebs(project, pipeline, branch, platform, build_type): delete_volume(ec2_client, volume_id) -def main(action, project, pipeline, branch, platform, build_type, disk_size, disk_type): +def main(action, repository_name, project, pipeline, branch, platform, build_type, disk_size, disk_type): if action == 'mount': - mount_ebs(project, pipeline, branch, platform, build_type, disk_size, disk_type) + mount_ebs(repository_name, project, pipeline, branch, platform, build_type, disk_size, disk_type) elif action == 'unmount': unmount_ebs() elif action == 'delete': - delete_ebs(project, pipeline, branch, platform, build_type) + delete_ebs(repository_name, project, pipeline, branch, platform, build_type) if __name__ == "__main__": args = parse_args() - ret = main(args.action, args.project, args.pipeline, args.branch, args.platform, args.build_type, args.disk_size, args.disk_type) + ret = main(args.action, args.repository_name, args.project, args.pipeline, args.branch, args.platform, args.build_type, args.disk_size, args.disk_type) sys.exit(ret) \ No newline at end of file From f5b7200f328471cad30d6cff050b75588fecfeaa Mon Sep 17 00:00:00 2001 From: shiranj Date: Mon, 19 Apr 2021 14:59:43 -0700 Subject: [PATCH 2/4] Fix typo --- scripts/build/bootstrap/incremental_build_util.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/scripts/build/bootstrap/incremental_build_util.py b/scripts/build/bootstrap/incremental_build_util.py index fec48e2bee..0228f129d0 100755 --- a/scripts/build/bootstrap/incremental_build_util.py +++ b/scripts/build/bootstrap/incremental_build_util.py @@ -94,8 +94,8 @@ def error(message): def parse_args(): parser = argparse.ArgumentParser() parser.add_argument('-a', '--action', dest="action", help="Action (mount|unmount|delete)") - parser.add_argument('-proj', '--repository_name', dest="repository_name", help="Repository name") - parser.add_argument('-proj', '--project', dest="project", help="Project") + parser.add_argument('-repository_name', '--repository_name', dest="repository_name", help="Repository name") + parser.add_argument('-project', '--project', dest="project", help="Project") parser.add_argument('-pipe', '--pipeline', dest="pipeline", help="Pipeline") parser.add_argument('-b', '--branch', dest="branch", help="Branch") parser.add_argument('-plat', '--platform', dest="platform", help="Platform") From 0472bc49aaf76351b6c7eedf3d873f61f9a9fe95 Mon Sep 17 00:00:00 2001 From: shiranj Date: Mon, 19 Apr 2021 15:25:51 -0700 Subject: [PATCH 3/4] Set winSlashReplacement to false when running incremental_build_util.py --- scripts/build/Jenkins/Jenkinsfile | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/scripts/build/Jenkins/Jenkinsfile b/scripts/build/Jenkins/Jenkinsfile index 7f269c0012..9751d5f31b 100644 --- a/scripts/build/Jenkins/Jenkinsfile +++ b/scripts/build/Jenkins/Jenkinsfile @@ -276,10 +276,10 @@ def PreBuildCommonSteps(Map pipelineConfig, String repositoryName, String projec else pythonCmd = 'python -u ' if(env.RECREATE_VOLUME?.toBoolean()) { - palSh("${pythonCmd} ${INCREMENTAL_BUILD_SCRIPT_PATH} --action delete --repository_name ${repositoryName} --project ${projectName} --pipeline ${pipeline} --branch ${branchName} --platform ${platform} --build_type ${buildType}", 'Deleting volume') + palSh("${pythonCmd} ${INCREMENTAL_BUILD_SCRIPT_PATH} --action delete --repository_name ${repositoryName} --project ${projectName} --pipeline ${pipeline} --branch ${branchName} --platform ${platform} --build_type ${buildType}", 'Deleting volume', winSlashReplacement=false) } timeout(5) { - palSh("${pythonCmd} ${INCREMENTAL_BUILD_SCRIPT_PATH} --action mount --repository_name ${repositoryName} --project ${projectName} --pipeline ${pipeline} --branch ${branchName} --platform ${platform} --build_type ${buildType}", 'Mounting volume') + palSh("${pythonCmd} ${INCREMENTAL_BUILD_SCRIPT_PATH} --action mount --repository_name ${repositoryName} --project ${projectName} --pipeline ${pipeline} --branch ${branchName} --platform ${platform} --build_type ${buildType}", 'Mounting volume', winSlashReplacement=false) } if(env.IS_UNIX) { From 23b9b3e12b4a9c870f38e7bc1753cd00dfca99de Mon Sep 17 00:00:00 2001 From: shiranj Date: Mon, 19 Apr 2021 15:52:43 -0700 Subject: [PATCH 4/4] Fix indentation --- scripts/build/bootstrap/incremental_build_util.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/build/bootstrap/incremental_build_util.py b/scripts/build/bootstrap/incremental_build_util.py index 0228f129d0..28bb70955e 100755 --- a/scripts/build/bootstrap/incremental_build_util.py +++ b/scripts/build/bootstrap/incremental_build_util.py @@ -205,7 +205,7 @@ def create_volume(ec2_client, availability_zone, repository_name, project, pipel 'ResourceType': 'volume', 'Tags': [ { 'Key': 'Name', 'Value': mount_name }, - {'Key': 'RepositoryName', 'Value': repository_name}, + { 'Key': 'RepositoryName', 'Value': repository_name}, { 'Key': 'Project', 'Value': project }, { 'Key': 'Pipeline', 'Value': pipeline }, { 'Key': 'BranchName', 'Value': branch },