From aef10339d5b452fc9dfffbfbba3001cd506330a4 Mon Sep 17 00:00:00 2001 From: Brian Herrera Date: Thu, 8 Apr 2021 17:36:00 -0700 Subject: [PATCH] Use complete Jenkins project name when creating EBS volumes This change will allow us to support multiple repos in our account by using the full project name for the volumes. Right now multiple repos running the default pipeline will use the same name (e.g. defaultmain) and will conflict when similar branch names are used. --- AutomatedReview/Jenkinsfile | 95 ++++++++++--------- .../build/bootstrap/incremental_build_util.py | 44 +++++---- 2 files changed, 75 insertions(+), 64 deletions(-) diff --git a/AutomatedReview/Jenkinsfile b/AutomatedReview/Jenkinsfile index 9ab7263945..19d965523d 100644 --- a/AutomatedReview/Jenkinsfile +++ b/AutomatedReview/Jenkinsfile @@ -23,7 +23,8 @@ def pipelineProperties = [] def pipelineParameters = [ // Build/clean Parameters // The CLEAN_OUTPUT_DIRECTORY is used by ci_build scripts. Creating the parameter here passes it as an environment variable to jobs and is consumed that way - booleanParam(defaultValue: false, description: 'Deletes the contents of the output directory before building. This will cause a \"clean\" build', name: 'CLEAN_OUTPUT_DIRECTORY'), + booleanParam(defaultValue: false, description: 'Deletes the contents of the output directory before building. This will cause a \"clean\" build. NOTE: does not imply CLEAN_ASSETS', name: 'CLEAN_OUTPUT_DIRECTORY'), + booleanParam(defaultValue: false, description: 'Deletes the contents of the output directories of the AssetProcessor before building.', name: 'CLEAN_ASSETS'), booleanParam(defaultValue: false, description: 'Deletes the contents of the workspace and forces a complete pull.', name: 'CLEAN_WORKSPACE'), booleanParam(defaultValue: false, description: 'Recreates the volume used for the workspace. The volume will be created out of a snapshot taken from main.', name: 'RECREATE_VOLUME'), string(defaultValue: '', description: 'Filters and overrides the list of jobs to run for each of the below platforms (comma-separated). Can\'t be used during a pull request.', name: 'JOB_LIST_OVERRIDE'), @@ -105,9 +106,9 @@ def GetRunningPipelineName(JENKINS_JOB_NAME) { // If the job name has an underscore def job_parts = JENKINS_JOB_NAME.tokenize('/')[0].tokenize('_') if (job_parts.size() > 1) { - return job_parts[job_parts.size()-1] + return [job_parts.take(job_parts.size() - 1).join('_'), job_parts[job_parts.size()-1]] } - return 'default' + return [job_parts[0], 'default'] } @NonCPS @@ -222,45 +223,39 @@ def PullFilesFromGit(String filenamePath, String branchName, boolean failIfNotFo folderPathParts.remove(folderPathParts.size()-1) // remove the filename def folderPath = folderPathParts.join('/') if (folderPath.contains('*')) { - - try { - def currentPath = '' - for (int i = 0; i < folderPathParts.size(); i++) { - if (folderPathParts[i] == '*') { - palMkdir(currentPath) - retry(3) { palSh("aws codecommit get-folder --repository-name ${repositoryName} --commit-specifier ${branchName} --folder-path ${currentPath} > ${currentPath}/.codecommit", "GetFolder ${currentPath}") } - def folderInfo = readJSON file: "${currentPath}/.codecommit" - folderInfo.subFolders.each { folder -> - def newSubPath = currentPath + '/' + folder.relativePath - for (int j = i+1; j < folderPathParts.size(); j++) { - newSubPath = newSubPath + '/' + folderPathParts[j] - } - newSubPath = newSubPath + '/' + filename - PullFilesFromGit(newSubPath, branchName, false, repositoryName) + + def currentPath = '' + for (int i = 0; i < folderPathParts.size(); i++) { + if (folderPathParts[i] == '*') { + palMkdir(currentPath) + retry(3) { palSh("aws codecommit get-folder --repository-name ${repositoryName} --commit-specifier ${branchName} --folder-path ${currentPath} > ${currentPath}/.codecommit", "GetFolder ${currentPath}") } + def folderInfo = readJSON file: "${currentPath}/.codecommit" + folderInfo.subFolders.each { folder -> + def newSubPath = currentPath + '/' + folder.relativePath + for (int j = i+1; j < folderPathParts.size(); j++) { + newSubPath = newSubPath + '/' + folderPathParts[j] } - palRm("${currentPath}/.codecommit") - } - if (i == 0) { - currentPath = folderPathParts[i] - } else { - currentPath = currentPath + '/' + folderPathParts[i] + newSubPath = newSubPath + '/' + filename + PullFilesFromGit(newSubPath, branchName, false, repositoryName) } + palRm("${currentPath}/.codecommit") + } + if (i == 0) { + currentPath = folderPathParts[i] + } else { + currentPath = currentPath + '/' + folderPathParts[i] } - } catch(Exception e) { } } else if (filename.contains('*')) { - try { - palMkdir(folderPath) - retry(3) { palSh("aws codecommit get-folder --repository-name ${repositoryName} --commit-specifier ${branchName} --folder-path ${folderPath} > ${folderPath}/.codecommit", "GetFolder ${folderPath}") } - def folderInfo = readJSON file: "${folderPath}/.codecommit" - folderInfo.files.each { file -> - PullFilesFromGit("${folderPath}/${filename}", branchName, false, repositoryName) - } - palRm("${folderPath}/.codecommit") - } catch(Exception e) { + palMkdir(folderPath) + retry(3) { palSh("aws codecommit get-folder --repository-name ${repositoryName} --commit-specifier ${branchName} --folder-path ${folderPath} > ${folderPath}/.codecommit", "GetFolder ${folderPath}") } + def folderInfo = readJSON file: "${folderPath}/.codecommit" + folderInfo.files.each { file -> + PullFilesFromGit("${folderPath}/${filename}", branchName, false, repositoryName) } + palRm("${folderPath}/.codecommit") } else { @@ -403,7 +398,7 @@ def CheckoutRepo(boolean disableSubmodules = false) { } } -def PreBuildCommonSteps(String pipeline, String branchName, String platform, String buildType, String workspace, boolean mount = true, boolean disableSubmodules = false) { +def PreBuildCommonSteps(Map pipelineConfig, String projectName, String pipeline, String branchName, String platform, String buildType, String workspace, boolean mount = true, boolean disableSubmodules = false) { echo 'Starting pre-build common steps...' if (mount) { @@ -413,11 +408,11 @@ def PreBuildCommonSteps(String pipeline, String branchName, String platform, Str if(env.IS_UNIX) pythonCmd = 'sudo -E python -u ' else pythonCmd = 'python -u ' - if(params.RECREATE_VOLUME) { - palSh("${pythonCmd} ${INCREMENTAL_BUILD_SCRIPT_PATH} --action delete --pipeline ${pipeline} --branch ${branchName} --platform ${platform} --build_type ${buildType}", 'Deleting volume') + if(env.RECREATE_VOLUME.toBoolean()) { + palSh("${pythonCmd} ${INCREMENTAL_BUILD_SCRIPT_PATH} --action delete --project ${projectName} --pipeline ${pipeline} --branch ${branchName} --platform ${platform} --build_type ${buildType}", 'Deleting volume') } timeout(5) { - palSh("${pythonCmd} ${INCREMENTAL_BUILD_SCRIPT_PATH} --action mount --pipeline ${pipeline} --branch ${branchName} --platform ${platform} --build_type ${buildType}", 'Mounting volume') + palSh("${pythonCmd} ${INCREMENTAL_BUILD_SCRIPT_PATH} --action mount --project ${projectName} --pipeline ${pipeline} --branch ${branchName} --platform ${platform} --build_type ${buildType}", 'Mounting volume') } if(env.IS_UNIX) { @@ -434,7 +429,7 @@ def PreBuildCommonSteps(String pipeline, String branchName, String platform, Str // Cleanup previous repo location, we are currently at the root of the workspace, if we have a .git folder // we need to cleanup. Once all branches take this relocation, we can remove this - if(params.CLEAN_WORKSPACE || fileExists("${workspace}/.git")) { + if(env.CLEAN_WORKSPACE.toBoolean() || fileExists("${workspace}/.git")) { if(fileExists(workspace)) { palRmDir(workspace) } @@ -453,6 +448,17 @@ def PreBuildCommonSteps(String pipeline, String branchName, String platform, Str bat label: 'Getting python', script: 'python/get_python.bat' } + + if(env.CLEAN_OUTPUT_DIRECTORY.toBoolean() || env.CLEAN_ASSETS.toBoolean()) { + def command = "${pipelineConfig.BUILD_ENTRY_POINT} --platform ${platform} --type clean" + if (env.IS_UNIX) { + sh label: "Running ${platform} clean", + script: "${pipelineConfig.PYTHON_DIR}/python.sh -u ${command}" + } else { + bat label: "Running ${platform} clean", + script: "${pipelineConfig.PYTHON_DIR}/python.cmd -u ${command}".replace('/','\\') + } + } } } } @@ -517,10 +523,10 @@ def PostBuildCommonSteps(String workspace, boolean mount = true) { } } -def CreateSetupStage(String pipelineName, String branchName, String platformName, String jobName, Map environmentVars) { +def CreateSetupStage(Map pipelineConfig, String projectName, String pipelineName, String branchName, String platformName, String jobName, Map environmentVars) { return { stage("Setup") { - PreBuildCommonSteps(pipelineName, branchName, platformName, jobName, environmentVars['WORKSPACE'], environmentVars['MOUNT_VOLUME']) + PreBuildCommonSteps(pipelineConfig, projectName, pipelineName, branchName, platformName, jobName, environmentVars['WORKSPACE'], environmentVars['MOUNT_VOLUME']) } } } @@ -549,6 +555,7 @@ def CreateTeardownStage(Map environmentVars) { } } +def projectName = '' def pipelineName = '' def branchName = '' def pipelineConfig = {} @@ -563,7 +570,7 @@ try { } withEnv(envVarList) { timestamps { - pipelineName = GetRunningPipelineName(env.JOB_NAME) // env.JOB_NAME is the name of the job given by Jenkins + (projectName, pipelineName) = GetRunningPipelineName(env.JOB_NAME) // env.JOB_NAME is the name of the job given by Jenkins scmType = GetSCMType() if(env.BRANCH_NAME) { @@ -629,7 +636,7 @@ try { try { def build_job_name = build_job.key - CreateSetupStage(pipelineName, branchName, platform.key, build_job.key, envVars).call() + CreateSetupStage(pipelineConfig, projectName, pipelineName, branchName, platform.key, build_job.key, envVars).call() if(build_job.value.steps) { //this is a pipe with many steps so create all the build stages build_job.value.steps.each { build_step -> @@ -687,7 +694,7 @@ finally { snsPublish( topicArn: env.SNS_TOPIC, subject:'Build Result', - message:"${currentBuild.currentResult}:${params.REPOSITORY_NAME}:${params.SOURCE_BRANCH}:${params.SOURCE_COMMIT}:${params.DESTINATION_COMMIT}:${params.PULL_REQUEST_ID}:${BUILD_URL}:${params.RECREATE_VOLUME}:${params.CLEAN_OUTPUT_DIRECTORY}" + message:"${currentBuild.currentResult}:${params.REPOSITORY_NAME}:${params.SOURCE_BRANCH}:${params.SOURCE_COMMIT}:${params.DESTINATION_COMMIT}:${params.PULL_REQUEST_ID}:${BUILD_URL}:${env.RECREATE_VOLUME}:${env.CLEAN_OUTPUT_DIRECTORY}:${env.CLEAN_ASSETS}" ) } step([ diff --git a/scripts/build/bootstrap/incremental_build_util.py b/scripts/build/bootstrap/incremental_build_util.py index d7e0d623c3..40b4a5cb4f 100755 --- a/scripts/build/bootstrap/incremental_build_util.py +++ b/scripts/build/bootstrap/incremental_build_util.py @@ -94,6 +94,7 @@ def error(message): def parse_args(): parser = argparse.ArgumentParser() parser.add_argument('-a', '--action', dest="action", help="Action (mount|unmount|delete)") + parser.add_argument('-proj', '--project', dest="project", help="Project") parser.add_argument('-pipe', '--pipeline', dest="pipeline", help="Pipeline") parser.add_argument('-b', '--branch', dest="branch", help="Branch") parser.add_argument('-plat', '--platform', dest="platform", help="Platform") @@ -107,6 +108,8 @@ def parse_args(): error('No action specified') args.action = args.action.lower() if args.action != 'unmount': + if args.project is None: + error('No project specified') if args.pipeline is None: error('No pipeline specified') if args.branch is None: @@ -118,8 +121,8 @@ def parse_args(): return args -def get_mount_name(pipeline, branch, platform, build_type): - mount_name = "{}_{}_{}_{}".format(pipeline, branch, platform, build_type) +def get_mount_name(project, pipeline, branch, platform, build_type): + mount_name = "{}_{}_{}_{}_{}".format(project, pipeline, branch, platform, build_type) mount_name = mount_name.replace('/','_').replace('\\','_') return mount_name @@ -171,8 +174,8 @@ def delete_volume(ec2_client, volume_id): response = ec2_client.delete_volume(VolumeId=volume_id) print 'Volume {} deleted'.format(volume_id) -def find_snapshot_id(ec2_client, pipeline, platform, build_type, disk_size): - mount_name = get_mount_name(pipeline, 'main', platform, build_type) # we take snapshots out of main +def find_snapshot_id(ec2_client, project, pipeline, platform, build_type, disk_size): + mount_name = get_mount_name(project, pipeline, 'main', platform, build_type) # we take snapshots out of main response = ec2_client.describe_snapshots(Filters= [{ 'Name': 'tag:Name', 'Values': [mount_name] }]) @@ -188,9 +191,9 @@ def find_snapshot_id(ec2_client, pipeline, platform, build_type, disk_size): snapshot_id = snapshot['SnapshotId'] return snapshot_id -def create_volume(ec2_client, availability_zone, pipeline, branch, platform, build_type, disk_size, disk_type): +def create_volume(ec2_client, availability_zone, project, pipeline, branch, platform, build_type, disk_size, disk_type): # The actual EBS default calculation for IOps is a floating point number, the closest approxmiation is 4x of the disk size for simplicity - mount_name = get_mount_name(pipeline, branch, platform, build_type) + mount_name = get_mount_name(project, pipeline, branch, platform, build_type) pipeline_and_branch = get_pipeline_and_branch(pipeline, branch) parameters = dict( AvailabilityZone = availability_zone, @@ -199,6 +202,7 @@ def create_volume(ec2_client, availability_zone, pipeline, branch, platform, bui 'ResourceType': 'volume', 'Tags': [ { 'Key': 'Name', 'Value': mount_name }, + { 'Key': 'Project', 'Value': project }, { 'Key': 'Pipeline', 'Value': pipeline }, { 'Key': 'BranchName', 'Value': branch }, { 'Key': 'Platform', 'Value': platform }, @@ -210,7 +214,7 @@ def create_volume(ec2_client, availability_zone, pipeline, branch, platform, bui if 'io1' in disk_type.lower(): parameters['Iops'] = (4 * disk_size) - snapshot_id = find_snapshot_id(ec2_client, pipeline, platform, build_type, disk_size) + snapshot_id = find_snapshot_id(ec2_client, project, pipeline, platform, build_type, disk_size) if snapshot_id: parameters['SnapshotId'] = snapshot_id created = False @@ -230,8 +234,8 @@ def create_volume(ec2_client, availability_zone, pipeline, branch, platform, bui time.sleep(1) response = ec2_client.describe_volumes(VolumeIds=[volume_id, ]) - print("Volume {} created\n\tSnapshot: {}\n\tPipeline {}\n\tBranch {}\n\tPlatform: {}\n\tBuild type: {}" - .format(volume_id, snapshot_id, pipeline, branch, platform, build_type)) + print("Volume {} created\n\tSnapshot: {}\n\tProject {}\n\tPipeline {}\n\tBranch {}\n\tPlatform: {}\n\tBuild type: {}" + .format(volume_id, snapshot_id, project, pipeline, branch, platform, build_type)) return volume_id, created @@ -355,7 +359,7 @@ def attach_ebs_and_create_partition_with_retry(volume, volume_id, ec2_instance_i mount_volume(created) attempt += 1 -def mount_ebs(pipeline, branch, platform, build_type, disk_size, disk_type): +def mount_ebs(project, pipeline, branch, platform, build_type, disk_size, disk_type): session = boto3.session.Session() region = session.region_name if region is None: @@ -375,7 +379,7 @@ def mount_ebs(pipeline, branch, platform, build_type, disk_size, disk_type): unmount_volume() detach_volume(volume, ec2_instance_id, False) # Force unmounts should not be used, as that will cause the EBS block device driver to fail the remount - mount_name = get_mount_name(pipeline, branch, platform, build_type) + mount_name = get_mount_name(project, pipeline, branch, platform, build_type) response = ec2_client.describe_volumes(Filters=[{ 'Name': 'tag:Name', 'Values': [mount_name] }]) @@ -384,7 +388,7 @@ def mount_ebs(pipeline, branch, platform, build_type, disk_size, disk_type): if 'Volumes' in response and not len(response['Volumes']): print 'Volume for {} doesn\'t exist creating it...'.format(mount_name) # volume doesn't exist, create it - volume_id, created = create_volume(ec2_client, ec2_availability_zone, pipeline, branch, platform, build_type, disk_size, disk_type) + volume_id, created = create_volume(ec2_client, ec2_availability_zone, project, pipeline, branch, platform, build_type, disk_size, disk_type) else: volume = response['Volumes'][0] volume_id = volume['VolumeId'] @@ -392,7 +396,7 @@ def mount_ebs(pipeline, branch, platform, build_type, disk_size, disk_type): if (volume['Size'] != disk_size or volume['VolumeType'] != disk_type): print 'Override disk attributes does not match the existing volume, deleting {} and replacing the volume'.format(volume_id) delete_volume(ec2_client, volume_id) - volume_id, created = create_volume(ec2_client, ec2_availability_zone, pipeline, branch, platform, build_type, disk_size, disk_type) + volume_id, created = create_volume(ec2_client, ec2_availability_zone, project, pipeline, branch, platform, build_type, disk_size, disk_type) if len(volume['Attachments']): # this is bad we shouldn't be attached, we should have detached at the end of a build attachment = volume['Attachments'][0] @@ -422,7 +426,7 @@ def mount_ebs(pipeline, branch, platform, build_type, disk_size, disk_type): print 'Error: EBS disk size reached to the allowed maximum disk size {}MB, please contact ly-infra@ and ly-build@ to investigate.'.format(MAX_EBS_DISK_SIZE) exit(1) print 'Recreating the EBS with disk size {}'.format(new_disk_size) - volume_id, created = create_volume(ec2_client, ec2_availability_zone, pipeline, branch, platform, build_type, new_disk_size, disk_type) + volume_id, created = create_volume(ec2_client, ec2_availability_zone, project, pipeline, branch, platform, build_type, new_disk_size, disk_type) volume = ec2_resource.Volume(volume_id) attach_ebs_and_create_partition_with_retry(volume, volume_id, ec2_instance_id, created) @@ -454,7 +458,7 @@ def unmount_ebs(): unmount_volume() detach_volume(volume, ec2_instance_id, False) -def delete_ebs(pipeline, branch, platform, build_type): +def delete_ebs(project, pipeline, branch, platform, build_type): unmount_ebs() session = boto3.session.Session() @@ -466,7 +470,7 @@ def delete_ebs(pipeline, branch, platform, build_type): ec2_resource = boto3.resource('ec2', region_name=region) ec2_instance = ec2_resource.Instance(ec2_instance_id) - mount_name = get_mount_name(pipeline, branch, platform, build_type) + mount_name = get_mount_name(project, pipeline, branch, platform, build_type) response = ec2_client.describe_volumes(Filters=[ { 'Name': 'tag:Name', 'Values': [mount_name] } ]) @@ -477,15 +481,15 @@ def delete_ebs(pipeline, branch, platform, build_type): delete_volume(ec2_client, volume_id) -def main(action, pipeline, branch, platform, build_type, disk_size, disk_type): +def main(action, project, pipeline, branch, platform, build_type, disk_size, disk_type): if action == 'mount': - mount_ebs(pipeline, branch, platform, build_type, disk_size, disk_type) + mount_ebs(project, pipeline, branch, platform, build_type, disk_size, disk_type) elif action == 'unmount': unmount_ebs() elif action == 'delete': - delete_ebs(pipeline, branch, platform, build_type) + delete_ebs(project, pipeline, branch, platform, build_type) if __name__ == "__main__": args = parse_args() - ret = main(args.action, args.pipeline, args.branch, args.platform, args.build_type, args.disk_size, args.disk_type) + ret = main(args.action, args.project, args.pipeline, args.branch, args.platform, args.build_type, args.disk_size, args.disk_type) sys.exit(ret) \ No newline at end of file