diff --git a/AutomatedReview/Jenkinsfile b/AutomatedReview/Jenkinsfile index 9ab7263945..19d965523d 100644 --- a/AutomatedReview/Jenkinsfile +++ b/AutomatedReview/Jenkinsfile @@ -23,7 +23,8 @@ def pipelineProperties = [] def pipelineParameters = [ // Build/clean Parameters // The CLEAN_OUTPUT_DIRECTORY is used by ci_build scripts. Creating the parameter here passes it as an environment variable to jobs and is consumed that way - booleanParam(defaultValue: false, description: 'Deletes the contents of the output directory before building. This will cause a \"clean\" build', name: 'CLEAN_OUTPUT_DIRECTORY'), + booleanParam(defaultValue: false, description: 'Deletes the contents of the output directory before building. This will cause a \"clean\" build. NOTE: does not imply CLEAN_ASSETS', name: 'CLEAN_OUTPUT_DIRECTORY'), + booleanParam(defaultValue: false, description: 'Deletes the contents of the output directories of the AssetProcessor before building.', name: 'CLEAN_ASSETS'), booleanParam(defaultValue: false, description: 'Deletes the contents of the workspace and forces a complete pull.', name: 'CLEAN_WORKSPACE'), booleanParam(defaultValue: false, description: 'Recreates the volume used for the workspace. The volume will be created out of a snapshot taken from main.', name: 'RECREATE_VOLUME'), string(defaultValue: '', description: 'Filters and overrides the list of jobs to run for each of the below platforms (comma-separated). Can\'t be used during a pull request.', name: 'JOB_LIST_OVERRIDE'), @@ -105,9 +106,9 @@ def GetRunningPipelineName(JENKINS_JOB_NAME) { // If the job name has an underscore def job_parts = JENKINS_JOB_NAME.tokenize('/')[0].tokenize('_') if (job_parts.size() > 1) { - return job_parts[job_parts.size()-1] + return [job_parts.take(job_parts.size() - 1).join('_'), job_parts[job_parts.size()-1]] } - return 'default' + return [job_parts[0], 'default'] } @NonCPS @@ -222,45 +223,39 @@ def PullFilesFromGit(String filenamePath, String branchName, boolean failIfNotFo folderPathParts.remove(folderPathParts.size()-1) // remove the filename def folderPath = folderPathParts.join('/') if (folderPath.contains('*')) { - - try { - def currentPath = '' - for (int i = 0; i < folderPathParts.size(); i++) { - if (folderPathParts[i] == '*') { - palMkdir(currentPath) - retry(3) { palSh("aws codecommit get-folder --repository-name ${repositoryName} --commit-specifier ${branchName} --folder-path ${currentPath} > ${currentPath}/.codecommit", "GetFolder ${currentPath}") } - def folderInfo = readJSON file: "${currentPath}/.codecommit" - folderInfo.subFolders.each { folder -> - def newSubPath = currentPath + '/' + folder.relativePath - for (int j = i+1; j < folderPathParts.size(); j++) { - newSubPath = newSubPath + '/' + folderPathParts[j] - } - newSubPath = newSubPath + '/' + filename - PullFilesFromGit(newSubPath, branchName, false, repositoryName) + + def currentPath = '' + for (int i = 0; i < folderPathParts.size(); i++) { + if (folderPathParts[i] == '*') { + palMkdir(currentPath) + retry(3) { palSh("aws codecommit get-folder --repository-name ${repositoryName} --commit-specifier ${branchName} --folder-path ${currentPath} > ${currentPath}/.codecommit", "GetFolder ${currentPath}") } + def folderInfo = readJSON file: "${currentPath}/.codecommit" + folderInfo.subFolders.each { folder -> + def newSubPath = currentPath + '/' + folder.relativePath + for (int j = i+1; j < folderPathParts.size(); j++) { + newSubPath = newSubPath + '/' + folderPathParts[j] } - palRm("${currentPath}/.codecommit") - } - if (i == 0) { - currentPath = folderPathParts[i] - } else { - currentPath = currentPath + '/' + folderPathParts[i] + newSubPath = newSubPath + '/' + filename + PullFilesFromGit(newSubPath, branchName, false, repositoryName) } + palRm("${currentPath}/.codecommit") + } + if (i == 0) { + currentPath = folderPathParts[i] + } else { + currentPath = currentPath + '/' + folderPathParts[i] } - } catch(Exception e) { } } else if (filename.contains('*')) { - try { - palMkdir(folderPath) - retry(3) { palSh("aws codecommit get-folder --repository-name ${repositoryName} --commit-specifier ${branchName} --folder-path ${folderPath} > ${folderPath}/.codecommit", "GetFolder ${folderPath}") } - def folderInfo = readJSON file: "${folderPath}/.codecommit" - folderInfo.files.each { file -> - PullFilesFromGit("${folderPath}/${filename}", branchName, false, repositoryName) - } - palRm("${folderPath}/.codecommit") - } catch(Exception e) { + palMkdir(folderPath) + retry(3) { palSh("aws codecommit get-folder --repository-name ${repositoryName} --commit-specifier ${branchName} --folder-path ${folderPath} > ${folderPath}/.codecommit", "GetFolder ${folderPath}") } + def folderInfo = readJSON file: "${folderPath}/.codecommit" + folderInfo.files.each { file -> + PullFilesFromGit("${folderPath}/${filename}", branchName, false, repositoryName) } + palRm("${folderPath}/.codecommit") } else { @@ -403,7 +398,7 @@ def CheckoutRepo(boolean disableSubmodules = false) { } } -def PreBuildCommonSteps(String pipeline, String branchName, String platform, String buildType, String workspace, boolean mount = true, boolean disableSubmodules = false) { +def PreBuildCommonSteps(Map pipelineConfig, String projectName, String pipeline, String branchName, String platform, String buildType, String workspace, boolean mount = true, boolean disableSubmodules = false) { echo 'Starting pre-build common steps...' if (mount) { @@ -413,11 +408,11 @@ def PreBuildCommonSteps(String pipeline, String branchName, String platform, Str if(env.IS_UNIX) pythonCmd = 'sudo -E python -u ' else pythonCmd = 'python -u ' - if(params.RECREATE_VOLUME) { - palSh("${pythonCmd} ${INCREMENTAL_BUILD_SCRIPT_PATH} --action delete --pipeline ${pipeline} --branch ${branchName} --platform ${platform} --build_type ${buildType}", 'Deleting volume') + if(env.RECREATE_VOLUME.toBoolean()) { + palSh("${pythonCmd} ${INCREMENTAL_BUILD_SCRIPT_PATH} --action delete --project ${projectName} --pipeline ${pipeline} --branch ${branchName} --platform ${platform} --build_type ${buildType}", 'Deleting volume') } timeout(5) { - palSh("${pythonCmd} ${INCREMENTAL_BUILD_SCRIPT_PATH} --action mount --pipeline ${pipeline} --branch ${branchName} --platform ${platform} --build_type ${buildType}", 'Mounting volume') + palSh("${pythonCmd} ${INCREMENTAL_BUILD_SCRIPT_PATH} --action mount --project ${projectName} --pipeline ${pipeline} --branch ${branchName} --platform ${platform} --build_type ${buildType}", 'Mounting volume') } if(env.IS_UNIX) { @@ -434,7 +429,7 @@ def PreBuildCommonSteps(String pipeline, String branchName, String platform, Str // Cleanup previous repo location, we are currently at the root of the workspace, if we have a .git folder // we need to cleanup. Once all branches take this relocation, we can remove this - if(params.CLEAN_WORKSPACE || fileExists("${workspace}/.git")) { + if(env.CLEAN_WORKSPACE.toBoolean() || fileExists("${workspace}/.git")) { if(fileExists(workspace)) { palRmDir(workspace) } @@ -453,6 +448,17 @@ def PreBuildCommonSteps(String pipeline, String branchName, String platform, Str bat label: 'Getting python', script: 'python/get_python.bat' } + + if(env.CLEAN_OUTPUT_DIRECTORY.toBoolean() || env.CLEAN_ASSETS.toBoolean()) { + def command = "${pipelineConfig.BUILD_ENTRY_POINT} --platform ${platform} --type clean" + if (env.IS_UNIX) { + sh label: "Running ${platform} clean", + script: "${pipelineConfig.PYTHON_DIR}/python.sh -u ${command}" + } else { + bat label: "Running ${platform} clean", + script: "${pipelineConfig.PYTHON_DIR}/python.cmd -u ${command}".replace('/','\\') + } + } } } } @@ -517,10 +523,10 @@ def PostBuildCommonSteps(String workspace, boolean mount = true) { } } -def CreateSetupStage(String pipelineName, String branchName, String platformName, String jobName, Map environmentVars) { +def CreateSetupStage(Map pipelineConfig, String projectName, String pipelineName, String branchName, String platformName, String jobName, Map environmentVars) { return { stage("Setup") { - PreBuildCommonSteps(pipelineName, branchName, platformName, jobName, environmentVars['WORKSPACE'], environmentVars['MOUNT_VOLUME']) + PreBuildCommonSteps(pipelineConfig, projectName, pipelineName, branchName, platformName, jobName, environmentVars['WORKSPACE'], environmentVars['MOUNT_VOLUME']) } } } @@ -549,6 +555,7 @@ def CreateTeardownStage(Map environmentVars) { } } +def projectName = '' def pipelineName = '' def branchName = '' def pipelineConfig = {} @@ -563,7 +570,7 @@ try { } withEnv(envVarList) { timestamps { - pipelineName = GetRunningPipelineName(env.JOB_NAME) // env.JOB_NAME is the name of the job given by Jenkins + (projectName, pipelineName) = GetRunningPipelineName(env.JOB_NAME) // env.JOB_NAME is the name of the job given by Jenkins scmType = GetSCMType() if(env.BRANCH_NAME) { @@ -629,7 +636,7 @@ try { try { def build_job_name = build_job.key - CreateSetupStage(pipelineName, branchName, platform.key, build_job.key, envVars).call() + CreateSetupStage(pipelineConfig, projectName, pipelineName, branchName, platform.key, build_job.key, envVars).call() if(build_job.value.steps) { //this is a pipe with many steps so create all the build stages build_job.value.steps.each { build_step -> @@ -687,7 +694,7 @@ finally { snsPublish( topicArn: env.SNS_TOPIC, subject:'Build Result', - message:"${currentBuild.currentResult}:${params.REPOSITORY_NAME}:${params.SOURCE_BRANCH}:${params.SOURCE_COMMIT}:${params.DESTINATION_COMMIT}:${params.PULL_REQUEST_ID}:${BUILD_URL}:${params.RECREATE_VOLUME}:${params.CLEAN_OUTPUT_DIRECTORY}" + message:"${currentBuild.currentResult}:${params.REPOSITORY_NAME}:${params.SOURCE_BRANCH}:${params.SOURCE_COMMIT}:${params.DESTINATION_COMMIT}:${params.PULL_REQUEST_ID}:${BUILD_URL}:${env.RECREATE_VOLUME}:${env.CLEAN_OUTPUT_DIRECTORY}:${env.CLEAN_ASSETS}" ) } step([ diff --git a/scripts/build/bootstrap/incremental_build_util.py b/scripts/build/bootstrap/incremental_build_util.py index d7e0d623c3..40b4a5cb4f 100755 --- a/scripts/build/bootstrap/incremental_build_util.py +++ b/scripts/build/bootstrap/incremental_build_util.py @@ -94,6 +94,7 @@ def error(message): def parse_args(): parser = argparse.ArgumentParser() parser.add_argument('-a', '--action', dest="action", help="Action (mount|unmount|delete)") + parser.add_argument('-proj', '--project', dest="project", help="Project") parser.add_argument('-pipe', '--pipeline', dest="pipeline", help="Pipeline") parser.add_argument('-b', '--branch', dest="branch", help="Branch") parser.add_argument('-plat', '--platform', dest="platform", help="Platform") @@ -107,6 +108,8 @@ def parse_args(): error('No action specified') args.action = args.action.lower() if args.action != 'unmount': + if args.project is None: + error('No project specified') if args.pipeline is None: error('No pipeline specified') if args.branch is None: @@ -118,8 +121,8 @@ def parse_args(): return args -def get_mount_name(pipeline, branch, platform, build_type): - mount_name = "{}_{}_{}_{}".format(pipeline, branch, platform, build_type) +def get_mount_name(project, pipeline, branch, platform, build_type): + mount_name = "{}_{}_{}_{}_{}".format(project, pipeline, branch, platform, build_type) mount_name = mount_name.replace('/','_').replace('\\','_') return mount_name @@ -171,8 +174,8 @@ def delete_volume(ec2_client, volume_id): response = ec2_client.delete_volume(VolumeId=volume_id) print 'Volume {} deleted'.format(volume_id) -def find_snapshot_id(ec2_client, pipeline, platform, build_type, disk_size): - mount_name = get_mount_name(pipeline, 'main', platform, build_type) # we take snapshots out of main +def find_snapshot_id(ec2_client, project, pipeline, platform, build_type, disk_size): + mount_name = get_mount_name(project, pipeline, 'main', platform, build_type) # we take snapshots out of main response = ec2_client.describe_snapshots(Filters= [{ 'Name': 'tag:Name', 'Values': [mount_name] }]) @@ -188,9 +191,9 @@ def find_snapshot_id(ec2_client, pipeline, platform, build_type, disk_size): snapshot_id = snapshot['SnapshotId'] return snapshot_id -def create_volume(ec2_client, availability_zone, pipeline, branch, platform, build_type, disk_size, disk_type): +def create_volume(ec2_client, availability_zone, project, pipeline, branch, platform, build_type, disk_size, disk_type): # The actual EBS default calculation for IOps is a floating point number, the closest approxmiation is 4x of the disk size for simplicity - mount_name = get_mount_name(pipeline, branch, platform, build_type) + mount_name = get_mount_name(project, pipeline, branch, platform, build_type) pipeline_and_branch = get_pipeline_and_branch(pipeline, branch) parameters = dict( AvailabilityZone = availability_zone, @@ -199,6 +202,7 @@ def create_volume(ec2_client, availability_zone, pipeline, branch, platform, bui 'ResourceType': 'volume', 'Tags': [ { 'Key': 'Name', 'Value': mount_name }, + { 'Key': 'Project', 'Value': project }, { 'Key': 'Pipeline', 'Value': pipeline }, { 'Key': 'BranchName', 'Value': branch }, { 'Key': 'Platform', 'Value': platform }, @@ -210,7 +214,7 @@ def create_volume(ec2_client, availability_zone, pipeline, branch, platform, bui if 'io1' in disk_type.lower(): parameters['Iops'] = (4 * disk_size) - snapshot_id = find_snapshot_id(ec2_client, pipeline, platform, build_type, disk_size) + snapshot_id = find_snapshot_id(ec2_client, project, pipeline, platform, build_type, disk_size) if snapshot_id: parameters['SnapshotId'] = snapshot_id created = False @@ -230,8 +234,8 @@ def create_volume(ec2_client, availability_zone, pipeline, branch, platform, bui time.sleep(1) response = ec2_client.describe_volumes(VolumeIds=[volume_id, ]) - print("Volume {} created\n\tSnapshot: {}\n\tPipeline {}\n\tBranch {}\n\tPlatform: {}\n\tBuild type: {}" - .format(volume_id, snapshot_id, pipeline, branch, platform, build_type)) + print("Volume {} created\n\tSnapshot: {}\n\tProject {}\n\tPipeline {}\n\tBranch {}\n\tPlatform: {}\n\tBuild type: {}" + .format(volume_id, snapshot_id, project, pipeline, branch, platform, build_type)) return volume_id, created @@ -355,7 +359,7 @@ def attach_ebs_and_create_partition_with_retry(volume, volume_id, ec2_instance_i mount_volume(created) attempt += 1 -def mount_ebs(pipeline, branch, platform, build_type, disk_size, disk_type): +def mount_ebs(project, pipeline, branch, platform, build_type, disk_size, disk_type): session = boto3.session.Session() region = session.region_name if region is None: @@ -375,7 +379,7 @@ def mount_ebs(pipeline, branch, platform, build_type, disk_size, disk_type): unmount_volume() detach_volume(volume, ec2_instance_id, False) # Force unmounts should not be used, as that will cause the EBS block device driver to fail the remount - mount_name = get_mount_name(pipeline, branch, platform, build_type) + mount_name = get_mount_name(project, pipeline, branch, platform, build_type) response = ec2_client.describe_volumes(Filters=[{ 'Name': 'tag:Name', 'Values': [mount_name] }]) @@ -384,7 +388,7 @@ def mount_ebs(pipeline, branch, platform, build_type, disk_size, disk_type): if 'Volumes' in response and not len(response['Volumes']): print 'Volume for {} doesn\'t exist creating it...'.format(mount_name) # volume doesn't exist, create it - volume_id, created = create_volume(ec2_client, ec2_availability_zone, pipeline, branch, platform, build_type, disk_size, disk_type) + volume_id, created = create_volume(ec2_client, ec2_availability_zone, project, pipeline, branch, platform, build_type, disk_size, disk_type) else: volume = response['Volumes'][0] volume_id = volume['VolumeId'] @@ -392,7 +396,7 @@ def mount_ebs(pipeline, branch, platform, build_type, disk_size, disk_type): if (volume['Size'] != disk_size or volume['VolumeType'] != disk_type): print 'Override disk attributes does not match the existing volume, deleting {} and replacing the volume'.format(volume_id) delete_volume(ec2_client, volume_id) - volume_id, created = create_volume(ec2_client, ec2_availability_zone, pipeline, branch, platform, build_type, disk_size, disk_type) + volume_id, created = create_volume(ec2_client, ec2_availability_zone, project, pipeline, branch, platform, build_type, disk_size, disk_type) if len(volume['Attachments']): # this is bad we shouldn't be attached, we should have detached at the end of a build attachment = volume['Attachments'][0] @@ -422,7 +426,7 @@ def mount_ebs(pipeline, branch, platform, build_type, disk_size, disk_type): print 'Error: EBS disk size reached to the allowed maximum disk size {}MB, please contact ly-infra@ and ly-build@ to investigate.'.format(MAX_EBS_DISK_SIZE) exit(1) print 'Recreating the EBS with disk size {}'.format(new_disk_size) - volume_id, created = create_volume(ec2_client, ec2_availability_zone, pipeline, branch, platform, build_type, new_disk_size, disk_type) + volume_id, created = create_volume(ec2_client, ec2_availability_zone, project, pipeline, branch, platform, build_type, new_disk_size, disk_type) volume = ec2_resource.Volume(volume_id) attach_ebs_and_create_partition_with_retry(volume, volume_id, ec2_instance_id, created) @@ -454,7 +458,7 @@ def unmount_ebs(): unmount_volume() detach_volume(volume, ec2_instance_id, False) -def delete_ebs(pipeline, branch, platform, build_type): +def delete_ebs(project, pipeline, branch, platform, build_type): unmount_ebs() session = boto3.session.Session() @@ -466,7 +470,7 @@ def delete_ebs(pipeline, branch, platform, build_type): ec2_resource = boto3.resource('ec2', region_name=region) ec2_instance = ec2_resource.Instance(ec2_instance_id) - mount_name = get_mount_name(pipeline, branch, platform, build_type) + mount_name = get_mount_name(project, pipeline, branch, platform, build_type) response = ec2_client.describe_volumes(Filters=[ { 'Name': 'tag:Name', 'Values': [mount_name] } ]) @@ -477,15 +481,15 @@ def delete_ebs(pipeline, branch, platform, build_type): delete_volume(ec2_client, volume_id) -def main(action, pipeline, branch, platform, build_type, disk_size, disk_type): +def main(action, project, pipeline, branch, platform, build_type, disk_size, disk_type): if action == 'mount': - mount_ebs(pipeline, branch, platform, build_type, disk_size, disk_type) + mount_ebs(project, pipeline, branch, platform, build_type, disk_size, disk_type) elif action == 'unmount': unmount_ebs() elif action == 'delete': - delete_ebs(pipeline, branch, platform, build_type) + delete_ebs(project, pipeline, branch, platform, build_type) if __name__ == "__main__": args = parse_args() - ret = main(args.action, args.pipeline, args.branch, args.platform, args.build_type, args.disk_size, args.disk_type) + ret = main(args.action, args.project, args.pipeline, args.branch, args.platform, args.build_type, args.disk_size, args.disk_type) sys.exit(ret) \ No newline at end of file