birkeh/o3de
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Use complete Jenkins project name when creating EBS volumes

Browse Source 
This change will allow us to support multiple repos in our account by using the full project name for the volumes. Right now multiple repos running the default pipeline will use the same name (e.g. defaultmain<platform>) and will conflict when similar branch names are used.
This commit is contained in:
Brian Herrera
2021-04-08 17:36:00 -07:00
parent b32aa9beda
commit aef10339d5
2 changed files with 75 additions and 64 deletions
Download Patch File Download Diff File Expand all files Collapse all files
AutomatedReview/Jenkinsfile
Vendored
+51 -44
View File
@@ -23,7 +23,8 @@ def pipelineProperties = []
def pipelineParameters = [
// Build/clean Parameters
// The CLEAN_OUTPUT_DIRECTORY is used by ci_build scripts. Creating the parameter here passes it as an environment variable to jobs and is consumed that way
booleanParam(defaultValue: false, description: 'Deletes the contents of the output directory before building. This will cause a \"clean\" build', name: 'CLEAN_OUTPUT_DIRECTORY'),
booleanParam(defaultValue: false, description: 'Deletes the contents of the output directory before building. This will cause a \"clean\" build. NOTE: does not imply CLEAN_ASSETS', name: 'CLEAN_OUTPUT_DIRECTORY'),
booleanParam(defaultValue: false, description: 'Deletes the contents of the output directories of the AssetProcessor before building.', name: 'CLEAN_ASSETS'),
booleanParam(defaultValue: false, description: 'Deletes the contents of the workspace and forces a complete pull.', name: 'CLEAN_WORKSPACE'),
booleanParam(defaultValue: false, description: 'Recreates the volume used for the workspace. The volume will be created out of a snapshot taken from main.', name: 'RECREATE_VOLUME'),
string(defaultValue: '', description: 'Filters and overrides the list of jobs to run for each of the below platforms (comma-separated). Can\'t be used during a pull request.', name: 'JOB_LIST_OVERRIDE'),
@@ -105,9 +106,9 @@ def GetRunningPipelineName(JENKINS_JOB_NAME) {
// If the job name has an underscore
def job_parts = JENKINS_JOB_NAME.tokenize('/')[0].tokenize('_')
if (job_parts.size() > 1) {
return job_parts[job_parts.size()-1]
return [job_parts.take(job_parts.size() - 1).join('_'), job_parts[job_parts.size()-1]]
}
return 'default'
return [job_parts[0], 'default']
}
@NonCPS
@@ -222,45 +223,39 @@ def PullFilesFromGit(String filenamePath, String branchName, boolean failIfNotFo
folderPathParts.remove(folderPathParts.size()-1) // remove the filename
def folderPath = folderPathParts.join('/')
if (folderPath.contains('*')) {
try {
def currentPath = ''
for (int i = 0; i < folderPathParts.size(); i++) {
if (folderPathParts[i] == '*') {
palMkdir(currentPath)
retry(3) { palSh("aws codecommit get-folder --repository-name ${repositoryName} --commit-specifier ${branchName} --folder-path ${currentPath} > ${currentPath}/.codecommit", "GetFolder ${currentPath}") }
def folderInfo = readJSON file: "${currentPath}/.codecommit"
folderInfo.subFolders.each { folder ->
def newSubPath = currentPath + '/' + folder.relativePath
for (int j = i+1; j < folderPathParts.size(); j++) {
newSubPath = newSubPath + '/' + folderPathParts[j]
}
newSubPath = newSubPath + '/' + filename
PullFilesFromGit(newSubPath, branchName, false, repositoryName)
def currentPath = ''
for (int i = 0; i < folderPathParts.size(); i++) {
if (folderPathParts[i] == '*') {
palMkdir(currentPath)
retry(3) { palSh("aws codecommit get-folder --repository-name ${repositoryName} --commit-specifier ${branchName} --folder-path ${currentPath} > ${currentPath}/.codecommit", "GetFolder ${currentPath}") }
def folderInfo = readJSON file: "${currentPath}/.codecommit"
folderInfo.subFolders.each { folder ->
def newSubPath = currentPath + '/' + folder.relativePath
for (int j = i+1; j < folderPathParts.size(); j++) {
newSubPath = newSubPath + '/' + folderPathParts[j]
}
palRm("${currentPath}/.codecommit")
}
if (i == 0) {
currentPath = folderPathParts[i]
} else {
currentPath = currentPath + '/' + folderPathParts[i]
newSubPath = newSubPath + '/' + filename
PullFilesFromGit(newSubPath, branchName, false, repositoryName)
}
palRm("${currentPath}/.codecommit")
}
if (i == 0) {
currentPath = folderPathParts[i]
} else {
currentPath = currentPath + '/' + folderPathParts[i]
}
} catch(Exception e) {
}
} else if (filename.contains('*')) {
try {
palMkdir(folderPath)
retry(3) { palSh("aws codecommit get-folder --repository-name ${repositoryName} --commit-specifier ${branchName} --folder-path ${folderPath} > ${folderPath}/.codecommit", "GetFolder ${folderPath}") }
def folderInfo = readJSON file: "${folderPath}/.codecommit"
folderInfo.files.each { file ->
PullFilesFromGit("${folderPath}/${filename}", branchName, false, repositoryName)
}
palRm("${folderPath}/.codecommit")
} catch(Exception e) {
palMkdir(folderPath)
retry(3) { palSh("aws codecommit get-folder --repository-name ${repositoryName} --commit-specifier ${branchName} --folder-path ${folderPath} > ${folderPath}/.codecommit", "GetFolder ${folderPath}") }
def folderInfo = readJSON file: "${folderPath}/.codecommit"
folderInfo.files.each { file ->
PullFilesFromGit("${folderPath}/${filename}", branchName, false, repositoryName)
}
palRm("${folderPath}/.codecommit")
} else {
@@ -403,7 +398,7 @@ def CheckoutRepo(boolean disableSubmodules = false) {
}
}
def PreBuildCommonSteps(String pipeline, String branchName, String platform, String buildType, String workspace, boolean mount = true, boolean disableSubmodules = false) {
def PreBuildCommonSteps(Map pipelineConfig, String projectName, String pipeline, String branchName, String platform, String buildType, String workspace, boolean mount = true, boolean disableSubmodules = false) {
echo 'Starting pre-build common steps...'
if (mount) {
@@ -413,11 +408,11 @@ def PreBuildCommonSteps(String pipeline, String branchName, String platform, Str
if(env.IS_UNIX) pythonCmd = 'sudo -E python -u '
else pythonCmd = 'python -u '
if(params.RECREATE_VOLUME) {
palSh("${pythonCmd} ${INCREMENTAL_BUILD_SCRIPT_PATH} --action delete --pipeline ${pipeline} --branch ${branchName} --platform ${platform} --build_type ${buildType}", 'Deleting volume')
if(env.RECREATE_VOLUME.toBoolean()) {
palSh("${pythonCmd} ${INCREMENTAL_BUILD_SCRIPT_PATH} --action delete --project ${projectName} --pipeline ${pipeline} --branch ${branchName} --platform ${platform} --build_type ${buildType}", 'Deleting volume')
}
timeout(5) {
palSh("${pythonCmd} ${INCREMENTAL_BUILD_SCRIPT_PATH} --action mount --pipeline ${pipeline} --branch ${branchName} --platform ${platform} --build_type ${buildType}", 'Mounting volume')
palSh("${pythonCmd} ${INCREMENTAL_BUILD_SCRIPT_PATH} --action mount --project ${projectName} --pipeline ${pipeline} --branch ${branchName} --platform ${platform} --build_type ${buildType}", 'Mounting volume')
}
if(env.IS_UNIX) {
@@ -434,7 +429,7 @@ def PreBuildCommonSteps(String pipeline, String branchName, String platform, Str
// Cleanup previous repo location, we are currently at the root of the workspace, if we have a .git folder
// we need to cleanup. Once all branches take this relocation, we can remove this
if(params.CLEAN_WORKSPACE || fileExists("${workspace}/.git")) {
if(env.CLEAN_WORKSPACE.toBoolean() || fileExists("${workspace}/.git")) {
if(fileExists(workspace)) {
palRmDir(workspace)
}
@@ -453,6 +448,17 @@ def PreBuildCommonSteps(String pipeline, String branchName, String platform, Str
bat label: 'Getting python',
script: 'python/get_python.bat'
}
if(env.CLEAN_OUTPUT_DIRECTORY.toBoolean() || env.CLEAN_ASSETS.toBoolean()) {
def command = "${pipelineConfig.BUILD_ENTRY_POINT} --platform ${platform} --type clean"
if (env.IS_UNIX) {
sh label: "Running ${platform} clean",
script: "${pipelineConfig.PYTHON_DIR}/python.sh -u ${command}"
} else {
bat label: "Running ${platform} clean",
script: "${pipelineConfig.PYTHON_DIR}/python.cmd -u ${command}".replace('/','\\')
}
}
}
}
}
@@ -517,10 +523,10 @@ def PostBuildCommonSteps(String workspace, boolean mount = true) {
}
}
def CreateSetupStage(String pipelineName, String branchName, String platformName, String jobName, Map environmentVars) {
def CreateSetupStage(Map pipelineConfig, String projectName, String pipelineName, String branchName, String platformName, String jobName, Map environmentVars) {
return {
stage("Setup") {
PreBuildCommonSteps(pipelineName, branchName, platformName, jobName, environmentVars['WORKSPACE'], environmentVars['MOUNT_VOLUME'])
PreBuildCommonSteps(pipelineConfig, projectName, pipelineName, branchName, platformName, jobName, environmentVars['WORKSPACE'], environmentVars['MOUNT_VOLUME'])
}
}
}
@@ -549,6 +555,7 @@ def CreateTeardownStage(Map environmentVars) {
}
}
def projectName = ''
def pipelineName = ''
def branchName = ''
def pipelineConfig = {}
@@ -563,7 +570,7 @@ try {
}
withEnv(envVarList) {
timestamps {
pipelineName = GetRunningPipelineName(env.JOB_NAME) // env.JOB_NAME is the name of the job given by Jenkins
(projectName, pipelineName) = GetRunningPipelineName(env.JOB_NAME) // env.JOB_NAME is the name of the job given by Jenkins
scmType = GetSCMType()
if(env.BRANCH_NAME) {
@@ -629,7 +636,7 @@ try {
try {
def build_job_name = build_job.key
CreateSetupStage(pipelineName, branchName, platform.key, build_job.key, envVars).call()
CreateSetupStage(pipelineConfig, projectName, pipelineName, branchName, platform.key, build_job.key, envVars).call()
if(build_job.value.steps) { //this is a pipe with many steps so create all the build stages
build_job.value.steps.each { build_step ->
@@ -687,7 +694,7 @@ finally {
snsPublish(
topicArn: env.SNS_TOPIC,
subject:'Build Result',
message:"${currentBuild.currentResult}:${params.REPOSITORY_NAME}:${params.SOURCE_BRANCH}:${params.SOURCE_COMMIT}:${params.DESTINATION_COMMIT}:${params.PULL_REQUEST_ID}:${BUILD_URL}:${params.RECREATE_VOLUME}:${params.CLEAN_OUTPUT_DIRECTORY}"
message:"${currentBuild.currentResult}:${params.REPOSITORY_NAME}:${params.SOURCE_BRANCH}:${params.SOURCE_COMMIT}:${params.DESTINATION_COMMIT}:${params.PULL_REQUEST_ID}:${BUILD_URL}:${env.RECREATE_VOLUME}:${env.CLEAN_OUTPUT_DIRECTORY}:${env.CLEAN_ASSETS}"
)
}
step([
scripts/build/bootstrap/incremental_build_util.py
+24 -20
View File
@@ -94,6 +94,7 @@ def error(message):
def parse_args():
parser = argparse.ArgumentParser()
parser.add_argument('-a', '--action', dest="action", help="Action (mount|unmount|delete)")
parser.add_argument('-proj', '--project', dest="project", help="Project")
parser.add_argument('-pipe', '--pipeline', dest="pipeline", help="Pipeline")
parser.add_argument('-b', '--branch', dest="branch", help="Branch")
parser.add_argument('-plat', '--platform', dest="platform", help="Platform")
@@ -107,6 +108,8 @@ def parse_args():
error('No action specified')
args.action = args.action.lower()
if args.action != 'unmount':
if args.project is None:
error('No project specified')
if args.pipeline is None:
error('No pipeline specified')
if args.branch is None:
@@ -118,8 +121,8 @@ def parse_args():
return args
def get_mount_name(pipeline, branch, platform, build_type):
mount_name = "{}_{}_{}_{}".format(pipeline, branch, platform, build_type)
def get_mount_name(project, pipeline, branch, platform, build_type):
mount_name = "{}_{}_{}_{}_{}".format(project, pipeline, branch, platform, build_type)
mount_name = mount_name.replace('/','_').replace('\\','_')
return mount_name
@@ -171,8 +174,8 @@ def delete_volume(ec2_client, volume_id):
response = ec2_client.delete_volume(VolumeId=volume_id)
print 'Volume {} deleted'.format(volume_id)
def find_snapshot_id(ec2_client, pipeline, platform, build_type, disk_size):
mount_name = get_mount_name(pipeline, 'main', platform, build_type) # we take snapshots out of main
def find_snapshot_id(ec2_client, project, pipeline, platform, build_type, disk_size):
mount_name = get_mount_name(project, pipeline, 'main', platform, build_type) # we take snapshots out of main
response = ec2_client.describe_snapshots(Filters= [{
'Name': 'tag:Name', 'Values': [mount_name]
}])
@@ -188,9 +191,9 @@ def find_snapshot_id(ec2_client, pipeline, platform, build_type, disk_size):
snapshot_id = snapshot['SnapshotId']
return snapshot_id
def create_volume(ec2_client, availability_zone, pipeline, branch, platform, build_type, disk_size, disk_type):
def create_volume(ec2_client, availability_zone, project, pipeline, branch, platform, build_type, disk_size, disk_type):
# The actual EBS default calculation for IOps is a floating point number, the closest approxmiation is 4x of the disk size for simplicity
mount_name = get_mount_name(pipeline, branch, platform, build_type)
mount_name = get_mount_name(project, pipeline, branch, platform, build_type)
pipeline_and_branch = get_pipeline_and_branch(pipeline, branch)
parameters = dict(
AvailabilityZone = availability_zone,
@@ -199,6 +202,7 @@ def create_volume(ec2_client, availability_zone, pipeline, branch, platform, bui
'ResourceType': 'volume',
'Tags': [
{ 'Key': 'Name', 'Value': mount_name },
{ 'Key': 'Project', 'Value': project },
{ 'Key': 'Pipeline', 'Value': pipeline },
{ 'Key': 'BranchName', 'Value': branch },
{ 'Key': 'Platform', 'Value': platform },
@@ -210,7 +214,7 @@ def create_volume(ec2_client, availability_zone, pipeline, branch, platform, bui
if 'io1' in disk_type.lower():
parameters['Iops'] = (4 * disk_size)
snapshot_id = find_snapshot_id(ec2_client, pipeline, platform, build_type, disk_size)
snapshot_id = find_snapshot_id(ec2_client, project, pipeline, platform, build_type, disk_size)
if snapshot_id:
parameters['SnapshotId'] = snapshot_id
created = False
@@ -230,8 +234,8 @@ def create_volume(ec2_client, availability_zone, pipeline, branch, platform, bui
time.sleep(1)
response = ec2_client.describe_volumes(VolumeIds=[volume_id, ])
print("Volume {} created\n\tSnapshot: {}\n\tPipeline {}\n\tBranch {}\n\tPlatform: {}\n\tBuild type: {}"
.format(volume_id, snapshot_id, pipeline, branch, platform, build_type))
print("Volume {} created\n\tSnapshot: {}\n\tProject {}\n\tPipeline {}\n\tBranch {}\n\tPlatform: {}\n\tBuild type: {}"
.format(volume_id, snapshot_id, project, pipeline, branch, platform, build_type))
return volume_id, created
@@ -355,7 +359,7 @@ def attach_ebs_and_create_partition_with_retry(volume, volume_id, ec2_instance_i
mount_volume(created)
attempt += 1
def mount_ebs(pipeline, branch, platform, build_type, disk_size, disk_type):
def mount_ebs(project, pipeline, branch, platform, build_type, disk_size, disk_type):
session = boto3.session.Session()
region = session.region_name
if region is None:
@@ -375,7 +379,7 @@ def mount_ebs(pipeline, branch, platform, build_type, disk_size, disk_type):
unmount_volume()
detach_volume(volume, ec2_instance_id, False) # Force unmounts should not be used, as that will cause the EBS block device driver to fail the remount
mount_name = get_mount_name(pipeline, branch, platform, build_type)
mount_name = get_mount_name(project, pipeline, branch, platform, build_type)
response = ec2_client.describe_volumes(Filters=[{
'Name': 'tag:Name', 'Values': [mount_name]
}])
@@ -384,7 +388,7 @@ def mount_ebs(pipeline, branch, platform, build_type, disk_size, disk_type):
if 'Volumes' in response and not len(response['Volumes']):
print 'Volume for {} doesn\'t exist creating it...'.format(mount_name)
# volume doesn't exist, create it
volume_id, created = create_volume(ec2_client, ec2_availability_zone, pipeline, branch, platform, build_type, disk_size, disk_type)
volume_id, created = create_volume(ec2_client, ec2_availability_zone, project, pipeline, branch, platform, build_type, disk_size, disk_type)
else:
volume = response['Volumes'][0]
volume_id = volume['VolumeId']
@@ -392,7 +396,7 @@ def mount_ebs(pipeline, branch, platform, build_type, disk_size, disk_type):
if (volume['Size'] != disk_size or volume['VolumeType'] != disk_type):
print 'Override disk attributes does not match the existing volume, deleting {} and replacing the volume'.format(volume_id)
delete_volume(ec2_client, volume_id)
volume_id, created = create_volume(ec2_client, ec2_availability_zone, pipeline, branch, platform, build_type, disk_size, disk_type)
volume_id, created = create_volume(ec2_client, ec2_availability_zone, project, pipeline, branch, platform, build_type, disk_size, disk_type)
if len(volume['Attachments']):
# this is bad we shouldn't be attached, we should have detached at the end of a build
attachment = volume['Attachments'][0]
@@ -422,7 +426,7 @@ def mount_ebs(pipeline, branch, platform, build_type, disk_size, disk_type):
print 'Error: EBS disk size reached to the allowed maximum disk size {}MB, please contact ly-infra@ and ly-build@ to investigate.'.format(MAX_EBS_DISK_SIZE)
exit(1)
print 'Recreating the EBS with disk size {}'.format(new_disk_size)
volume_id, created = create_volume(ec2_client, ec2_availability_zone, pipeline, branch, platform, build_type, new_disk_size, disk_type)
volume_id, created = create_volume(ec2_client, ec2_availability_zone, project, pipeline, branch, platform, build_type, new_disk_size, disk_type)
volume = ec2_resource.Volume(volume_id)
attach_ebs_and_create_partition_with_retry(volume, volume_id, ec2_instance_id, created)
@@ -454,7 +458,7 @@ def unmount_ebs():
unmount_volume()
detach_volume(volume, ec2_instance_id, False)
def delete_ebs(pipeline, branch, platform, build_type):
def delete_ebs(project, pipeline, branch, platform, build_type):
unmount_ebs()
session = boto3.session.Session()
@@ -466,7 +470,7 @@ def delete_ebs(pipeline, branch, platform, build_type):
ec2_resource = boto3.resource('ec2', region_name=region)
ec2_instance = ec2_resource.Instance(ec2_instance_id)
mount_name = get_mount_name(pipeline, branch, platform, build_type)
mount_name = get_mount_name(project, pipeline, branch, platform, build_type)
response = ec2_client.describe_volumes(Filters=[
{ 'Name': 'tag:Name', 'Values': [mount_name] }
])
@@ -477,15 +481,15 @@ def delete_ebs(pipeline, branch, platform, build_type):
delete_volume(ec2_client, volume_id)
def main(action, pipeline, branch, platform, build_type, disk_size, disk_type):
def main(action, project, pipeline, branch, platform, build_type, disk_size, disk_type):
if action == 'mount':
mount_ebs(pipeline, branch, platform, build_type, disk_size, disk_type)
mount_ebs(project, pipeline, branch, platform, build_type, disk_size, disk_type)
elif action == 'unmount':
unmount_ebs()
elif action == 'delete':
delete_ebs(pipeline, branch, platform, build_type)
delete_ebs(project, pipeline, branch, platform, build_type)
if __name__ == "__main__":
args = parse_args()
ret = main(args.action, args.pipeline, args.branch, args.platform, args.build_type, args.disk_size, args.disk_type)
ret = main(args.action, args.project, args.pipeline, args.branch, args.platform, args.build_type, args.disk_size, args.disk_type)
sys.exit(ret)