birkeh
/
o3de
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Pipeline fails when using a new empty ebs volume

Browse Source
main
shiranj 5 years ago
parent 69fdd42d59
commit 04d171ecef
1 changed files with 72 additions and 80 deletions
Show Stats Download Patch File Download Diff File

152
AutomatedReview/Jenkinsfile vendored
Unescape Escape View File

@ -16,7 +16,7 @@ INCREMENTAL_BUILD_SCRIPT_PATH = 'scripts/build/bootstrap/incremental_build_util.
EMPTY_JSON = readJSON text: '{}' EMPTY_JSON = readJSON text: '{}'
ENGINE_REPOSITORY_NAME = 'o3de' ENGINE_REPOSITORY_NAME = env.DEFAULT_REPOSITORY_NAME
def pipelineProperties = [] def pipelineProperties = []
@ -96,7 +96,7 @@ def IsJobEnabled(buildTypeMap, pipelineName, platformName) {
if (params[platformName]) { if (params[platformName]) {
if(buildTypeMap.value.TAGS) { if(buildTypeMap.value.TAGS) {
return buildTypeMap.value.TAGS.contains(pipelineName) return buildTypeMap.value.TAGS.contains(pipelineName)
} }
} }
} }
return false return false
@ -194,9 +194,9 @@ def GetBuildEnvVars(Map platformEnv, Map buildTypeEnv, String pipelineName) {
} }
buildTypeEnv.each { var -> buildTypeEnv.each { var ->
// This may override the above one if there is an entry defined by the job // This may override the above one if there is an entry defined by the job
envVarMap[var.key] = var.value envVarMap[var.key] = var.value
} }
// Environment that only applies to to Jenkins tweaks. // Environment that only applies to to Jenkins tweaks.
// For 3rdParty downloads, we store them in the EBS volume so we can reuse them across node // For 3rdParty downloads, we store them in the EBS volume so we can reuse them across node
// instances. This allow us to scale up and down without having to re-download 3rdParty // instances. This allow us to scale up and down without having to re-download 3rdParty
@ -223,7 +223,7 @@ def PullFilesFromGit(String filenamePath, String branchName, boolean failIfNotFo
folderPathParts.remove(folderPathParts.size()-1) // remove the filename folderPathParts.remove(folderPathParts.size()-1) // remove the filename
def folderPath = folderPathParts.join('/') def folderPath = folderPathParts.join('/')
if (folderPath.contains('*')) { if (folderPath.contains('*')) {
def currentPath = '' def currentPath = ''
for (int i = 0; i < folderPathParts.size(); i++) { for (int i = 0; i < folderPathParts.size(); i++) {
if (folderPathParts[i] == '*') { if (folderPathParts[i] == '*') {
@ -259,7 +259,7 @@ def PullFilesFromGit(String filenamePath, String branchName, boolean failIfNotFo
} else { } else {
def errorFile = "${folderPath}/error.txt" def errorFile = "${folderPath}/error.txt"
palMkdir(folderPath) palMkdir(folderPath)
retry(3) { retry(3) {
try { try {
@ -273,7 +273,7 @@ def PullFilesFromGit(String filenamePath, String branchName, boolean failIfNotFo
win_filenamePath = filenamePath.replace('/', '\\') win_filenamePath = filenamePath.replace('/', '\\')
bat label: "Downloading ${win_filenamePath}", bat label: "Downloading ${win_filenamePath}",
script: "aws codecommit get-file --repository-name ${repositoryName} --commit-specifier ${branchName} --file-path ${filenamePath} --query fileContent --output text 2>${errorFile} > ${win_filenamePath}_encoded" script: "aws codecommit get-file --repository-name ${repositoryName} --commit-specifier ${branchName} --file-path ${filenamePath} --query fileContent --output text 2>${errorFile} > ${win_filenamePath}_encoded"
bat label: 'Decoding', bat label: 'Decoding',
script: "certutil -decode ${win_filenamePath}_encoded ${win_filenamePath}" script: "certutil -decode ${win_filenamePath}_encoded ${win_filenamePath}"
} }
palRm("${filenamePath}_encoded") palRm("${filenamePath}_encoded")
@ -296,7 +296,7 @@ def SetLfsCredentials(cmd, lbl = '') {
if (env.IS_UNIX) { if (env.IS_UNIX) {
sh label: lbl, sh label: lbl,
script: cmd script: cmd
} else { } else {
bat label: lbl, bat label: lbl,
script: cmd script: cmd
} }
@ -325,19 +325,17 @@ def CheckoutBootstrapScripts(String branchName) {
} }
def CheckoutRepo(boolean disableSubmodules = false) { def CheckoutRepo(boolean disableSubmodules = false) {
dir(ENGINE_REPOSITORY_NAME) { palSh('git lfs uninstall', 'Git LFS Uninstall') // Prevent git from pulling lfs objects during checkout
palSh('git lfs uninstall', 'Git LFS Uninstall') // Prevent git from pulling lfs objects during checkout
if(fileExists('.git')) {
if(fileExists('.git')) { // If the repository after checkout is locked, likely we took a snapshot while git was running,
// If the repository after checkout is locked, likely we took a snapshot while git was running, // to leave the repo in a usable state, garbagecollect. This also helps in situations where
// to leave the repo in a usable state, garbagecollect. This also helps in situations where def indexLockFile = '.git/index.lock'
def indexLockFile = '.git/index.lock' if(fileExists(indexLockFile)) {
if(fileExists(indexLockFile)) { palSh('git gc', 'Git GarbageCollect')
palSh('git gc', 'Git GarbageCollect') }
} if(fileExists(indexLockFile)) { // if it is still there, remove it
if(fileExists(indexLockFile)) { // if it is still there, remove it palRm(indexLockFile)
palRm(indexLockFile)
}
} }
} }
@ -350,52 +348,41 @@ def CheckoutRepo(boolean disableSubmodules = false) {
retryAttempt = retryAttempt + 1 retryAttempt = retryAttempt + 1
if(params.PULL_REQUEST_ID) { if(params.PULL_REQUEST_ID) {
// This is a pull request build. Perform merge with destination branch before building. // This is a pull request build. Perform merge with destination branch before building.
dir(ENGINE_REPOSITORY_NAME) { checkout scm: [
checkout scm: [ $class: 'GitSCM',
$class: 'GitSCM', branches: scm.branches,
branches: scm.branches, extensions: [
extensions: [ [$class: 'PreBuildMerge', options: [mergeRemote: 'origin', mergeTarget: params.DESTINATION_BRANCH]],
[$class: 'PreBuildMerge', options: [mergeRemote: 'origin', mergeTarget: params.DESTINATION_BRANCH]], [$class: 'SubmoduleOption', disableSubmodules: disableSubmodules, recursiveSubmodules: true],
[$class: 'SubmoduleOption', disableSubmodules: disableSubmodules, recursiveSubmodules: true], [$class: 'CheckoutOption', timeout: 60]
[$class: 'CheckoutOption', timeout: 60] ],
], userRemoteConfigs: scm.userRemoteConfigs
userRemoteConfigs: scm.userRemoteConfigs ]
]
}
} else { } else {
dir(ENGINE_REPOSITORY_NAME) { checkout scm: [
checkout scm: [ $class: 'GitSCM',
$class: 'GitSCM', branches: scm.branches,
branches: scm.branches, extensions: [
extensions: [ [$class: 'SubmoduleOption', disableSubmodules: disableSubmodules, recursiveSubmodules: true],
[$class: 'SubmoduleOption', disableSubmodules: disableSubmodules, recursiveSubmodules: true], [$class: 'CheckoutOption', timeout: 60]
[$class: 'CheckoutOption', timeout: 60] ],
], userRemoteConfigs: scm.userRemoteConfigs
userRemoteConfigs: scm.userRemoteConfigs ]
]
}
} }
} }
// Add folder where we will store the 3rdParty downloads and packages // Run lfs in a separate step. Jenkins is unable to load the credentials for the custom LFS endpoint
if(!fileExists('3rdParty')) { withCredentials([usernamePassword(credentialsId: "${env.GITHUB_USER}", passwordVariable: 'accesstoken', usernameVariable: 'username')]) {
palMkdir('3rdParty') SetLfsCredentials("git config -f .lfsconfig lfs.url https://${username}:${accesstoken}@${env.LFS_URL}", 'Set credentials')
}
dir(ENGINE_REPOSITORY_NAME) {
// Run lfs in a separate step. Jenkins is unable to load the credentials for the custom LFS endpoint
withCredentials([usernamePassword(credentialsId: "${env.GITHUB_USER}", passwordVariable: 'accesstoken', usernameVariable: 'username')]) {
SetLfsCredentials("git config -f .lfsconfig lfs.url https://${username}:${accesstoken}@${env.LFS_URL}", 'Set credentials')
}
palSh('git lfs install', 'Git LFS Install')
palSh('git lfs pull', 'Git LFS Pull')
// CHANGE_ID is used by some scripts to identify uniquely the current change (usually metric jobs)
palSh('git rev-parse HEAD > commitid', 'Getting commit id')
env.CHANGE_ID = readFile file: 'commitid'
env.CHANGE_ID = env.CHANGE_ID.trim()
palRm('commitid')
} }
palSh('git lfs install', 'Git LFS Install')
palSh('git lfs pull', 'Git LFS Pull')
// CHANGE_ID is used by some scripts to identify uniquely the current change (usually metric jobs)
palSh('git rev-parse HEAD > commitid', 'Getting commit id')
env.CHANGE_ID = readFile file: 'commitid'
env.CHANGE_ID = env.CHANGE_ID.trim()
palRm('commitid')
} }
def PreBuildCommonSteps(Map pipelineConfig, String projectName, String pipeline, String branchName, String platform, String buildType, String workspace, boolean mount = true, boolean disableSubmodules = false) { def PreBuildCommonSteps(Map pipelineConfig, String projectName, String pipeline, String branchName, String platform, String buildType, String workspace, boolean mount = true, boolean disableSubmodules = false) {
@ -419,7 +406,7 @@ def PreBuildCommonSteps(Map pipelineConfig, String projectName, String pipeline,
sh label: 'Setting volume\'s ownership', sh label: 'Setting volume\'s ownership',
script: """ script: """
if sudo test ! -d "${workspace}"; then if sudo test ! -d "${workspace}"; then
sudo mkdir -p ${workspace} sudo mkdir -p ${workspace}
cd ${workspace}/.. cd ${workspace}/..
sudo chown -R lybuilder:root . sudo chown -R lybuilder:root .
fi fi
@ -436,28 +423,33 @@ def PreBuildCommonSteps(Map pipelineConfig, String projectName, String pipeline,
} }
dir(workspace) { dir(workspace) {
// Add folder where we will store the 3rdParty downloads and packages
if(!fileExists('3rdParty')) {
palMkdir('3rdParty')
}
}
dir("${workspace}/${ENGINE_REPOSITORY_NAME}") {
CheckoutRepo(disableSubmodules) CheckoutRepo(disableSubmodules)
// Get python // Get python
dir(ENGINE_REPOSITORY_NAME) { if(env.IS_UNIX) {
if(env.IS_UNIX) { sh label: 'Getting python',
sh label: 'Getting python', script: 'python/get_python.sh'
script: 'python/get_python.sh' } else {
} else { bat label: 'Getting python',
bat label: 'Getting python', script: 'python/get_python.bat'
script: 'python/get_python.bat' }
}
if(env.CLEAN_OUTPUT_DIRECTORY.toBoolean() || env.CLEAN_ASSETS.toBoolean()) { if(env.CLEAN_OUTPUT_DIRECTORY.toBoolean() || env.CLEAN_ASSETS.toBoolean()) {
def command = "${pipelineConfig.BUILD_ENTRY_POINT} --platform ${platform} --type clean" def command = "${pipelineConfig.BUILD_ENTRY_POINT} --platform ${platform} --type clean"
if (env.IS_UNIX) { if (env.IS_UNIX) {
sh label: "Running ${platform} clean", sh label: "Running ${platform} clean",
script: "${pipelineConfig.PYTHON_DIR}/python.sh -u ${command}" script: "${pipelineConfig.PYTHON_DIR}/python.sh -u ${command}"
} else { } else {
bat label: "Running ${platform} clean", bat label: "Running ${platform} clean",
script: "${pipelineConfig.PYTHON_DIR}/python.cmd -u ${command}".replace('/','\\') script: "${pipelineConfig.PYTHON_DIR}/python.cmd -u ${command}".replace('/','\\')
}
} }
} }
} }

Write Preview
Loading…
Cancel
Save