birkeh/o3de
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Merge branch 'main' into hultonha_LYN-2528_whitebox_prefab

Browse Source
This commit is contained in:
hultonha
2021-04-15 18:26:54 +01:00
parent 7ff0c5c33a 14f7883425
commit 9d84fdcb23
242 changed files with 482 additions and 15686 deletions
Download Patch File Download Diff File Expand all files Collapse all files
AutomatedReview/Jenkinsfile
Vendored
-709
View File
@@ -1,709 +0,0 @@
#!/usr/bin/env groovy
/*
* All or portions of this file Copyright (c) Amazon.com, Inc. or its affiliates or
* its licensors.
*
* For complete copyright and license terms please see the LICENSE at the root of this
* distribution (the "License"). All use of this software is governed by the License,
* or, if provided, by the license below or the license accompanying this file. Do not
* remove or modify any license notices. This file is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
*
*/
PIPELINE_CONFIG_FILE = 'AutomatedReview/lumberyard.json'
INCREMENTAL_BUILD_SCRIPT_PATH = 'scripts/build/bootstrap/incremental_build_util.py'
EMPTY_JSON = readJSON text: '{}'
ENGINE_REPOSITORY_NAME = 'o3de'
def pipelineProperties = []
def pipelineParameters = [
// Build/clean Parameters
// The CLEAN_OUTPUT_DIRECTORY is used by ci_build scripts. Creating the parameter here passes it as an environment variable to jobs and is consumed that way
booleanParam(defaultValue: false, description: 'Deletes the contents of the output directory before building. This will cause a \"clean\" build. NOTE: does not imply CLEAN_ASSETS', name: 'CLEAN_OUTPUT_DIRECTORY'),
booleanParam(defaultValue: false, description: 'Deletes the contents of the output directories of the AssetProcessor before building.', name: 'CLEAN_ASSETS'),
booleanParam(defaultValue: false, description: 'Deletes the contents of the workspace and forces a complete pull.', name: 'CLEAN_WORKSPACE'),
booleanParam(defaultValue: false, description: 'Recreates the volume used for the workspace. The volume will be created out of a snapshot taken from main.', name: 'RECREATE_VOLUME'),
string(defaultValue: '', description: 'Filters and overrides the list of jobs to run for each of the below platforms (comma-separated). Can\'t be used during a pull request.', name: 'JOB_LIST_OVERRIDE'),
// Pull Request Parameters
string(defaultValue: '', description: '', name: 'DESTINATION_BRANCH'),
string(defaultValue: '', description: '', name: 'DESTINATION_COMMIT'),
string(defaultValue: '', description: '', name: 'PULL_REQUEST_ID'),
string(defaultValue: '', description: '', name: 'REPOSITORY_NAME'),
string(defaultValue: '', description: '', name: 'SOURCE_BRANCH'),
string(defaultValue: '', description: '', name: 'SOURCE_COMMIT')
]
def palSh(cmd, lbl = '', winSlashReplacement = true) {
if (env.IS_UNIX) {
sh label: lbl,
script: cmd
} else if (winSlashReplacement) {
bat label: lbl,
script: cmd.replace('/','\\')
} else {
bat label: lbl,
script: cmd
}
}
def palMkdir(path) {
if (env.IS_UNIX) {
sh label: "Making directories ${path}",
script: "mkdir -p ${path}"
} else {
def win_path = path.replace('/','\\')
bat label: "Making directories ${win_path}",
script: "mkdir ${win_path}."
}
}
def palRm(path) {
if (env.IS_UNIX) {
sh label: "Removing ${path}",
script: "rm ${path}"
} else {
def win_path = path.replace('/','\\')
bat label: "Removing ${win_path}",
script: "del ${win_path}"
}
}
def palRmDir(path) {
if (env.IS_UNIX) {
sh label: "Removing ${path}",
script: "rm -rf ${path}"
} else {
def win_path = path.replace('/','\\')
bat label: "Removing ${win_path}",
script: "rd /s /q ${win_path}"
}
}
def IsJobEnabled(buildTypeMap, pipelineName, platformName) {
def job_list_override = params.JOB_LIST_OVERRIDE.tokenize(',')
if(params.PULL_REQUEST_ID) { // dont allow pull requests to filter platforms/jobs
if(buildTypeMap.value.TAGS) {
return buildTypeMap.value.TAGS.contains(pipelineName)
}
} else if (!job_list_override.isEmpty()) {
return params[platformName] && job_list_override.contains(buildTypeMap.key);
} else {
if (params[platformName]) {
if(buildTypeMap.value.TAGS) {
return buildTypeMap.value.TAGS.contains(pipelineName)
}
}
}
return false
}
def GetRunningPipelineName(JENKINS_JOB_NAME) {
// If the job name has an underscore
def job_parts = JENKINS_JOB_NAME.tokenize('/')[0].tokenize('_')
if (job_parts.size() > 1) {
return [job_parts.take(job_parts.size() - 1).join('_'), job_parts[job_parts.size()-1]]
}
return [job_parts[0], 'default']
}
@NonCPS
def RegexMatcher(str, regex) {
def matcher = (str =~ regex)
return matcher ? matcher.group(1) : null
}
def LoadPipelineConfig(String pipelineName, String branchName, String scmType) {
echo 'Loading pipeline config'
if (scmType == 'codecommit') {
PullFilesFromGit(PIPELINE_CONFIG_FILE, branchName, true, ENGINE_REPOSITORY_NAME)
}
def pipelineConfig = {}
pipelineConfig = readJSON file: PIPELINE_CONFIG_FILE
palRm(PIPELINE_CONFIG_FILE)
pipelineConfig.platforms = EMPTY_JSON
// Load the pipeline configs per platform
pipelineConfig.PIPELINE_CONFIGS.each { pipeline_config ->
def platform_regex = pipeline_config.replace('.','\\.').replace('*', '(.*)')
if (!env.IS_UNIX) {
platform_regex = platform_regex.replace('/','\\\\')
}
echo "Downloading platform pipeline configs ${pipeline_config}"
if (scmType == 'codecommit') {
PullFilesFromGit(pipeline_config, branchName, false, ENGINE_REPOSITORY_NAME)
}
echo "Searching platform pipeline configs in ${pipeline_config} using ${platform_regex}"
for (pipeline_config_path in findFiles(glob: pipeline_config)) {
echo "\tFound platform pipeline config ${pipeline_config_path}"
def platform = RegexMatcher(pipeline_config_path, platform_regex)
if(platform) {
pipelineConfig.platforms[platform] = EMPTY_JSON
pipelineConfig.platforms[platform].PIPELINE_ENV = readJSON file: pipeline_config_path.toString()
}
palRm(pipeline_config_path.toString())
}
}
// Load the build configs
pipelineConfig.BUILD_CONFIGS.each { build_config ->
def platform_regex = build_config.replace('.','\\.').replace('*', '(.*)')
if (!env.IS_UNIX) {
platform_regex = platform_regex.replace('/','\\\\')
}
echo "Downloading configs ${build_config}"
if (scmType == 'codecommit') {
PullFilesFromGit(build_config, branchName, false, ENGINE_REPOSITORY_NAME)
}
echo "Searching configs in ${build_config} using ${platform_regex}"
for (build_config_path in findFiles(glob: build_config)) {
echo "\tFound config ${build_config_path}"
def platform = RegexMatcher(build_config_path, platform_regex)
if(platform) {
pipelineConfig.platforms[platform].build_types = readJSON file: build_config_path.toString()
}
}
}
return pipelineConfig
}
def GetSCMType() {
def gitUrl = scm.getUserRemoteConfigs()[0].getUrl()
if (gitUrl ==~ /https:\/\/git-codecommit.*/) {
return 'codecommit'
} else if (gitUrl ==~ /https:\/\/github.com.*/) {
return 'github'
}
return 'unknown'
}
def GetBuildEnvVars(Map platformEnv, Map buildTypeEnv, String pipelineName) {
def envVarMap = [:]
platformPipelineEnv = platformEnv['ENV'] ?: [:]
platformPipelineEnv.each { var ->
envVarMap[var.key] = var.value
}
platformEnvOverride = platformEnv['PIPELINE_ENV_OVERRIDE'] ?: [:]
platformPipelineEnvOverride = platformEnvOverride[pipelineName] ?: [:]
platformPipelineEnvOverride.each { var ->
envVarMap[var.key] = var.value
}
buildTypeEnv.each { var ->
// This may override the above one if there is an entry defined by the job
envVarMap[var.key] = var.value
}
// Environment that only applies to to Jenkins tweaks.
// For 3rdParty downloads, we store them in the EBS volume so we can reuse them across node
// instances. This allow us to scale up and down without having to re-download 3rdParty
envVarMap['LY_PACKAGE_DOWNLOAD_CACHE_LOCATION'] = "${envVarMap['WORKSPACE']}/3rdParty/downloaded_packages"
envVarMap['LY_PACKAGE_UNPACK_LOCATION'] = "${envVarMap['WORKSPACE']}/3rdParty/packages"
return envVarMap
}
def GetEnvStringList(Map envVarMap) {
def strList = []
envVarMap.each { var ->
strList.add("${var.key}=${var.value}")
}
return strList
}
// Pulls/downloads files from the repo through codecommit. Despite Glob matching is NOT supported, '*' is supported
// as a folder or filename (not a portion, it has to be the whole folder or filename)
def PullFilesFromGit(String filenamePath, String branchName, boolean failIfNotFound = true, String repositoryName = env.DEFAULT_REPOSITORY_NAME) {
echo "PullFilesFromGit filenamePath=${filenamePath} branchName=${branchName} repositoryName=${repositoryName}"
def folderPathParts = filenamePath.tokenize('/')
def filename = folderPathParts[folderPathParts.size()-1]
folderPathParts.remove(folderPathParts.size()-1) // remove the filename
def folderPath = folderPathParts.join('/')
if (folderPath.contains('*')) {
def currentPath = ''
for (int i = 0; i < folderPathParts.size(); i++) {
if (folderPathParts[i] == '*') {
palMkdir(currentPath)
retry(3) { palSh("aws codecommit get-folder --repository-name ${repositoryName} --commit-specifier ${branchName} --folder-path ${currentPath} > ${currentPath}/.codecommit", "GetFolder ${currentPath}") }
def folderInfo = readJSON file: "${currentPath}/.codecommit"
folderInfo.subFolders.each { folder ->
def newSubPath = currentPath + '/' + folder.relativePath
for (int j = i+1; j < folderPathParts.size(); j++) {
newSubPath = newSubPath + '/' + folderPathParts[j]
}
newSubPath = newSubPath + '/' + filename
PullFilesFromGit(newSubPath, branchName, false, repositoryName)
}
palRm("${currentPath}/.codecommit")
}
if (i == 0) {
currentPath = folderPathParts[i]
} else {
currentPath = currentPath + '/' + folderPathParts[i]
}
}
} else if (filename.contains('*')) {
palMkdir(folderPath)
retry(3) { palSh("aws codecommit get-folder --repository-name ${repositoryName} --commit-specifier ${branchName} --folder-path ${folderPath} > ${folderPath}/.codecommit", "GetFolder ${folderPath}") }
def folderInfo = readJSON file: "${folderPath}/.codecommit"
folderInfo.files.each { file ->
PullFilesFromGit("${folderPath}/${filename}", branchName, false, repositoryName)
}
palRm("${folderPath}/.codecommit")
} else {
def errorFile = "${folderPath}/error.txt"
palMkdir(folderPath)
retry(3) {
try {
if(env.IS_UNIX) {
sh label: "Downloading ${filenamePath}",
script: "aws codecommit get-file --repository-name ${repositoryName} --commit-specifier ${branchName} --file-path ${filenamePath} --query fileContent --output text 2>${errorFile} > ${filenamePath}_encoded"
sh label: 'Decoding',
script: "base64 --decode ${filenamePath}_encoded > ${filenamePath}"
} else {
errorFile = errorFile.replace('/','\\')
win_filenamePath = filenamePath.replace('/', '\\')
bat label: "Downloading ${win_filenamePath}",
script: "aws codecommit get-file --repository-name ${repositoryName} --commit-specifier ${branchName} --file-path ${filenamePath} --query fileContent --output text 2>${errorFile} > ${win_filenamePath}_encoded"
bat label: 'Decoding',
script: "certutil -decode ${win_filenamePath}_encoded ${win_filenamePath}"
}
palRm("${filenamePath}_encoded")
} catch (Exception ex) {
def error = ''
if(fileExists(errorFile)) {
error = readFile errorFile
}
if (!error || !(!failIfNotFound && error.contains('FileDoesNotExistException'))) {
palRm("${errorFile} ${filenamePath}.encoded ${filenamePath}")
throw new Exception("Could not get file: ${filenamePath}, ex: ${ex}, stderr: ${error}")
}
}
palRm(errorFile)
}
}
}
def SetLfsCredentials(cmd, lbl = '') {
if (env.IS_UNIX) {
sh label: lbl,
script: cmd
} else {
bat label: lbl,
script: cmd
}
}
def CheckoutBootstrapScripts(String branchName) {
checkout([$class: "GitSCM",
branches: [[name: "*/${branchName}"]],
doGenerateSubmoduleConfigurations: false,
extensions: [
[
$class: "SparseCheckoutPaths",
sparseCheckoutPaths: [
[ $class: "SparseCheckoutPath", path: "AutomatedReview/" ],
[ $class: "SparseCheckoutPath", path: "scripts/build/bootstrap/" ],
[ $class: "SparseCheckoutPath", path: "Tools/build/JenkinsScripts/build/Platform" ]
]
],
[
$class: "CloneOption", depth: 1, noTags: false, reference: "", shallow: true
]
],
submoduleCfg: [],
userRemoteConfigs: scm.userRemoteConfigs
])
}
def CheckoutRepo(boolean disableSubmodules = false) {
dir(ENGINE_REPOSITORY_NAME) {
palSh('git lfs uninstall', 'Git LFS Uninstall') // Prevent git from pulling lfs objects during checkout
if(fileExists('.git')) {
// If the repository after checkout is locked, likely we took a snapshot while git was running,
// to leave the repo in a usable state, garbagecollect. This also helps in situations where
def indexLockFile = '.git/index.lock'
if(fileExists(indexLockFile)) {
palSh('git gc', 'Git GarbageCollect')
}
if(fileExists(indexLockFile)) { // if it is still there, remove it
palRm(indexLockFile)
}
}
}
def random = new Random()
def retryAttempt = 0
retry(5) {
if (retryAttempt > 0) {
sleep random.nextInt(60 * retryAttempt) // Stagger checkouts to prevent HTTP 429 (Too Many Requests) response from CodeCommit
}
retryAttempt = retryAttempt + 1
if(params.PULL_REQUEST_ID) {
// This is a pull request build. Perform merge with destination branch before building.
dir(ENGINE_REPOSITORY_NAME) {
checkout scm: [
$class: 'GitSCM',
branches: scm.branches,
extensions: [
[$class: 'PreBuildMerge', options: [mergeRemote: 'origin', mergeTarget: params.DESTINATION_BRANCH]],
[$class: 'SubmoduleOption', disableSubmodules: disableSubmodules, recursiveSubmodules: true],
[$class: 'CheckoutOption', timeout: 60]
],
userRemoteConfigs: scm.userRemoteConfigs
]
}
} else {
dir(ENGINE_REPOSITORY_NAME) {
checkout scm: [
$class: 'GitSCM',
branches: scm.branches,
extensions: [
[$class: 'SubmoduleOption', disableSubmodules: disableSubmodules, recursiveSubmodules: true],
[$class: 'CheckoutOption', timeout: 60]
],
userRemoteConfigs: scm.userRemoteConfigs
]
}
}
}
// Add folder where we will store the 3rdParty downloads and packages
if(!fileExists('3rdParty')) {
palMkdir('3rdParty')
}
dir(ENGINE_REPOSITORY_NAME) {
// Run lfs in a separate step. Jenkins is unable to load the credentials for the custom LFS endpoint
withCredentials([usernamePassword(credentialsId: "${env.GITHUB_USER}", passwordVariable: 'accesstoken', usernameVariable: 'username')]) {
SetLfsCredentials("git config -f .lfsconfig lfs.url https://${username}:${accesstoken}@${env.LFS_URL}", 'Set credentials')
}
palSh('git lfs install', 'Git LFS Install')
palSh('git lfs pull', 'Git LFS Pull')
// CHANGE_ID is used by some scripts to identify uniquely the current change (usually metric jobs)
palSh('git rev-parse HEAD > commitid', 'Getting commit id')
env.CHANGE_ID = readFile file: 'commitid'
env.CHANGE_ID = env.CHANGE_ID.trim()
palRm('commitid')
}
}
def PreBuildCommonSteps(Map pipelineConfig, String projectName, String pipeline, String branchName, String platform, String buildType, String workspace, boolean mount = true, boolean disableSubmodules = false) {
echo 'Starting pre-build common steps...'
if (mount) {
unstash name: 'incremental_build_script'
def pythonCmd = ''
if(env.IS_UNIX) pythonCmd = 'sudo -E python -u '
else pythonCmd = 'python -u '
if(env.RECREATE_VOLUME.toBoolean()) {
palSh("${pythonCmd} ${INCREMENTAL_BUILD_SCRIPT_PATH} --action delete --project ${projectName} --pipeline ${pipeline} --branch ${branchName} --platform ${platform} --build_type ${buildType}", 'Deleting volume')
}
timeout(5) {
palSh("${pythonCmd} ${INCREMENTAL_BUILD_SCRIPT_PATH} --action mount --project ${projectName} --pipeline ${pipeline} --branch ${branchName} --platform ${platform} --build_type ${buildType}", 'Mounting volume')
}
if(env.IS_UNIX) {
sh label: 'Setting volume\'s ownership',
script: """
if sudo test ! -d "${workspace}"; then
sudo mkdir -p ${workspace}
cd ${workspace}/..
sudo chown -R lybuilder:root .
fi
"""
}
}
// Cleanup previous repo location, we are currently at the root of the workspace, if we have a .git folder
// we need to cleanup. Once all branches take this relocation, we can remove this
if(env.CLEAN_WORKSPACE.toBoolean() || fileExists("${workspace}/.git")) {
if(fileExists(workspace)) {
palRmDir(workspace)
}
}
dir(workspace) {
CheckoutRepo(disableSubmodules)
// Get python
dir(ENGINE_REPOSITORY_NAME) {
if(env.IS_UNIX) {
sh label: 'Getting python',
script: 'python/get_python.sh'
} else {
bat label: 'Getting python',
script: 'python/get_python.bat'
}
if(env.CLEAN_OUTPUT_DIRECTORY.toBoolean() || env.CLEAN_ASSETS.toBoolean()) {
def command = "${pipelineConfig.BUILD_ENTRY_POINT} --platform ${platform} --type clean"
if (env.IS_UNIX) {
sh label: "Running ${platform} clean",
script: "${pipelineConfig.PYTHON_DIR}/python.sh -u ${command}"
} else {
bat label: "Running ${platform} clean",
script: "${pipelineConfig.PYTHON_DIR}/python.cmd -u ${command}".replace('/','\\')
}
}
}
}
}
def Build(Map options, String platform, String type, String workspace) {
def command = "${options.BUILD_ENTRY_POINT} --platform ${platform} --type ${type}"
dir("${workspace}/${ENGINE_REPOSITORY_NAME}") {
if (env.IS_UNIX) {
sh label: "Running ${platform} ${type}",
script: "${options.PYTHON_DIR}/python.sh -u ${command}"
} else {
bat label: "Running ${platform} ${type}",
script: "${options.PYTHON_DIR}/python.cmd -u ${command}".replace('/','\\')
}
}
}
def TestMetrics(Map options, String workspace, String branchName, String repoName, String buildJobName, String outputDirectory, String configuration) {
catchError(buildResult: null, stageResult: null) {
def cmakeBuildDir = [workspace, ENGINE_REPOSITORY_NAME, outputDirectory].join('/')
dir("${workspace}/${ENGINE_REPOSITORY_NAME}") {
checkout scm: [
$class: 'GitSCM',
branches: [[name: '*/main']],
extensions: [[$class: 'RelativeTargetDirectory', relativeTargetDir: 'mars']],
userRemoteConfigs: [[url: "${env.MARS_REPO}", name: 'mars', credentialsId: "${env.GITHUB_USER}"]]
]
withCredentials([usernamePassword(credentialsId: "${env.SERVICE_USER}", passwordVariable: 'apitoken', usernameVariable: 'username')]) {
def command = "${options.PYTHON_DIR}/python.cmd -u mars/scripts/python/ctest_test_metric_scraper.py -e jenkins.creds.user ${username} -e jenkins.creds.pass ${apitoken} ${cmakeBuildDir} ${branchName} %BUILD_NUMBER% AR ${configuration} ${repoName} "
bat label: "Publishing ${buildJobName} Test Metrics",
script: command
}
}
}
}
def PostBuildCommonSteps(String workspace, boolean mount = true) {
echo 'Starting post-build common steps...'
if(params.PULL_REQUEST_ID) {
dir("${workspace}/${ENGINE_REPOSITORY_NAME}") {
if(fileExists('.git')) {
palSh('git reset --hard HEAD', 'Discard PR merge, git reset')
}
}
}
if (mount) {
def pythonCmd = ''
if(env.IS_UNIX) pythonCmd = 'sudo -E python -u '
else pythonCmd = 'python -u '
try {
timeout(5) {
palSh("${pythonCmd} ${INCREMENTAL_BUILD_SCRIPT_PATH} --action unmount", 'Unmounting volume')
}
} catch (Exception e) {
echo "Unmount script error ${e}"
}
}
}
def CreateSetupStage(Map pipelineConfig, String projectName, String pipelineName, String branchName, String platformName, String jobName, Map environmentVars) {
return {
stage("Setup") {
PreBuildCommonSteps(pipelineConfig, projectName, pipelineName, branchName, platformName, jobName, environmentVars['WORKSPACE'], environmentVars['MOUNT_VOLUME'])
}
}
}
def CreateBuildStage(Map pipelineConfig, String platformName, String jobName, Map environmentVars) {
return {
stage("${jobName}") {
Build(pipelineConfig, platformName, jobName, environmentVars['WORKSPACE'])
}
}
}
def CreateTestMetricsStage(Map pipelineConfig, String branchName, Map environmentVars, String buildJobName, String outputDirectory, String configuration) {
return {
stage("${buildJobName}_metrics") {
TestMetrics(pipelineConfig, environmentVars['WORKSPACE'], branchName, env.DEFAULT_REPOSITORY_NAME, buildJobName, outputDirectory, configuration)
}
}
}
def CreateTeardownStage(Map environmentVars) {
return {
stage("Teardown") {
PostBuildCommonSteps(environmentVars['WORKSPACE'], environmentVars['MOUNT_VOLUME'])
}
}
}
def projectName = ''
def pipelineName = ''
def branchName = ''
def pipelineConfig = {}
// Start Pipeline
try {
stage('Setup Pipeline') {
node('controller') {
def envVarList = []
if(isUnix()) {
envVarList.add('IS_UNIX=1')
}
withEnv(envVarList) {
timestamps {
(projectName, pipelineName) = GetRunningPipelineName(env.JOB_NAME) // env.JOB_NAME is the name of the job given by Jenkins
scmType = GetSCMType()
if(env.BRANCH_NAME) {
branchName = env.BRANCH_NAME
} else {
branchName = scm.branches[0].name // for non-multibranch pipelines
env.BRANCH_NAME = branchName // so scripts that read this environment have it (e.g. incremental_build_util.py)
}
pipelineProperties.add(disableConcurrentBuilds())
echo "Running \"${pipelineName}\" for \"${branchName}\"..."
if (scmType == 'github') {
CheckoutBootstrapScripts(branchName)
}
// Load configs
pipelineConfig = LoadPipelineConfig(pipelineName, branchName, scmType)
// Add each platform as a parameter that the user can disable if needed
pipelineConfig.platforms.each { platform ->
pipelineParameters.add(booleanParam(defaultValue: true, description: '', name: platform.key))
}
pipelineProperties.add(parameters(pipelineParameters))
properties(pipelineProperties)
// Stash the INCREMENTAL_BUILD_SCRIPT_PATH since all nodes will use it
if (scmType == 'codecommit') {
PullFilesFromGit(INCREMENTAL_BUILD_SCRIPT_PATH, branchName, true, ENGINE_REPOSITORY_NAME)
}
stash name: 'incremental_build_script',
includes: INCREMENTAL_BUILD_SCRIPT_PATH
}
}
}
}
if(env.BUILD_NUMBER == '1') {
// Exit pipeline early on the intial build. This allows Jenkins to load the pipeline for the branch and enables users
// to select build parameters on their first actual build. See https://issues.jenkins.io/browse/JENKINS-41929
currentBuild.result = 'SUCCESS'
return
}
// Build and Post-Build Testing Stage
def buildConfigs = [:]
// Platform Builds run on EC2
pipelineConfig.platforms.each { platform ->
platform.value.build_types.each { build_job ->
if (IsJobEnabled(build_job, pipelineName, platform.key)) { // User can filter jobs, jobs are tagged by pipeline
def envVars = GetBuildEnvVars(platform.value.PIPELINE_ENV ?: EMPTY_JSON, build_job.value.PIPELINE_ENV ?: EMPTY_JSON, pipelineName)
envVars['JOB_NAME'] = "${branchName}_${platform.key}_${build_job.key}" // backwards compatibility, some scripts rely on this
def nodeLabel = envVars['NODE_LABEL']
buildConfigs["${platform.key} [${build_job.key}]"] = {
node("${nodeLabel}") {
if(isUnix()) { // Has to happen inside a node
envVars['IS_UNIX'] = 1
}
withEnv(GetEnvStringList(envVars)) {
timeout(time: envVars['TIMEOUT'], unit: 'MINUTES', activity: true) {
try {
def build_job_name = build_job.key
CreateSetupStage(pipelineConfig, projectName, pipelineName, branchName, platform.key, build_job.key, envVars).call()
if(build_job.value.steps) { //this is a pipe with many steps so create all the build stages
build_job.value.steps.each { build_step ->
build_job_name = build_step
CreateBuildStage(pipelineConfig, platform.key, build_step, envVars).call()
}
} else {
CreateBuildStage(pipelineConfig, platform.key, build_job.key, envVars).call()
}
if (env.MARS_REPO && platform.key == 'Windows' && build_job_name.startsWith('test')) {
def output_directory = platform.value.build_types[build_job_name].PARAMETERS.OUTPUT_DIRECTORY
def configuration = platform.value.build_types[build_job_name].PARAMETERS.CONFIGURATION
CreateTestMetricsStage(pipelineConfig, branchName, envVars, build_job_name, output_directory, configuration).call()
}
}
catch(Exception e) {
// https://github.com/jenkinsci/jenkins/blob/master/core/src/main/java/hudson/model/Result.java
// {SUCCESS,UNSTABLE,FAILURE,NOT_BUILT,ABORTED}
def currentResult = envVars['ON_FAILURE_MARK'] ?: 'FAILURE'
if (currentResult == 'FAILURE') {
currentBuild.result = 'FAILURE'
error "FAILURE: ${e}"
} else if (currentResult == 'UNSTABLE') {
currentBuild.result = 'UNSTABLE'
unstable(message: "UNSTABLE: ${e}")
}
}
finally {
CreateTeardownStage(envVars).call()
}
}
}
}
}
}
}
}
timestamps {
stage('Build') {
parallel buildConfigs // Run parallel builds
}
echo 'All builds successful'
}
}
catch(Exception e) {
error "Exception: ${e}"
}
finally {
try {
if(env.SNS_TOPIC) {
snsPublish(
topicArn: env.SNS_TOPIC,
subject:'Build Result',
message:"${currentBuild.currentResult}:${params.REPOSITORY_NAME}:${params.SOURCE_BRANCH}:${params.SOURCE_COMMIT}:${params.DESTINATION_COMMIT}:${params.PULL_REQUEST_ID}:${BUILD_URL}:${env.RECREATE_VOLUME}:${env.CLEAN_OUTPUT_DIRECTORY}:${env.CLEAN_ASSETS}"
)
}
step([
$class: 'Mailer',
notifyEveryUnstableBuild: true,
sendToIndividuals: true,
recipients: emailextrecipients([
[$class: 'CulpritsRecipientProvider'],
[$class: 'RequesterRecipientProvider']
])
])
} catch(Exception e) {
}
}
AutomatedReview/lumberyard.json
-12
View File
@@ -1,12 +0,0 @@
{
"BUILD_ENTRY_POINT": "Tools/build/JenkinsScripts/build/ci_build.py",
"PIPELINE_CONFIGS": [
"Tools/build/JenkinsScripts/build/Platform/*/pipeline.json",
"restricted/*/Tools/build/JenkinsScripts/build/pipeline.json"
],
"BUILD_CONFIGS": [
"Tools/build/JenkinsScripts/build/Platform/*/build_config.json",
"restricted/*/Tools/build/JenkinsScripts/build/build_config.json"
],
"PYTHON_DIR": "python"
}
AutomatedTesting/CMakeLists.txt
+33 -7
View File
@@ -9,12 +9,38 @@
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
#
file(READ "${CMAKE_CURRENT_LIST_DIR}/project.json" project_json)
#! Adds the --project-path argument to the VS IDE debugger command arguments
function(add_vs_debugger_arguments)
# Inject the project root into the --project-path argument into the Visual Studio Debugger arguments by defaults
list(APPEND app_targets AutomatedTesting.GameLauncher AutomatedTesting.ServerLauncher)
list(APPEND app_targets AssetBuilder AssetProcessor AssetProcessorBatch Editor)
foreach(app_target IN LISTS app_targets)
if (TARGET ${app_target})
set_property(TARGET ${app_target} APPEND PROPERTY VS_DEBUGGER_COMMAND_ARGUMENTS "--project-path=\"${CMAKE_CURRENT_LIST_DIR}\"")
endif()
endforeach()
endfunction()
string(JSON project_target_name ERROR_VARIABLE json_error GET ${project_json} "project_name")
if(${json_error})
message(FATAL_ERROR "Unable to read key 'project_name' from 'project.json'")
endif()
if(NOT PROJECT_NAME)
cmake_minimum_required(VERSION 3.19)
project(AutomatedTesting
LANGUAGES C CXX
VERSION 1.0.0.0
)
include(EngineFinder.cmake OPTIONAL)
find_package(o3de REQUIRED)
o3de_initialize()
add_vs_debugger_arguments()
else()
# Add the project_name to global LY_PROJECTS_TARGET_NAME property
file(READ "${CMAKE_CURRENT_LIST_DIR}/project.json" project_json)
set_property(GLOBAL APPEND PROPERTY LY_PROJECTS_TARGET_NAME ${project_target_name})
add_subdirectory(Gem)
string(JSON project_target_name ERROR_VARIABLE json_error GET ${project_json} "project_name")
if(json_error)
message(FATAL_ERROR "Unable to read key 'project_name' from 'project.json'")
endif()
set_property(GLOBAL APPEND PROPERTY LY_PROJECTS_TARGET_NAME ${project_target_name})
add_subdirectory(Gem)
endif()
AutomatedTesting/Gem/Code/runtime_dependencies.cmake
+1 -1
View File
@@ -42,7 +42,6 @@ set(GEM_DEPENDENCIES
Gem::SurfaceData
Gem::GradientSignal
Gem::Vegetation
Gem::Atom_RHI.Private
Gem::Atom_RPI.Private
Gem::Atom_Feature_Common
@@ -54,4 +53,5 @@ set(GEM_DEPENDENCIES
Gem::ImguiAtom
Gem::Atom_AtomBridge
Gem::AtomFont
Gem::Blast
)
AutomatedTesting/Gem/Code/tool_dependencies.cmake
+1
View File
@@ -68,4 +68,5 @@ set(GEM_DEPENDENCIES
Gem::ImguiAtom
Gem::AtomFont
Gem::AtomToolsFramework.Editor
Gem::Blast.Editor
)
AutomatedTesting/Gem/PythonTests/Blast/TestSuite_Active.py
+7 -7
View File
@@ -27,28 +27,28 @@ from base import TestAutomationBase
class TestAutomation(TestAutomationBase):
def test_ActorSplitsAfterCollision(self, request, workspace, editor, launcher_platform):
from . import ActorSplitsAfterCollision as test_module
self._run_test(request, workspace, editor, test_module, expected_lines=[], unexpected_lines=["Assert"])
self._run_test(request, workspace, editor, test_module)
def test_ActorSplitsAfterRadialDamage(self, request, workspace, editor, launcher_platform):
from . import ActorSplitsAfterRadialDamage as test_module
self._run_test(request, workspace, editor, test_module, expected_lines=[], unexpected_lines=["Assert"])
self._run_test(request, workspace, editor, test_module)
def test_ActorSplitsAfterCapsuleDamage(self, request, workspace, editor, launcher_platform):
from . import ActorSplitsAfterCapsuleDamage as test_module
self._run_test(request, workspace, editor, test_module, expected_lines=[], unexpected_lines=["Assert"])
self._run_test(request, workspace, editor, test_module)
def test_ActorSplitsAfterImpactSpreadDamage(self, request, workspace, editor, launcher_platform):
from . import ActorSplitsAfterImpactSpreadDamage as test_module
self._run_test(request, workspace, editor, test_module, expected_lines=[], unexpected_lines=["Assert"])
self._run_test(request, workspace, editor, test_module)
def test_ActorSplitsAfterShearDamage(self, request, workspace, editor, launcher_platform):
from . import ActorSplitsAfterShearDamage as test_module
self._run_test(request, workspace, editor, test_module, expected_lines=[], unexpected_lines=["Assert"])
self._run_test(request, workspace, editor, test_module)
def test_ActorSplitsAfterTriangleDamage(self, request, workspace, editor, launcher_platform):
from . import ActorSplitsAfterTriangleDamage as test_module
self._run_test(request, workspace, editor, test_module, expected_lines=[], unexpected_lines=["Assert"])
self._run_test(request, workspace, editor, test_module)
def test_ActorSplitsAfterStressDamage(self, request, workspace, editor, launcher_platform):
from . import ActorSplitsAfterStressDamage as test_module
self._run_test(request, workspace, editor, test_module, expected_lines=[], unexpected_lines=["Assert"])
self._run_test(request, workspace, editor, test_module)
AutomatedTesting/Gem/PythonTests/CMakeLists.txt
+12 -14
View File
@@ -135,20 +135,18 @@ if(PAL_TRAIT_BUILD_TESTS_SUPPORTED AND PAL_TRAIT_BUILD_HOST_TOOLS)
endif()
## Blast ##
# Disabled until AutomatedTesting runs with Atom.
# if(PAL_TRAIT_BUILD_TESTS_SUPPORTED AND PAL_TRAIT_BUILD_HOST_TOOLS)
# ly_add_pytest(
# NAME AutomatedTesting::BlastTests
# TEST_SERIAL TRUE
# PATH ${CMAKE_CURRENT_LIST_DIR}/Blast/TestSuite_Active.py
# TIMEOUT 500
# RUNTIME_DEPENDENCIES
# Legacy::Editor
# Legacy::CryRenderNULL
# AZ::AssetProcessor
# AutomatedTesting.Assets
# )
# endif()
if(PAL_TRAIT_BUILD_TESTS_SUPPORTED AND PAL_TRAIT_BUILD_HOST_TOOLS)
ly_add_pytest(
NAME AutomatedTesting::BlastTests
TEST_SERIAL TRUE
PATH ${CMAKE_CURRENT_LIST_DIR}/Blast/TestSuite_Active.py
TIMEOUT 3600
RUNTIME_DEPENDENCIES
Legacy::Editor
AZ::AssetProcessor
AutomatedTesting.Assets
)
endif()
#############
AutomatedTesting/ShaderLib/raytracingscenesrg.srgi
+28
View File
@@ -0,0 +1,28 @@
/*
* All or portions of this file Copyright (c) Amazon.com, Inc. or its affiliates or
* its licensors.
*
* For complete copyright and license terms please see the LICENSE at the root of this
* distribution (the "License"). All use of this software is governed by the License,
* or, if provided, by the license below or the license accompanying this file. Do not
* remove or modify any license notices. This file is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
*
*/
#pragma once
// Please read README.md for an explanation on why scenesrg.srgi and viewsrg.srgi are
// located in this folder (And how you can optionally customize your own scenesrg.srgi
// and viewsrg.srgi in your game project).
#include <Atom/Features/SrgSemantics.azsli>
partial ShaderResourceGroup RayTracingSceneSrg : SRG_RayTracingScene
{
/* Intentionally Empty. Helps define the SrgSemantic for RayTracingSceneSrg once.*/
};
#define AZ_COLLECTING_PARTIAL_SRGS
#include <Atom/Feature/Common/Assets/ShaderResourceGroups/RayTracingSceneSrgAll.azsli>
#undef AZ_COLLECTING_PARTIAL_SRGS
AutomatedTesting/default.blastconfiguration
+1 -1
View File
@@ -1,6 +1,6 @@
<ObjectStream version="3">
<Class name="BlastGlobalConfiguration" version="1" type="{0B9DB6DD-0008-4EF6-9D75-141061144353}">
<Class name="Asset" field="BlastMaterialLibrary" value="id={251AC171-6B9C-562D-A235-4EF5E1AE6871}:0,type={55F38C86-0767-4E7F-830A-A4BF624BE4DA},hint={assets/destruction/automated_testing.blastmaterial}" version="1" type="{77A19D40-8731-4D3C-9041-1B43047366A4}"/>
<Class name="Asset" field="BlastMaterialLibrary" value="id={251AC171-6B9C-562D-A235-4EF5E1AE6871}:0,type={55F38C86-0767-4E7F-830A-A4BF624BE4DA},hint={assets/destruction/automated_testing.blastmaterial},loadBehavior=1" version="2" type="{77A19D40-8731-4D3C-9041-1B43047366A4}"/>
<Class name="unsigned int" field="StressSolverIterations" value="180" type="{43DA906B-7DEF-4CA8-9790-854106D3F983}"/>
</Class>
</ObjectStream>
Code/CryEngine/Cry3DEngine/cvars.cpp
+1 -1
View File
@@ -922,7 +922,7 @@ void CVars::Init()
"Will not render CGFs past the given amount of drawcalls\n"
"(<=0 off (default), >0 draw calls limit)");
REGISTER_CVAR(e_CheckOctreeObjectsBoxSize, 1, VF_NULL, "CryWarning for crazy sized COctreeNode m_objectsBoxes");
REGISTER_CVAR(e_CheckOctreeObjectsBoxSize, 1, VF_NULL, "Warning for crazy sized COctreeNode m_objectsBoxes");
REGISTER_CVAR(e_DebugGeomPrep, 0, VF_NULL, "enable logging of Geom preparation");
DefineConstIntCVar(e_GeomCaches, 1, VF_NULL, "Activates drawing of geometry caches");
REGISTER_CVAR(e_GeomCacheBufferSize, 128, VF_CHEAT, "Geometry cache stream buffer upper limit size in MB. Default: 128");
Code/CryEngine/CrySystem/AZCrySystemInitLogSink.cpp
+2 -2
View File
@@ -35,7 +35,7 @@ namespace AZ
}
AZ::OSString msgBoxMessage;
msgBoxMessage.append("CrySystem could not initialize correctly for the following reason(s):");
msgBoxMessage.append("O3DE could not initialize correctly for the following reason(s):");
for (const AZ::OSString& errMsg : m_errorStringsCollected)
{
@@ -47,7 +47,7 @@ namespace AZ
Trace::Output(nullptr, msgBoxMessage.c_str());
Trace::Output(nullptr, "\n==================================================================\n");
EBUS_EVENT(AZ::NativeUI::NativeUIRequestBus, DisplayOkDialog, "CrySystem Initialization Failed", msgBoxMessage.c_str(), false);
EBUS_EVENT(AZ::NativeUI::NativeUIRequestBus, DisplayOkDialog, "O3DE Initialization Failed", msgBoxMessage.c_str(), false);
}
} // namespace Debug
} // namespace AZ
Code/CryEngine/CrySystem/SystemWin32.cpp
+3 -3
View File
@@ -605,7 +605,7 @@ void CSystem::DebugStats([[maybe_unused]] bool checkpoint, [[maybe_unused]] bool
{
if (!dbgmodules[i].handle)
{
CryLogAlways("WARNING: <CrySystem> CSystem::DebugStats: NULL handle for %s", dbgmodules[i].name.c_str());
CryLogAlways("WARNING: CSystem::DebugStats: NULL handle for %s", dbgmodules[i].name.c_str());
nolib++;
continue;
}
@@ -642,7 +642,7 @@ void CSystem::DebugStats([[maybe_unused]] bool checkpoint, [[maybe_unused]] bool
}
else
{
CryLogAlways("WARNING: <CrySystem> CSystem::DebugStats: could not retrieve function from DLL %s", dbgmodules[i].name.c_str());
CryLogAlways("WARNING: CSystem::DebugStats: could not retrieve function from DLL %s", dbgmodules[i].name.c_str());
nolib++;
};
#endif
@@ -1066,7 +1066,7 @@ void CSystem::FatalError(const char* format, ...)
if (szSysErrorMessage)
{
CryLogAlways("<CrySystem> Last System Error: %s", szSysErrorMessage);
CryLogAlways("Last System Error: %s", szSysErrorMessage);
}
if (GetUserCallback())
Code/Framework/AzCore/AzCore/Asset/AssetManager.cpp
+1 -1
View File
@@ -1117,7 +1117,7 @@ namespace AZ
return asset;
}
void AssetManager::UpdateDebugStatus(AZ::Data::Asset<AZ::Data::AssetData> asset)
void AssetManager::UpdateDebugStatus(const AZ::Data::Asset<AZ::Data::AssetData>& asset)
{
if(!m_debugAssetEvents)
{
Code/Framework/AzCore/AzCore/Asset/AssetManager.h
+1 -1
View File
@@ -358,7 +358,7 @@ namespace AZ
Asset<AssetData> GetAssetInternal(const AssetId& assetId, const AssetType& assetType, AssetLoadBehavior assetReferenceLoadBehavior, const AssetLoadParameters& loadParams = AssetLoadParameters{}, AssetInfo assetInfo = AssetInfo(), bool signalLoaded = false);
void UpdateDebugStatus(AZ::Data::Asset<AZ::Data::AssetData> asset);
void UpdateDebugStatus(const AZ::Data::Asset<AZ::Data::AssetData>& asset);
/**
* Gets a root asset and dependencies as individual async loads if necessary.
Code/Framework/AzCore/AzCore/Component/NonUniformScaleBus.h
-3
View File
@@ -19,9 +19,6 @@ namespace AZ
{
class Vector3;
//! Do not allow the scale to be zero to avoid problems with inverting scale.
static constexpr float MinNonUniformScale = 1e-3f;
using NonUniformScaleChangedEvent = AZ::Event<const AZ::Vector3&>;
//! Requests for working with non-uniform scale.
Code/Framework/AzCore/AzCore/Debug/EventTrace.h
+2 -2
View File
@@ -42,9 +42,8 @@ namespace AZ
}
}
#define AZ_TRACE_METHOD_NAME_CATEGORY(name, category) AZ::Debug::EventTrace::ScopedSlice AZ_JOIN(ScopedSlice__, __LINE__)(name, category);
#ifdef AZ_PROFILE_TELEMETRY
# define AZ_TRACE_METHOD_NAME_CATEGORY(name, category) AZ::Debug::EventTrace::ScopedSlice AZ_JOIN(ScopedSlice__, __LINE__)(name, category);
# define AZ_TRACE_METHOD_NAME(name) \
AZ_TRACE_METHOD_NAME_CATEGORY(name, "") \
AZ_PROFILE_SCOPE(AZ::Debug::ProfileCategory::AzTrace, name)
@@ -53,6 +52,7 @@ namespace AZ
AZ_TRACE_METHOD_NAME_CATEGORY(AZ_FUNCTION_SIGNATURE, "") \
AZ_PROFILE_FUNCTION(AZ::Debug::ProfileCategory::AzTrace)
#else
# define AZ_TRACE_METHOD_NAME_CATEGORY(name, category)
# define AZ_TRACE_METHOD_NAME(name) AZ_TRACE_METHOD_NAME_CATEGORY(name, "")
# define AZ_TRACE_METHOD() AZ_TRACE_METHOD_NAME(AZ_FUNCTION_SIGNATURE)
#endif
Code/Framework/AzCore/AzCore/Math/Transform.h
+7
View File
@@ -38,6 +38,13 @@ namespace AZ
bool CompareValueData(const void* lhs, const void* rhs) override;
};
//! Limits for transform scale values.
//! The scale should not be zero to avoid problems with inverting.
//! @{
static constexpr float MinTransformScale = 1e-2f;
static constexpr float MaxTransformScale = 1e9f;
//! @}
//! The basic transformation class, represented using a quaternion rotation, vector scale and vector translation.
//! By design, cannot represent skew transformations.
class Transform
Code/Framework/AzCore/AzCore/UnitTest/Mocks/MockSettingsRegistry.h
+59
View File
@@ -0,0 +1,59 @@
/*
* All or portions of this file Copyright (c) Amazon.com, Inc. or its affiliates or
* its licensors.
*
* For complete copyright and license terms please see the LICENSE at the root of this
* distribution (the "License"). All use of this software is governed by the License,
* or, if provided, by the license below or the license accompanying this file. Do not
* remove or modify any license notices. This file is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
*
*/
#pragma once
#include <AzCore/UnitTest/UnitTest.h>
#include <AzCore/Settings/SettingsRegistry.h>
#include <gmock/gmock.h>
namespace AZ
{
class MockSettingsRegistry;
using NiceSettingsRegistrySimpleMock = ::testing::NiceMock<MockSettingsRegistry>;
class MockSettingsRegistry
: public AZ::SettingsRegistryInterface
{
public:
MOCK_CONST_METHOD1(GetType, Type(AZStd::string_view));
MOCK_CONST_METHOD2(Visit, bool(Visitor&, AZStd::string_view));
MOCK_CONST_METHOD2(Visit, bool(const VisitorCallback&, AZStd::string_view));
MOCK_METHOD1(RegisterNotifier, NotifyEventHandler(const NotifyCallback&));
MOCK_METHOD1(RegisterNotifier, NotifyEventHandler(NotifyCallback&&));
MOCK_CONST_METHOD2(Get, bool(bool&, AZStd::string_view));
MOCK_CONST_METHOD2(Get, bool(s64&, AZStd::string_view));
MOCK_CONST_METHOD2(Get, bool(u64&, AZStd::string_view));
MOCK_CONST_METHOD2(Get, bool(double&, AZStd::string_view));
MOCK_CONST_METHOD2(Get, bool(AZStd::string&, AZStd::string_view));
MOCK_CONST_METHOD2(Get, bool(FixedValueString&, AZStd::string_view));
MOCK_CONST_METHOD3(GetObject, bool(void*, Uuid, AZStd::string_view));
MOCK_METHOD2(Set, bool(AZStd::string_view, bool));
MOCK_METHOD2(Set, bool(AZStd::string_view, s64));
MOCK_METHOD2(Set, bool(AZStd::string_view, u64));
MOCK_METHOD2(Set, bool(AZStd::string_view, double));
MOCK_METHOD2(Set, bool(AZStd::string_view, AZStd::string_view));
MOCK_METHOD2(Set, bool(AZStd::string_view, const char*));
MOCK_METHOD3(SetObject, bool(AZStd::string_view, const void*, Uuid));
MOCK_METHOD1(Remove, bool(AZStd::string_view));
MOCK_METHOD3(MergeCommandLineArgument, bool(AZStd::string_view, AZStd::string_view, const CommandLineArgumentSettings&));
MOCK_METHOD2(MergeSettings, bool(AZStd::string_view, Format));
MOCK_METHOD4(MergeSettingsFile, bool(AZStd::string_view, Format, AZStd::string_view, AZStd::vector<char>*));
MOCK_METHOD5(
MergeSettingsFolder,
bool(AZStd::string_view, const Specializations&, AZStd::string_view, AZStd::string_view, AZStd::vector<char>*));
};
} // namespace AZ
Code/Framework/AzCore/AzCore/azcoretestcommon_files.cmake
+1
View File
@@ -15,4 +15,5 @@ set(FILES
UnitTest/UnitTest.h
UnitTest/TestTypes.h
UnitTest/Mocks/MockFileIOBase.h
UnitTest/Mocks/MockSettingsRegistry.h
)
Code/Framework/AzFramework/AzFramework/Asset/AssetCatalog.cpp
+2 -2
View File
@@ -617,9 +617,9 @@ namespace AzFramework
// won't free the mutex until the load is complete.
// So instead, queue the notification until the next tick, so that it doesn't occur within the AssetCatalogRequestBus mutex, and also
// so that the entire AssetCatalog initialization is complete.
AZ::TickBus::QueueFunction([catalogRegistryFile]()
AZ::TickBus::QueueFunction([catalogRegistryString = AZStd::string(catalogRegistryFile)]()
{
AssetCatalogEventBus::Broadcast(&AssetCatalogEventBus::Events::OnCatalogLoaded, catalogRegistryFile);
AssetCatalogEventBus::Broadcast(&AssetCatalogEventBus::Events::OnCatalogLoaded, catalogRegistryString.c_str());
});
}
}
Code/Framework/AzFramework/AzFramework/Components/NonUniformScaleComponent.cpp
+3 -2
View File
@@ -12,6 +12,7 @@
#include <AzFramework/Components/NonUniformScaleComponent.h>
#include <AzCore/Serialization/SerializeContext.h>
#include <AzCore/Math/Transform.h>
#include <AzCore/Math/ToString.h>
#include <AzCore/Component/Entity.h>
@@ -81,13 +82,13 @@ namespace AzFramework
void NonUniformScaleComponent::SetScale(const AZ::Vector3& scale)
{
if (scale.GetMinElement() >= AZ::MinNonUniformScale)
if (scale.GetMinElement() >= AZ::MinTransformScale && scale.GetMaxElement() <= AZ::MaxTransformScale)
{
m_scale = scale;
}
else
{
AZ::Vector3 clampedScale = scale.GetMax(AZ::Vector3(AZ::MinNonUniformScale));
AZ::Vector3 clampedScale = scale.GetClamp(AZ::Vector3(AZ::MinTransformScale), AZ::Vector3(AZ::MaxTransformScale));
AZ_Warning("Non-uniform Scale Component", false, "SetScale value was clamped from %s to %s for entity %s",
AZ::ToString(scale).c_str(), AZ::ToString(clampedScale).c_str(), GetEntity()->GetName().c_str());
m_scale = clampedScale;
Code/Framework/AzToolsFramework/AzToolsFramework/ToolsComponents/EditorNonUniformScaleComponent.cpp
+6 -3
View File
@@ -13,6 +13,7 @@
#include <AzToolsFramework/ToolsComponents/EditorNonUniformScaleComponent.h>
#include <AzCore/Serialization/EditContext.h>
#include <AzFramework/Components/NonUniformScaleComponent.h>
#include <AzCore/Math/Transform.h>
#include <AzCore/Math/ToString.h>
namespace AzToolsFramework
@@ -44,7 +45,9 @@ namespace AzToolsFramework
->DataElement(
AZ::Edit::UIHandlers::Default, &EditorNonUniformScaleComponent::m_scale, "Non-uniform Scale",
"Non-uniform scale for this entity only (does not propagate through hierarchy)")
->Attribute(AZ::Edit::Attributes::Min, AZ::MinNonUniformScale)
->Attribute(AZ::Edit::Attributes::Min, AZ::MinTransformScale)
->Attribute(AZ::Edit::Attributes::Max, AZ::MaxTransformScale)
->Attribute(AZ::Edit::Attributes::Step, 0.1f)
->Attribute(AZ::Edit::Attributes::ChangeNotify, &EditorNonUniformScaleComponent::OnScaleChanged)
;
}
@@ -106,13 +109,13 @@ namespace AzToolsFramework
void EditorNonUniformScaleComponent::SetScale(const AZ::Vector3& scale)
{
if (scale.GetMinElement() >= AZ::MinNonUniformScale)
if (scale.GetMinElement() >= AZ::MinTransformScale && scale.GetMaxElement() <= AZ::MaxTransformScale)
{
m_scale = scale;
}
else
{
AZ::Vector3 clampedScale = scale.GetMax(AZ::Vector3(AZ::MinNonUniformScale));
AZ::Vector3 clampedScale = scale.GetClamp(AZ::Vector3(AZ::MinTransformScale), AZ::Vector3(AZ::MaxTransformScale));
AZ_Warning("Editor Non-uniform Scale Component", false, "SetScale value was clamped from %s to %s for entity %s",
AZ::ToString(scale).c_str(), AZ::ToString(clampedScale).c_str(), GetEntity()->GetName().c_str());
m_scale = clampedScale;
Code/Framework/AzToolsFramework/AzToolsFramework/ToolsComponents/TransformComponent.cpp
-1
View File
@@ -1276,7 +1276,6 @@ namespace AzToolsFramework
Attribute(AZ::Edit::Attributes::SliceFlags, AZ::Edit::SliceFlags::NotPushableOnSliceRoot)->
DataElement(TransformScaleHandler, &EditorTransform::m_scale, "Scale", "Local Scale")->
Attribute(AZ::Edit::Attributes::Step, 0.1f)->
Attribute(AZ::Edit::Attributes::Min, 0.01f)->
Attribute(AZ::Edit::Attributes::ReadOnly, &EditorTransform::m_locked)
;
}
Code/Framework/AzToolsFramework/AzToolsFramework/ToolsComponents/TransformScalePropertyHandler.cpp
+3 -2
View File
@@ -12,6 +12,7 @@
#include "AzToolsFramework_precompiled.h"
#include <ToolsComponents/TransformScalePropertyHandler.h>
#include <AzCore/Math/Transform.h>
#include <AzCore/Math/Vector3.h>
namespace AzToolsFramework
@@ -36,8 +37,8 @@ namespace AzToolsFramework
AzToolsFramework::PropertyEditorGUIMessages::Bus::Broadcast(&AzToolsFramework::PropertyEditorGUIMessages::RequestWrite, newCtrl);
});
newCtrl->setMinimum(0.01f);
newCtrl->setMaximum(std::numeric_limits<float>::max());
newCtrl->setMinimum(AZ::MinTransformScale);
newCtrl->setMaximum(AZ::MaxTransformScale);
return newCtrl;
}
Code/Framework/AzToolsFramework/AzToolsFramework/Viewport/EditorContextMenu.cpp
+1 -1
View File
@@ -65,7 +65,7 @@ namespace AzToolsFramework
if (!contextMenu.m_menu->isEmpty())
{
contextMenu.m_menu->popup(QCursor::pos());
contextMenu.m_menu->exec(QCursor::pos());
}
}
}
Code/Framework/AzToolsFramework/AzToolsFramework/ViewportSelection/EditorTransformComponentSelection.cpp
+1 -1
View File
@@ -1603,7 +1603,7 @@ namespace AzToolsFramework
const AZ::Vector3 uniformScale = AZ::Vector3(action.m_start.m_sign * sumVectorElements(action.LocalScaleOffset()));
const AZ::Vector3 scale = (AZ::Vector3::CreateOne() +
(uniformScale / initialScale)).GetMax(AZ::Vector3(0.01f));
(uniformScale / initialScale)).GetClamp(AZ::Vector3(AZ::MinTransformScale), AZ::Vector3(AZ::MaxTransformScale));
const AZ::Transform scaleTransform = AZ::Transform::CreateScale(scale);
if (action.m_modifiers.Alt())
Code/LauncherUnified/Launcher.cpp
+1 -1
View File
@@ -313,7 +313,7 @@ namespace O3DELauncher
return "Failed to initialize the CrySystem Interface";
case ReturnCode::ErrCryEnvironment:
return "Failed to initialize the CryEngine global environment";
return "Failed to initialize the global environment";
case ReturnCode::ErrAssetProccessor:
return "Failed to connect to AssetProcessor while the /Amazon/AzCore/Bootstrap/wait_for_connect value is 1\n."
Code/LauncherUnified/Platform/iOS/LumberyardApplicationDelegate_iOS.mm → Code/LauncherUnified/Platform/iOS/O3DEApplicationDelegate_iOS.mm
+10 -10
View File
@@ -39,15 +39,15 @@ namespace
}
@interface LumberyardApplicationDelegate_iOS : NSObject<UIApplicationDelegate>
@interface O3DEApplicationDelegate_iOS : NSObject<UIApplicationDelegate>
{
}
@end // LumberyardApplicationDelegate_iOS Interface
@end // O3DEApplicationDelegate_iOS Interface
@implementation LumberyardApplicationDelegate_iOS
@implementation O3DEApplicationDelegate_iOS
- (int)runLumberyardApplication
- (int)runO3DEApplication
{
#if AZ_TESTS_ENABLED
@@ -55,7 +55,7 @@ namespace
return static_cast<int>(ReturnCode::ErrUnitTestNotSupported);
#else
using namespace LumberyardLauncher;
using namespace O3DELauncher;
PlatformMainInfo mainInfo;
mainInfo.m_updateResourceLimits = IncreaseResourceLimits;
@@ -79,19 +79,19 @@ namespace
#endif // AZ_TESTS_ENABLED
}
- (void)launchLumberyardApplication
- (void)launchO3DEApplication
{
const int exitCode = [self runLumberyardApplication];
const int exitCode = [self runO3DEApplication];
exit(exitCode);
}
- (BOOL)application:(UIApplication*)application didFinishLaunchingWithOptions:(NSDictionary*)launchOptions
{
// prevent the lumberyard runtime from running when launched in a xctest environment, otherwise the
// prevent the o3de runtime from running when launched in a xctest environment, otherwise the
// testing framework will kill the "app" due to the lengthy bootstrap process
if ([[NSProcessInfo processInfo] environment][@"XCTestConfigurationFilePath"] == nil)
{
[self performSelector:@selector(launchLumberyardApplication) withObject:nil afterDelay:0.0];
[self performSelector:@selector(launchO3DEApplication) withObject:nil afterDelay:0.0];
}
return YES;
}
@@ -132,4 +132,4 @@ namespace
&AzFramework::IosLifecycleEvents::Bus::Events::OnDidReceiveMemoryWarning);
}
@end // LumberyardApplicationDelegate_iOS Implementation
@end // O3DEApplicationDelegate_iOS Implementation
Code/LauncherUnified/Platform/iOS/LumberyardApplication_iOS.mm → Code/LauncherUnified/Platform/iOS/O3DEApplication_iOS.mm
+4 -4
View File
@@ -16,12 +16,12 @@
#include <AzFramework/Input/Buses/Notifications/RawInputNotificationBus_Platform.h>
@interface LumberyardApplication_iOS : UIApplication
@interface O3DEApplication_iOS : UIApplication
{
}
@end // LumberyardApplication_iOS Interface
@end // O3DEApplication_iOS Interface
@implementation LumberyardApplication_iOS
@implementation O3DEApplication_iOS
- (void)touchesBegan: (NSSet<UITouch*>*)touches withEvent: (UIEvent*)event
{
@@ -65,4 +65,4 @@
[self touchesEnded: touches withEvent: event];
}
@end // LumberyardApplication_iOS Implementation
@end // O3DEApplication_iOS Implementation
Code/LauncherUnified/Platform/iOS/platform_ios_files.cmake
+2 -2
View File
@@ -13,8 +13,8 @@ set(FILES
Launcher_iOS.mm
Launcher_Traits_iOS.h
Launcher_Traits_Platform.h
LumberyardApplication_iOS.mm
LumberyardApplicationDelegate_iOS.mm
O3DEApplication_iOS.mm
O3DEApplicationDelegate_iOS.mm
../Common/Apple/Launcher_Apple.mm
../Common/Apple/Launcher_Apple.h
../Common/UnixLike/Launcher_UnixLike.cpp
Code/LauncherUnified/launcher_generator.cmake
+15
View File
@@ -18,6 +18,21 @@ foreach(project_name project_path IN ZIP_LISTS LY_PROJECTS_TARGET_NAME LY_PROJEC
# If the project_path is relative, it is evaluated relative to the ${LY_ROOT_FOLDER}
# Otherwise the the absolute project_path is returned with symlinks resolved
file(REAL_PATH ${project_path} project_real_path BASE_DIRECTORY ${LY_ROOT_FOLDER})
if(NOT project_name)
if(NOT EXISTS ${project_real_path}/project.json)
message(FATAL_ERROR "The specified project path of ${project_real_path} does not contain a project.json file")
else()
# Add the project_name to global LY_PROJECTS_TARGET_NAME property
file(READ "${project_real_path}/project.json" project_json)
string(JSON project_name ERROR_VARIABLE json_error GET ${project_json} "project_name")
if(json_error)
message(FATAL_ERROR "There is an error reading the \"project_name\" key from the '${project_real_path}/project.json' file: ${json_error}")
endif()
message(WARNING "The project located at path ${project_real_path} has a valid \"project name\" of '${project_name}' read from it's project.json file."
" This indicates that the ${project_real_path}/CMakeLists.txt is not properly appending the \"project name\" "
"to the LY_PROJECTS_TARGET_NAME global property. Other configuration errors might occur")
endif()
endif()
################################################################################
# Monolithic game
################################################################################
Code/Sandbox/Editor/CryEdit.cpp
+1 -1
View File
@@ -5702,7 +5702,7 @@ extern "C" int AZ_DLL_EXPORT CryEditMain(int argc, char* argv[])
int exitCode = 0;
BOOL didCryEditStart = CCryEditApp::instance()->InitInstance();
AZ_Error("Editor", didCryEditStart, "CryEditor did not initialize correctly, and will close."
AZ_Error("Editor", didCryEditStart, "O3DE Editor did not initialize correctly, and will close."
"\nThis could be because of incorrectly configured components, or missing required gems."
"\nSee other errors for more details.");
Code/Sandbox/Editor/CryEditDoc.cpp
+1 -1
View File
@@ -1931,7 +1931,7 @@ void CCryEditDoc::Fetch(const QString& holdName, const QString& relativeHoldPath
if (!LoadXmlArchiveArray(arrXmlAr, holdFilename, holdPath))
{
QMessageBox::critical(QApplication::activeWindow(), "Error", "The temporary 'Hold' level failed to load successfully. Your level might be corrupted, you should restart the Editor.", QMessageBox::Ok);
AZ_Error("CryEditDoc", false, "Fetch failed to load the Xml Archive");
AZ_Error("EditDoc", false, "Fetch failed to load the Xml Archive");
return;
}
Code/Sandbox/Editor/LegacyViewportCameraController.cpp
-1
View File
@@ -342,7 +342,6 @@ bool LegacyViewportCameraControllerInstance::HandleInputChannelEvent(const AzFra
m_inRotateMode = true;
}
shouldConsumeEvent = true;
shouldCaptureCursor = true;
}
else if (state == InputChannel::State::Ended)
Code/Sandbox/Editor/ViewManager.cpp
+1 -1
View File
@@ -37,7 +37,7 @@
#include <AzFramework/API/AtomActiveInterface.h>
#include <AzCore/Console/IConsole.h>
AZ_CVAR(bool, ed_useAtomNativeViewport, false, nullptr, AZ::ConsoleFunctorFlags::Null, "Use the new Atom-native Editor viewport (experimental, not yet stable");
AZ_CVAR(bool, ed_useAtomNativeViewport, true, nullptr, AZ::ConsoleFunctorFlags::Null, "Use the new Atom-native Editor viewport (experimental, not yet stable");
bool CViewManager::IsMultiViewportEnabled()
{
Code/Sandbox/Editor/ViewportManipulatorController.cpp
+2 -2
View File
@@ -21,8 +21,8 @@
#include <QApplication>
static const auto ManipulatorPriority = AzFramework::ViewportControllerPriority::Highest;
static const auto InteractionPriority = AzFramework::ViewportControllerPriority::High;
static const auto ManipulatorPriority = AzFramework::ViewportControllerPriority::High;
static const auto InteractionPriority = AzFramework::ViewportControllerPriority::Low;
namespace SandboxEditor
{
Code/Tools/SceneAPI/SceneCore/DllMain.cpp
+2
View File
@@ -42,6 +42,7 @@
#include <SceneAPI/SceneCore/DataTypes/Rules/IMeshAdvancedRule.h>
#include <SceneAPI/SceneCore/DataTypes/Rules/ILodRule.h>
#include <SceneAPI/SceneCore/DataTypes/Rules/ISkeletonProxyRule.h>
#include <SceneAPI/SceneCore/DataTypes/Rules/IScriptProcessorRule.h>
#include <SceneAPI/SceneCore/DataTypes/GraphData/IAnimationData.h>
#include <SceneAPI/SceneCore/DataTypes/GraphData/IBlendShapeData.h>
#include <SceneAPI/SceneCore/DataTypes/GraphData/IBoneData.h>
@@ -168,6 +169,7 @@ namespace AZ
context->Class<AZ::SceneAPI::DataTypes::IMeshAdvancedRule, AZ::SceneAPI::DataTypes::IRule>()->Version(1);
context->Class<AZ::SceneAPI::DataTypes::ILodRule, AZ::SceneAPI::DataTypes::IRule>()->Version(1);
context->Class<AZ::SceneAPI::DataTypes::ISkeletonProxyRule, AZ::SceneAPI::DataTypes::IRule>()->Version(1);
context->Class<AZ::SceneAPI::DataTypes::IScriptProcessorRule, AZ::SceneAPI::DataTypes::IRule>()->Version(1);
// Register graph data interfaces
context->Class<AZ::SceneAPI::DataTypes::IAnimationData, AZ::SceneAPI::DataTypes::IGraphObject>()->Version(1);
context->Class<AZ::SceneAPI::DataTypes::IBlendShapeData, AZ::SceneAPI::DataTypes::IGraphObject>()->Version(1);
Code/Tools/SceneAPI/SceneData/Behaviors/ScriptProcessorRuleBehavior.h
+5 -4
View File
@@ -12,6 +12,7 @@
#pragma once
#include <SceneAPI/SceneData/SceneDataConfiguration.h>
#include <AzCore/std/string/string.h>
#include <SceneAPI/SceneCore/Components/BehaviorComponent.h>
#include <SceneAPI/SceneCore/Events/AssetImportRequest.h>
@@ -27,7 +28,7 @@ namespace AZ
{
namespace Behaviors
{
class ScriptProcessorRuleBehavior
class SCENE_DATA_CLASS ScriptProcessorRuleBehavior
: public SceneCore::BehaviorComponent
, public Events::AssetImportRequestBus::Handler
{
@@ -36,12 +37,12 @@ namespace AZ
~ScriptProcessorRuleBehavior() override = default;
void Activate() override;
void Deactivate() override;
SCENE_DATA_API void Activate() override;
SCENE_DATA_API void Deactivate() override;
static void Reflect(ReflectContext* context);
// AssetImportRequestBus::Handler
Events::ProcessingResult UpdateManifest(
SCENE_DATA_API Events::ProcessingResult UpdateManifest(
Containers::Scene& scene,
ManifestAction action,
RequestingApplication requester) override;
Code/Tools/SceneAPI/SceneData/ManifestMetaInfoHandler.cpp
-2
View File
@@ -26,7 +26,6 @@
#include <SceneAPI/SceneData/Rules/LodRule.h>
#include <SceneAPI/SceneData/Rules/MaterialRule.h>
#include <SceneAPI/SceneData/Rules/StaticMeshAdvancedRule.h>
#include <SceneAPI/SceneData/Rules/ScriptProcessorRule.h>
#include <SceneAPI/SceneData/Rules/SkeletonProxyRule.h>
#include <SceneAPI/SceneData/Rules/SkinMeshAdvancedRule.h>
#include <SceneAPI/SceneData/Rules/SkinRule.h>
@@ -55,7 +54,6 @@ namespace AZ
{
AZ_TraceContext("Object Type", target.RTTI_GetTypeName());
modifiers.push_back(SceneData::CommentRule::TYPEINFO_Uuid());
modifiers.push_back(SceneData::ScriptProcessorRule::TYPEINFO_Uuid());
if (target.RTTI_IsTypeOf(DataTypes::IMeshGroup::TYPEINFO_Uuid()))
{
Code/Tools/SceneAPI/SceneData/Tests/SceneManifest/SceneManifestRuleTests.cpp
+54 -2
View File
@@ -13,19 +13,23 @@
#include <AzTest/AzTest.h>
#include <SceneAPI/SceneCore/Containers/SceneManifest.h>
#include <SceneAPI/SceneCore/Containers/Scene.h>
#include <SceneAPI/SceneCore/DataTypes/Rules/IScriptProcessorRule.h>
#include <SceneAPI/SceneData/ReflectionRegistrar.h>
#include <SceneAPI/SceneData/Rules/CoordinateSystemRule.h>
#include <SceneAPI/SceneData/Behaviors/ScriptProcessorRuleBehavior.h>
#include <AzCore/Math/Quaternion.h>
#include <AzCore/Name/NameDictionary.h>
#include <AzCore/RTTI/BehaviorContext.h>
#include <AzCore/RTTI/ReflectionManager.h>
#include <AzCore/Serialization/Json/RegistrationContext.h>
#include <AzCore/Serialization/Json/JsonSystemComponent.h>
#include <AzCore/Serialization/Json/RegistrationContext.h>
#include <AzCore/std/smart_ptr/make_shared.h>
#include <AzCore/std/smart_ptr/shared_ptr.h>
#include <AzCore/UnitTest/Mocks/MockSettingsRegistry.h>
#include <AzCore/UnitTest/TestTypes.h>
#include <AzFramework/FileFunc/FileFunc.h>
#include <AzCore/Math/Quaternion.h>
namespace AZ
{
@@ -94,6 +98,19 @@ namespace AZ
m_jsonSystemComponent = AZStd::make_unique<JsonSystemComponent>();
m_jsonSystemComponent->Reflect(m_jsonRegistrationContext.get());
m_data.reset(new DataMembers);
using FixedValueString = AZ::SettingsRegistryInterface::FixedValueString;
ON_CALL(m_data->m_settings, Get(::testing::Matcher<FixedValueString&>(::testing::_), testing::_))
.WillByDefault([](FixedValueString& value, AZStd::string_view) -> bool
{
value = "mock_path";
return true;
});
AZ::SettingsRegistry::Register(&m_data->m_settings);
}
void TearDown() override
@@ -106,9 +123,19 @@ namespace AZ
m_jsonRegistrationContext.reset();
m_jsonSystemComponent.reset();
AZ::SettingsRegistry::Unregister(&m_data->m_settings);
m_data.reset();
AZ::NameDictionary::Destroy();
UnitTest::AllocatorsFixture::TearDown();
}
struct DataMembers
{
AZ::NiceSettingsRegistrySimpleMock m_settings;
};
AZStd::unique_ptr<DataMembers> m_data;
};
TEST_F(SceneManifest_JSON, LoadFromString_BlankManifest_HasDefaultParts)
@@ -223,5 +250,30 @@ namespace AZ
EXPECT_THAT(jsonText.c_str(), ::testing::HasSubstr(R"(3.0)"));
EXPECT_THAT(jsonText.c_str(), ::testing::HasSubstr(R"("scale": 10.0)"));
}
TEST_F(SceneManifest_JSON, ScriptProcessorRule_LoadWithEmptyScriptFilename_ReturnsEarly)
{
using namespace SceneAPI::Containers;
using namespace SceneAPI::Events;
constexpr const char* jsonManifest = { R"JSON(
{
"values": [
{
"$type": "ScriptProcessorRule",
"scriptFilename": ""
}
]
})JSON" };
auto scene = AZ::SceneAPI::Containers::Scene("mock");
auto result = scene.GetManifest().LoadFromString(jsonManifest, m_serializeContext.get(), m_jsonRegistrationContext.get());
EXPECT_TRUE(result.IsSuccess());
EXPECT_FALSE(scene.GetManifest().IsEmpty());
auto scriptProcessorRuleBehavior = AZ::SceneAPI::Behaviors::ScriptProcessorRuleBehavior();
auto update = scriptProcessorRuleBehavior.UpdateManifest(scene, AssetImportRequest::Update, AssetImportRequest::Generic);
EXPECT_EQ(update, ProcessingResult::Ignored);
}
}
}
Gems/Atom/Asset/ImageProcessingAtom/Code/Include/Atom/ImageProcessing/ImageProcessingBus.h
+1
View File
@@ -25,6 +25,7 @@ namespace ImageProcessingAtom
// EBusTraits overrides
static const AZ::EBusHandlerPolicy HandlerPolicy = AZ::EBusHandlerPolicy::Single;
static const AZ::EBusAddressPolicy AddressPolicy = AZ::EBusAddressPolicy::Single;
typedef AZStd::recursive_mutex MutexType;
//////////////////////////////////////////////////////////////////////////
// Loads an image from a source file path
Gems/Atom/Asset/Shader/Code/Source/Editor/AzslShaderBuilderSystemComponent.cpp
+4 -3
View File
@@ -102,7 +102,8 @@ namespace AZ
// Register Shader Resource Group Layout Builder
AssetBuilderSDK::AssetBuilderDesc srgLayoutBuilderDescriptor;
srgLayoutBuilderDescriptor.m_name = "Shader Resource Group Layout Builder";
srgLayoutBuilderDescriptor.m_version = 52; // ATOM-14780
srgLayoutBuilderDescriptor.m_version = 53; // ATOM-15196
srgLayoutBuilderDescriptor.m_patterns.push_back(AssetBuilderSDK::AssetBuilderPattern("*.azsl", AssetBuilderSDK::AssetBuilderPattern::PatternType::Wildcard));
srgLayoutBuilderDescriptor.m_patterns.push_back(AssetBuilderSDK::AssetBuilderPattern("*.azsli", AssetBuilderSDK::AssetBuilderPattern::PatternType::Wildcard));
srgLayoutBuilderDescriptor.m_patterns.push_back(AssetBuilderSDK::AssetBuilderPattern(AZStd::string::format("*.%s", SrgLayoutBuilder::MergedPartialSrgsExtension), AssetBuilderSDK::AssetBuilderPattern::PatternType::Wildcard));
@@ -117,7 +118,7 @@ namespace AZ
// Register Shader Asset Builder
AssetBuilderSDK::AssetBuilderDesc shaderAssetBuilderDescriptor;
shaderAssetBuilderDescriptor.m_name = "Shader Asset Builder";
shaderAssetBuilderDescriptor.m_version = 96; // SPEC-6065
shaderAssetBuilderDescriptor.m_version = 97; // ATOM-15196
// .shader file changes trigger rebuilds
shaderAssetBuilderDescriptor.m_patterns.push_back(AssetBuilderSDK::AssetBuilderPattern( AZStd::string::format("*.%s", RPI::ShaderSourceData::Extension), AssetBuilderSDK::AssetBuilderPattern::PatternType::Wildcard));
shaderAssetBuilderDescriptor.m_busId = azrtti_typeid<ShaderAssetBuilder>();
@@ -132,7 +133,7 @@ namespace AZ
shaderVariantAssetBuilderDescriptor.m_name = "Shader Variant Asset Builder";
// Both "Shader Variant Asset Builder" and "Shader Asset Builder" produce ShaderVariantAsset products. If you update
// ShaderVariantAsset you will need to update BOTH version numbers, not just "Shader Variant Asset Builder".
shaderVariantAssetBuilderDescriptor.m_version = 17; // SPEC-6065
shaderVariantAssetBuilderDescriptor.m_version = 18; // ATOM-15196
shaderVariantAssetBuilderDescriptor.m_patterns.push_back(AssetBuilderSDK::AssetBuilderPattern(AZStd::string::format("*.%s", RPI::ShaderVariantListSourceData::Extension), AssetBuilderSDK::AssetBuilderPattern::PatternType::Wildcard));
shaderVariantAssetBuilderDescriptor.m_busId = azrtti_typeid<ShaderVariantAssetBuilder>();
shaderVariantAssetBuilderDescriptor.m_createJobFunction = AZStd::bind(&ShaderVariantAssetBuilder::CreateJobs, &m_shaderVariantAssetBuilder, AZStd::placeholders::_1, AZStd::placeholders::_2);
Gems/Atom/Feature/Common/Code/Source/ImGui/ImGuiPass.cpp
+2 -2
View File
@@ -646,8 +646,8 @@ namespace AZ
return 0; // Nothing to draw.
}
auto vertexBuffer = RPI::DynamicDrawInterface::Get()->GetDynamicBuffer(totalVtxBufferSize);
auto indexBuffer = RPI::DynamicDrawInterface::Get()->GetDynamicBuffer(totalIdxBufferSize);
auto vertexBuffer = RPI::DynamicDrawInterface::Get()->GetDynamicBuffer(totalVtxBufferSize, RHI::Alignment::InputAssembly);
auto indexBuffer = RPI::DynamicDrawInterface::Get()->GetDynamicBuffer(totalIdxBufferSize, RHI::Alignment::InputAssembly);
if (!vertexBuffer || !indexBuffer)
{
Gems/Atom/Feature/Common/Code/Source/RayTracing/RayTracingFeatureProcessor.cpp
+1
View File
@@ -66,6 +66,7 @@ namespace AZ
// load the RayTracingSceneSrg asset
Data::Asset<RPI::ShaderResourceGroupAsset> rayTracingSceneSrgAsset =
RPI::AssetUtils::LoadAssetByProductPath<RPI::ShaderResourceGroupAsset>("shaderlib/raytracingscenesrg_raytracingscenesrg.azsrg", RPI::AssetUtils::TraceLevel::Error);
AZ_Assert(rayTracingSceneSrgAsset.IsReady(), "Failed to load RayTracingSceneSrg asset");
m_rayTracingSceneSrg = RPI::ShaderResourceGroup::Create(rayTracingSceneSrgAsset);
}
Gems/Atom/RHI/Code/Include/Atom/RHI.Reflect/BufferDescriptor.h
+15 -11
View File
@@ -30,38 +30,42 @@ namespace AZ
{
None = 0,
/// Supports input assembly access through a IndexBufferView or StreamBufferView.
/// Supports input assembly access through a IndexBufferView or StreamBufferView. This flag is for buffers that are not updated often
InputAssembly = AZ_BIT(0),
/// Supports input assembly access through a IndexBufferView or StreamBufferView. This flag is for buffers that are updated frequently
DynamicInputAssembly = AZ_BIT(1),
/// Supports constant access through a ShaderResourceGroup.
Constant = AZ_BIT(1),
Constant = AZ_BIT(2),
/// Supports read access through a ShaderResourceGroup.
ShaderRead = AZ_BIT(2),
ShaderRead = AZ_BIT(3),
/// Supports write access through ShaderResourceGroup.
ShaderWrite = AZ_BIT(3),
ShaderWrite = AZ_BIT(4),
/// Supports read-write access through a ShaderResourceGroup.
ShaderReadWrite = ShaderRead | ShaderWrite,
/// Supports read access for GPU copy operations.
CopyRead = AZ_BIT(4),
CopyRead = AZ_BIT(5),
/// Supports write access for GPU copy operations.
CopyWrite = AZ_BIT(5),
CopyWrite = AZ_BIT(6),
/// Supports predication access for conditional rendering.
Predication = AZ_BIT(6),
Predication = AZ_BIT(7),
/// Supports indirect buffer access for indirect draw/dispatch.
Indirect = AZ_BIT(7),
Indirect = AZ_BIT(8),
/// Supports ray tracing acceleration structure usage.
RayTracingAccelerationStructure = AZ_BIT(8),
RayTracingAccelerationStructure = AZ_BIT(9),
/// Supports ray tracing shader table usage.
RayTracingShaderTable = AZ_BIT(9)
RayTracingShaderTable = AZ_BIT(10)
};
AZ_DEFINE_ENUM_BITWISE_OPERATORS(AZ::RHI::BufferBindFlags);
Gems/Atom/RHI/Code/Source/RHI.Reflect/ReflectSystemComponent.cpp
+2 -1
View File
@@ -54,7 +54,7 @@ namespace AZ
if (SerializeContext* serializeContext = azrtti_cast<SerializeContext*>(context))
{
serializeContext->Class<ReflectSystemComponent, AZ::Component>()
->Version(2);
->Version(3);
}
ReflectNamedEnums(context);
@@ -266,6 +266,7 @@ namespace AZ
serializeContext->Enum<BufferBindFlags>()
->Value("None", BufferBindFlags::None)
->Value("InputAssembly", BufferBindFlags::InputAssembly)
->Value("DynamicInputAssembly", BufferBindFlags::DynamicInputAssembly)
->Value("Constant", BufferBindFlags::Constant)
->Value("CopyRead", BufferBindFlags::CopyRead)
->Value("CopyWrite", BufferBindFlags::CopyWrite)
Gems/Atom/RHI/DX12/Code/Source/RHI/BufferMemoryAllocator.cpp
+1 -1
View File
@@ -39,7 +39,7 @@ namespace AZ
// needs to be a multiple of elementsize as well as divisible by DX12::Alignment types.
m_usePageAllocator = false;
if (!RHI::CheckBitsAny(descriptor.m_bindFlags, RHI::BufferBindFlags::ShaderWrite | RHI::BufferBindFlags::CopyWrite | RHI::BufferBindFlags::InputAssembly))
if (!RHI::CheckBitsAny(descriptor.m_bindFlags, RHI::BufferBindFlags::ShaderWrite | RHI::BufferBindFlags::CopyWrite | RHI::BufferBindFlags::InputAssembly | RHI::BufferBindFlags::DynamicInputAssembly))
{
m_usePageAllocator = true;
Gems/Atom/RHI/DX12/Code/Source/RHI/BufferPool.cpp
+1 -1
View File
@@ -39,7 +39,7 @@ namespace AZ
{
m_device = &device;
if (RHI::CheckBitsAll(descriptor.m_bindFlags, RHI::BufferBindFlags::InputAssembly))
if(RHI::CheckBitsAny(descriptor.m_bindFlags, RHI::BufferBindFlags::InputAssembly | RHI::BufferBindFlags::DynamicInputAssembly))
{
m_readOnlyState |= D3D12_RESOURCE_STATE_VERTEX_AND_CONSTANT_BUFFER | D3D12_RESOURCE_STATE_INDEX_BUFFER;
}
Gems/Atom/RHI/Metal/Code/Source/RHI.Builders/ShaderPlatformInterface.cpp
+12 -3
View File
@@ -670,7 +670,13 @@ namespace AZ
}
else
{
result &= AddExistingResourceEntry("texture", resourceStartPos, regId, argBufferStr);
bool isAdditionSuccessfull = AddExistingResourceEntry("texture", resourceStartPos, regId, argBufferStr);
if(!isAdditionSuccessfull)
{
//In metal depth textures use keyword depth2d/depth2d_array/depthcube/depthcube_array/depth2d_ms/depth2d_ms_array
isAdditionSuccessfull |= AddExistingResourceEntry("depth", resourceStartPos, regId, argBufferStr);
}
result &= isAdditionSuccessfull;
}
}
return result;
@@ -827,10 +833,13 @@ namespace AZ
AZStd::string& argBufferStr) const
{
size_t prevEndOfLine = argBufferStr.rfind("\n", resourceStartPos);
size_t nextEndOfLine = argBufferStr.find("\n", resourceStartPos);
size_t startOfEntryPos = argBufferStr.find(resourceStr, prevEndOfLine);
if(startOfEntryPos == AZStd::string::npos)
//Check to see if a valid entry is found.
if(startOfEntryPos == AZStd::string::npos || startOfEntryPos > nextEndOfLine)
{
AZ_Error(MetalShaderPlatformName, false, "Entry-> %s not found within Descriptor set %s", resourceStr, argBufferStr.c_str());
AZ_Error(MetalShaderPlatformName, startOfEntryPos != AZStd::string::npos, "Entry-> %s not found within Descriptor set %s", resourceStr, argBufferStr.c_str());
return false;
}
else
Gems/Atom/RHI/Metal/Code/Source/RHI/AsyncUploadQueue.cpp
+1 -1
View File
@@ -295,7 +295,7 @@ namespace AZ
const RHI::Size sourceSize = RHI::Size(subresourceLayout.m_size.m_width, heightToCopy, 1);
const RHI::Origin sourceOrigin = RHI::Origin(0, destHeight, depth);
CopyBufferToImage(framePacket, image, stagingRowPitch, stagingSlicePitch,
CopyBufferToImage(framePacket, image, stagingRowPitch, bytesCopied,
curMip, arraySlice, sourceSize, sourceOrigin);
framePacket->m_dataOffset += stagingSize;
Gems/Atom/RHI/Metal/Code/Source/RHI/Conversions.cpp
+6
View File
@@ -210,6 +210,12 @@ namespace AZ
{
return GetCPUGPUMemoryMode();
}
//This flag is used for IA buffers that is updated frequently and hence shared mmory is the best fit
if (RHI::CheckBitsAll(descriptor.m_bindFlags, RHI::BufferBindFlags::DynamicInputAssembly))
{
return MTLStorageModeShared;
}
return GetCPUGPUMemoryMode();
}
Gems/Atom/RHI/Vulkan/Code/Source/RHI/BufferPool.cpp
+1
View File
@@ -107,6 +107,7 @@ namespace AZ
bool forceUnique = RHI::CheckBitsAny(
bufferDescriptor.m_bindFlags,
RHI::BufferBindFlags::InputAssembly |
RHI::BufferBindFlags::DynamicInputAssembly |
RHI::BufferBindFlags::RayTracingAccelerationStructure |
RHI::BufferBindFlags::RayTracingShaderTable);
Gems/Atom/RHI/Vulkan/Code/Source/RHI/Conversion.cpp
+3 -3
View File
@@ -685,7 +685,7 @@ namespace AZ
using BindFlags = RHI::BufferBindFlags;
VkBufferUsageFlags usageFlags{ 0 };
if (RHI::CheckBitsAny(bindFlags, BindFlags::InputAssembly))
if (RHI::CheckBitsAny(bindFlags, BindFlags::InputAssembly | BindFlags::DynamicInputAssembly))
{
usageFlags |=
VK_BUFFER_USAGE_INDEX_BUFFER_BIT |
@@ -932,7 +932,7 @@ namespace AZ
VkPipelineStageFlags GetResourcePipelineStateFlags(const RHI::BufferBindFlags& bindFlags)
{
VkPipelineStageFlags stagesFlags = {};
if (RHI::CheckBitsAny(bindFlags, RHI::BufferBindFlags::InputAssembly))
if (RHI::CheckBitsAny(bindFlags, RHI::BufferBindFlags::InputAssembly | RHI::BufferBindFlags::DynamicInputAssembly))
{
stagesFlags |= VK_PIPELINE_STAGE_DRAW_INDIRECT_BIT | VK_PIPELINE_STAGE_VERTEX_INPUT_BIT;
}
@@ -1042,7 +1042,7 @@ namespace AZ
VkAccessFlags GetResourceAccessFlags(const RHI::BufferBindFlags& bindFlags)
{
VkAccessFlags accessFlags = {};
if (RHI::CheckBitsAny(bindFlags, RHI::BufferBindFlags::InputAssembly))
if (RHI::CheckBitsAny(bindFlags, RHI::BufferBindFlags::InputAssembly | RHI::BufferBindFlags::DynamicInputAssembly))
{
accessFlags |= VK_ACCESS_INDIRECT_COMMAND_READ_BIT | VK_ACCESS_VERTEX_ATTRIBUTE_READ_BIT | VK_ACCESS_INDEX_READ_BIT;
}
Gems/Atom/RPI/Assets/ResourcePools/DefaultDynInputBufferPool.resourcepool
+2 -2
View File
@@ -8,6 +8,6 @@
"BudgetInBytes": 25165824,
"BufferPoolHeapMemoryLevel": "Host",
"BufferPoolhostMemoryAccess": "Write",
"BufferPoolBindFlags": "InputAssembly"
"BufferPoolBindFlags": "DynamicInputAssembly"
}
}
}
Gems/Atom/RPI/Code/Include/Atom/RPI.Public/DynamicDraw/DynamicBuffer.h
+1 -1
View File
@@ -36,7 +36,7 @@ namespace AZ
//! buffer->Write(data, size);
//! // Use the buffer view for DrawItem or etc.
//! }
//! Note: DynamicBuffer should only be used for InputAssembly buffer or Constant buffer (not supported yet).
//! Note: DynamicBuffer should only be used for DynamicInputAssembly buffer or Constant buffer (not supported yet).
class DynamicBuffer
: public AZStd::intrusive_base
{
Gems/Atom/RPI/Code/Source/RPI.Public/Buffer/Buffer.cpp
+4 -3
View File
@@ -74,8 +74,9 @@ namespace AZ
const RHI::BufferView* Buffer::GetBufferView() const
{
if (m_rhiBuffer->GetDescriptor().m_bindFlags == RHI::BufferBindFlags::InputAssembly)
if(RHI::CheckBitsAny(m_rhiBuffer->GetDescriptor().m_bindFlags, RHI::BufferBindFlags::InputAssembly | RHI::BufferBindFlags::DynamicInputAssembly))
{
AZ_Assert(false, "Input assembly buffer doesn't need a regular buffer view, it requires a stream or index buffer view.");
return nullptr;
}
@@ -203,11 +204,11 @@ namespace AZ
void Buffer::InitBufferView()
{
// Skip buffer view creation for input assembly buffers
if (m_rhiBuffer->GetDescriptor().m_bindFlags == RHI::BufferBindFlags::InputAssembly)
if(RHI::CheckBitsAny(m_rhiBuffer->GetDescriptor().m_bindFlags, RHI::BufferBindFlags::InputAssembly | RHI::BufferBindFlags::DynamicInputAssembly))
{
return;
}
m_bufferView = m_rhiBuffer->GetBufferView(m_bufferViewDescriptor);
if(!m_bufferView.get())
Gems/Atom/RPI/Code/Source/RPI.Public/Buffer/BufferSystem.cpp
+1 -1
View File
@@ -105,7 +105,7 @@ namespace AZ
bufferPoolDesc.m_hostMemoryAccess = RHI::HostMemoryAccess::Write;
break;
case CommonBufferPoolType::DynamicInputAssembly:
bufferPoolDesc.m_bindFlags = RHI::BufferBindFlags::InputAssembly;
bufferPoolDesc.m_bindFlags = RHI::BufferBindFlags::DynamicInputAssembly;
bufferPoolDesc.m_heapMemoryLevel = RHI::HeapMemoryLevel::Host;
bufferPoolDesc.m_hostMemoryAccess = RHI::HostMemoryAccess::Write;
break;
Gems/Atom/RPI/Code/Source/RPI.Public/DynamicDraw/DynamicBufferAllocator.cpp
+1
View File
@@ -63,6 +63,7 @@ namespace AZ
// [GFX TODO][ATOM-13182] Add unit tests for DynamicBufferAllocator's Allocate function
RHI::Ptr<DynamicBuffer> DynamicBufferAllocator::Allocate(uint32_t size, [[maybe_unused]]uint32_t alignment)
{
size = RHI::AlignUp(size, alignment);
uint32_t allocatePosition = 0;
//m_ringBufferStartAddress can be null for Null back end
Gems/Atom/RPI/Code/Source/RPI.Public/Pass/PassAttachment.cpp
+1 -1
View File
@@ -174,7 +174,7 @@ namespace AZ
}
else if (GetAttachmentType() == RHI::AttachmentType::Buffer)
{
bool isInputAssembly = RHI::CheckBitsAny(m_descriptor.m_buffer.m_bindFlags, RHI::BufferBindFlags::InputAssembly);
bool isInputAssembly = RHI::CheckBitsAny(m_descriptor.m_buffer.m_bindFlags, RHI::BufferBindFlags::InputAssembly | RHI::BufferBindFlags::DynamicInputAssembly);
bool isConstant = RHI::CheckBitsAny(m_descriptor.m_buffer.m_bindFlags, RHI::BufferBindFlags::Constant);
// Since InputAssembly and Constant cannot be inferred they are set manually. If those flags are set we don't want to add inferred flags on top as it may have a performance penalty
Gems/Atom/RPI/Code/Source/RPI.Public/Shader/ShaderVariantAsyncLoader.cpp
+1
View File
@@ -173,6 +173,7 @@ namespace AZ
m_serviceThread.join();
Data::AssetBus::MultiHandler::BusDisconnect();
m_newShaderVariantPendingRequests.clear();
m_shaderVariantTreePendingRequests.clear();
m_shaderVariantPendingRequests.clear();
m_shaderVariantData.clear();
Gems/AtomLyIntegration/CommonFeatures/Code/Source/Thumbnails/Rendering/ThumbnailRendererSteps/InitializeStep.cpp
+2 -1
View File
@@ -54,8 +54,9 @@ namespace AZ
RPI::SceneDescriptor sceneDesc;
sceneDesc.m_featureProcessorNames.push_back("AZ::Render::TransformServiceFeatureProcessor");
sceneDesc.m_featureProcessorNames.push_back("AZ::Render::MeshFeatureProcessor");
sceneDesc.m_featureProcessorNames.push_back("AZ::Render::SimplePointLightFeatureProcessor");
sceneDesc.m_featureProcessorNames.push_back("AZ::Render::SimpleSpotLightFeatureProcessor");
sceneDesc.m_featureProcessorNames.push_back("AZ::Render::PointLightFeatureProcessor");
sceneDesc.m_featureProcessorNames.push_back("AZ::Render::SpotLightFeatureProcessor");
// There is currently a bug where having multiple DirectionalLightFeatureProcessors active can result in shadow flickering [ATOM-13568]
// as well as continually rebuilding MeshDrawPackets [ATOM-13633]. Lets just disable the directional light FP for now.
// Possibly re-enable with [GFX TODO][ATOM-13639]
Gems/Blast/Code/Source/Editor/EditorBlastMeshDataComponent.cpp
+1 -1
View File
@@ -179,7 +179,7 @@ namespace Blast
void EditorBlastMeshDataComponent::RegisterModel()
{
if (m_meshFeatureProcessor && m_meshAssets[0].GetId().IsValid())
if (m_meshFeatureProcessor && !m_meshAssets.empty() && m_meshAssets[0].GetId().IsValid())
{
AZ::Render::MaterialAssignmentMap materials;
AZ::Render::MaterialComponentRequestBus::EventResult(
Gems/Camera/Code/Source/CameraComponentController.cpp
+4 -3
View File
@@ -98,15 +98,16 @@ namespace Camera
AZ_Assert(m_atomCamera, "Attempted to activate Atom camera before component activation");
const AZ::Name contextName = atomViewportRequests->GetDefaultViewportContextName();
atomViewportRequests->PushView(contextName, m_atomCamera);
AZ::RPI::ViewportContextNotificationBus::Handler::BusConnect(contextName);
// Ensure the Atom camera is updated with our current transform state
AZ::Transform localTransform;
AZ::TransformBus::EventResult(localTransform, m_entityId, &AZ::TransformBus::Events::GetLocalTM);
AZ::Transform worldTransform;
AZ::TransformBus::EventResult(worldTransform, m_entityId, &AZ::TransformBus::Events::GetWorldTM);
OnTransformChanged(localTransform, worldTransform);
// Push the Atom camera after we make sure we're up-to-date with our component's transform to ensure the viewport reads the correct state
atomViewportRequests->PushView(contextName, m_atomCamera);
AZ::RPI::ViewportContextNotificationBus::Handler::BusConnect(contextName);
UpdateCamera();
}
}
Gems/SurfaceData/Code/Source/Editor/EditorSurfaceDataSystemComponent.cpp
+19 -7
View File
@@ -145,24 +145,36 @@ namespace SurfaceData
void EditorSurfaceDataSystemComponent::OnCatalogLoaded(const char* /*catalogFile*/)
{
//automatically register all surface tag list assets
//automatically register all existing surface tag list assets at Editor startup
// First run through all the assets and trigger loads on them.
AZStd::vector<AZ::Data::AssetId> surfaceTagAssetIds;
// First run through all the assets and gather up the asset IDs for all surface tag list assets
AZ::Data::AssetCatalogRequestBus::Broadcast(&AZ::Data::AssetCatalogRequestBus::Events::EnumerateAssets,
nullptr,
[this](const AZ::Data::AssetId assetId, const AZ::Data::AssetInfo& assetInfo) {
[&surfaceTagAssetIds](const AZ::Data::AssetId assetId, const AZ::Data::AssetInfo& assetInfo) {
const auto assetType = azrtti_typeid<EditorSurfaceTagListAsset>();
if (assetInfo.m_assetType == assetType)
{
m_surfaceTagNameAssets[assetId] = AZ::Data::AssetManager::Instance().GetAsset(assetId, assetType, AZ::Data::AssetLoadBehavior::Default);
surfaceTagAssetIds.emplace_back(assetId);
}
},
nullptr);
// After all the loads are triggered, block to make sure they've all completed.
for (auto& asset : m_surfaceTagNameAssets)
// Next, trigger all the loads. This is done outside of EnumerateAssets to ensure that we don't have any deadlocks caused by
// lock inversion. If this thread locks AssetCatalogRequestBus mutex with EnumerateAssets, then locks m_assetMutex in
// AssetManager::FindOrCreateAsset, it's possible for those locks to get locked in reverse on a loading thread, causing a deadlock.
for (auto& assetId : surfaceTagAssetIds)
{
asset.second.BlockUntilLoadComplete();
m_surfaceTagNameAssets[assetId] = AZ::Data::AssetManager::Instance().GetAsset(
assetId, azrtti_typeid<EditorSurfaceTagListAsset>(), AZ::Data::AssetLoadBehavior::Default);
// If any assets are still loading (which they likely will be), listen for the OnAssetReady event and refresh the Editor
// UI as each one finishes loading.
if (!m_surfaceTagNameAssets[assetId].IsReady())
{
AZ::Data::AssetBus::MultiHandler::BusConnect(assetId);
}
}
}
Templates/DefaultProject/Template/CMakeLists.txt
+1 -1
View File
@@ -38,7 +38,7 @@ else()
file(READ "${CMAKE_CURRENT_LIST_DIR}/project.json" project_json)
string(JSON project_target_name ERROR_VARIABLE json_error GET ${project_json} "project_name")
if(${json_error})
if(json_error)
message(FATAL_ERROR "Unable to read key 'project_name' from 'project.json'")
endif()
Templates/DefaultProject/Template/ShaderLib/raytracingscenesrg.srgi
+28
View File
@@ -0,0 +1,28 @@
/*
* All or portions of this file Copyright (c) Amazon.com, Inc. or its affiliates or
* its licensors.
*
* For complete copyright and license terms please see the LICENSE at the root of this
* distribution (the "License"). All use of this software is governed by the License,
* or, if provided, by the license below or the license accompanying this file. Do not
* remove or modify any license notices. This file is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
*
*/
#pragma once
// Please read README.md for an explanation on why scenesrg.srgi and viewsrg.srgi are
// located in this folder (And how you can optionally customize your own scenesrg.srgi
// and viewsrg.srgi in your game project).
#include <Atom/Features/SrgSemantics.azsli>
partial ShaderResourceGroup RayTracingSceneSrg : SRG_RayTracingScene
{
/* Intentionally Empty. Helps define the SrgSemantic for RayTracingSceneSrg once.*/
};
#define AZ_COLLECTING_PARTIAL_SRGS
#include <Atom/Feature/Common/Assets/ShaderResourceGroups/RayTracingSceneSrgAll.azsli>
#undef AZ_COLLECTING_PARTIAL_SRGS
Tools/build/JenkinsScripts/build/PackageEnv.py
-193
View File
@@ -1,193 +0,0 @@
#
# All or portions of this file Copyright (c) Amazon.com, Inc. or its affiliates or
# its licensors.
#
# For complete copyright and license terms please see the LICENSE at the root of this
# distribution (the "License"). All use of this software is governed by the License,
# or, if provided, by the license below or the license accompanying this file. Do not
# remove or modify any license notices. This file is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
#
from Params import Params
from utils.util import *
class PackageEnv(Params):
def __init__(self, target_platform, json_file):
super(PackageEnv, self).__init__()
self.__cur_dir = os.path.dirname(os.path.abspath(__file__))
with open(json_file, 'r') as source:
self.__data = json.load(source)
self.__platforms = self.__data.get('platforms')
if target_platform not in self.__platforms:
ly_build_error('Target platform {} is not supported'.format(target_platform))
self.__target_platform = target_platform
# visited_platform is used to track the platform reference chain, in order to avoid chain cycle.
visited_platform = [target_platform]
platform_env = self.__platforms.get(target_platform)
# If platform_env starts with @, it means that platform_env references another platform
while isinstance(platform_env, str) and platform_env.startswith('@'):
referenced_platform = platform_env.lstrip('@')
if referenced_platform in visited_platform:
ly_build_error('Found reference chain cycle started from {}.\nSee {}'.format(referenced_platform, json_file))
visited_platform.append(referenced_platform)
platform_env = self.__platforms.get(referenced_platform)
self.__platform_env = platform_env
self.__global_env = self.__data.get('global')
def get_target_platform(self):
return self.__target_platform
def get_global_env(self):
return self.__global_env
def get_platform_env(self):
return self.__platform_env
def __get_global_value(self, key):
key = key.upper()
value = self.__global_env.get(key)
if value is None:
ly_build_error('{} is not defined in global env'.format(key))
return value
def __get_platform_value(self, key):
key = key.upper()
value = self.__platform_env.get(key)
if value is None:
ly_build_error('{} is not defined in platform env for {}'.format(key, self.__target_platform))
return value
def __evaluate_boolean(self, v):
return str(v).lower() in ['1', 'true']
def __get_engine_root(self):
def validate_engine_root(engine_root):
if not os.path.isdir(engine_root):
return False
return os.path.exists(os.path.join(engine_root, 'engine.json'))
# Jenkins only
workspace = os.getenv('WORKSPACE')
if workspace is not None:
print('Environment variable WORKSPACE={} detected'.format(workspace))
if validate_engine_root(workspace):
print('Setting ENGINE_ROOT to {}'.format(workspace))
return workspace
engine_root = os.path.join(workspace, 'dev')
if validate_engine_root(engine_root):
print('Setting ENGINE_ROOT to {}'.format(engine_root))
return engine_root
print('Cannot locate ENGINE_ROOT with Environment variable WORKSPACE')
# End Jenkins only
engine_root = os.getenv('ENGINE_ROOT', '')
if validate_engine_root(engine_root):
return engine_root
print('Environment variable ENGINE_ROOT is not set or invalid, checking ENGINE_ROOT in env json file')
engine_root = self.__global_env.get('ENGINE_ROOT')
if validate_engine_root(engine_root):
return engine_root
# Set engine_root based on script location
engine_root = os.path.dirname(os.path.dirname(os.path.dirname(os.path.dirname(self.__cur_dir))))
print('ENGINE_ROOT from env json file is invalid, defaulting to {}'.format(engine_root))
if validate_engine_root(engine_root):
return engine_root
else:
error('Cannot Locate ENGINE_ROOT')
def __get_thirdparty_home(self):
third_party_home = os.getenv('ENV_3RDPARTY_PATH', '')
if os.path.exists(third_party_home):
print('ENV_3RDPARTY_PATH found, using {} as 3rdParty path.'.format(third_party_home))
return third_party_home
third_party_home = self.__get_global_value('THIRDPARTY_HOME')
if os.path.isdir(third_party_home):
return third_party_home
# Set engine_root based on script location
print('THIRDPARTY_HOME is not valid, looking for THIRD_PARTY_HOME')
# Finding THIRD_PARTY_HOME
cur_dir = self.__get_engine_root()
last_dir = None
while last_dir != cur_dir:
third_party_home = os.path.join(cur_dir, '3rdParty')
print('Cheking THIRDPARTY_HOME {}'.format(third_party_home))
if os.path.exists(os.path.join(third_party_home, '3rdParty.txt')):
print('Setting THIRDPARTY_HOME to {}'.format(third_party_home))
return third_party_home
last_dir = cur_dir
cur_dir = os.path.dirname(cur_dir)
error('Cannot locate THIRDPARTY_HOME')
def __get_package_name_pattern(self):
package_name_pattern = self.__get_global_value('PACKAGE_NAME_PATTERN')
if os.getenv('PACKAGE_NAME_PATTERN') is not None:
package_name_pattern = os.getenv('PACKAGE_NAME_PATTERN')
return package_name_pattern
def __get_build_number(self):
build_number = self.__get_global_value('BUILD_NUMBER')
if os.getenv('BUILD_NUMBER') is not None:
build_number = os.getenv('BUILD_NUMBER')
return build_number
def __get_p4_changelist(self):
p4_changelist = self.__get_global_value('P4_CHANGELIST')
if os.getenv('P4_CHANGELIST') is not None:
p4_changelist = os.getenv('P4_CHANGELIST')
return p4_changelist
def __get_major_version(self):
major_version = self.__get_global_value('MAJOR_VERSION')
if os.getenv('MAJOR_VERSION') is not None:
major_version = os.getenv('MAJOR_VERSION')
return major_version
def __get_minor_version(self):
minor_version = self.__get_global_value('MINOR_VERSION')
if os.getenv('MINOR_VERSION') is not None:
minor_version = os.getenv('MINOR_VERSION')
return minor_version
def __get_scrub_params(self):
return self.__get_platform_value('SCRUB_PARAMS')
def __get_validator_platforms(self):
return self.__get_platform_value('VALIDATOR_PLATFORMS')
def __get_package_targets(self):
return self.__get_platform_value('PACKAGE_TARGETS')
def __get_build_targets(self):
return self.__get_platform_value('BUILD_TARGETS')
def __get_asset_processor_path(self):
return self.__get_platform_value('ASSET_PROCESSOR_PATH')
def __get_asset_game_folders(self):
return self.__get_platform_value('ASSET_GAME_FOLDERS')
def __get_asset_platform(self):
return self.__get_platform_value('ASSET_PLATFORM')
def __get_bootstrap_cfg_game_folder(self):
return self.__get_platform_value('BOOTSTRAP_CFG_GAME_FOLDER')
def __get_run_launcher_unit_test(self):
run_launcher_unit_test = os.getenv('RUN_LAUNCHER_UNIT_TEST')
if run_launcher_unit_test is None:
run_launcher_unit_test = self.__platform_env.get('RUN_LAUNCHER_UNIT_TEST')
return self.__evaluate_boolean(run_launcher_unit_test)
def __get_skip_build(self):
skip_build = os.getenv('SKIP_BUILD')
if skip_build is None:
skip_build = self.__platform_env.get('SKIP_BUILD')
return self.__evaluate_boolean(skip_build)
Tools/build/JenkinsScripts/build/Params.py
-81
View File
@@ -1,81 +0,0 @@
#
# All or portions of this file Copyright (c) Amazon.com, Inc. or its affiliates or
# its licensors.
#
# For complete copyright and license terms please see the LICENSE at the root of this
# distribution (the "License"). All use of this software is governed by the License,
# or, if provided, by the license below or the license accompanying this file. Do not
# remove or modify any license notices. This file is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
#
from utils.util import *
class Params(object):
def __init__(self):
# Cache params
self.__params = {}
def get(self, param_name):
param_value = self.__params.get(param_name)
if param_value is not None:
return param_value
# Call __get_${param_name} function
func = getattr(self, '_{}__get_{}'.format(self.__class__.__name__, param_name.lower()), None)
if func is not None:
param_value = func()
# Replace all ${env} in value
if isinstance(param_value, str):
param_value = self.__process_string(param_name, param_value)
elif isinstance(param_value, list):
param_value = self.__process_list(param_name, param_value)
elif isinstance(param_value, dict):
param_value = self.__process_dict(param_name, param_value)
# Cache param
self.__params[param_name] = param_value
return param_value
ly_build_error('method __get_{} is not defined in class {}'.format(param_name.lower(), self.__class__.__name__))
def set(self, param_name, param_value):
self.__params[param_name] = param_value
def exists(self, param_name):
try:
self.get(param_name)
except LyBuildError:
return False
return True
def __process_string(self, param_name, param_value):
# Find all param with format ${param}
params = re.findall('\${(\w+)}', param_value)
# Avoid using the same param name in value, like 'WORKSPACE': '${WORKSPACE} some string'
if param_name in params:
ly_build_error('The use of same parameter name({}) in value is not allowed'.format(param_name))
# Replace ${param} with actual value
for param in params:
param_value = param_value.replace('${' + param + '}', self.get(param))
return param_value
def __process_list(self, param_name, param_value):
processed_list = []
for entry in param_value:
if isinstance(entry, str):
entry = self.__process_string(param_name, entry)
elif isinstance(entry, list):
entry = self.__process_list(param_name, entry)
elif isinstance(entry, dict):
entry = self.__process_dict(param_name, entry)
processed_list.append(entry)
return processed_list
def __process_dict(self, param_name, param_value):
for key in param_value:
if isinstance(param_value[key], str):
param_value[key] = self.__process_string(param_name, param_value[key])
elif isinstance(param_value[key], list):
param_value[key] = self.__process_list(param_name, param_value[key])
elif isinstance(param_value[key], dict):
param_value[key] = self.__process_dict(param_name, param_value[key])
return param_value
Tools/build/JenkinsScripts/build/cmake_package.py
-358
View File
@@ -1,358 +0,0 @@
#
# All or portions of this file Copyright (c) Amazon.com, Inc. or its affiliates or
# its licensors.
#
# For complete copyright and license terms please see the LICENSE at the root of this
# distribution (the "License"). All use of this software is governed by the License,
# or, if provided, by the license below or the license accompanying this file. Do not
# remove or modify any license notices. This file is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
#
import sys
import glob_to_regex
import zipfile
import timeit
import stat
from optparse import OptionParser
from PackageEnv import PackageEnv
from ci_build import build
from utils.util import *
from utils.lib.glob3 import glob
def package(options):
package_platform = options.package_platform
package_env = PackageEnv(package_platform, options.package_env)
engine_root = package_env.get('ENGINE_ROOT')
# Ask the validator code to tell us which files need to be removed from the package
prohibited_file_mask = get_prohibited_file_mask(package_platform, engine_root)
# Scrub files. This is destructive, but is necessary to allow the current file existance checks to work properly. Better to copy and then build, or to
# mask on sync, but this is what we have for now
# No need to run scrubbing script since all restricted platform codes are moved to dev/restricted folder
#scrub_files(package_env, prohibited_file_mask)
# validate files
validate_restricted_files(package_platform, package_env)
# Override values in bootstrap.cfg for PC package
override_bootstrap_cfg(package_env)
# Generate GameTemplates whitelist information for metrics reporting
template_whitelist_script = os.path.join(engine_root, 'Tools/build/JenkinsScripts/distribution/Metrics/GameTemplates/buildGameTemplateWhitelist.py')
if os.path.exists(template_whitelist_script):
if sys.platform == 'win32':
python = os.path.join(engine_root, 'Tools', 'Python', 'python.cmd')
else:
python = os.path.join(engine_root, 'Tools', 'Python', 'python.sh')
project_templates_folder = os.path.join(engine_root, 'ProjectTemplates')
args = [python, template_whitelist_script, '--projectTemplatesFolder', project_templates_folder]
#execute_system_call(args)
if not package_env.get('SKIP_BUILD'):
print('SKIP_BUILD is False, running CMake build...')
cmake_build(package_env)
# TODO Compile Assets
#if package_env.exists('ASSET_PROCESSOR_PATH'):
# compile_assets(package_env)
#create packages
create_packages(package_env)
def override_bootstrap_cfg(package_env):
print('Override values in bootstrap.cfg')
engine_root = package_env.get('ENGINE_ROOT')
bootstrap_path = os.path.join(engine_root, 'bootstrap.cfg')
replace_values = {'project_path':'{}'.format(package_env.get('BOOTSTRAP_CFG_GAME_FOLDER'))}
try:
with open(bootstrap_path, 'r') as bootstrap_cfg:
content = bootstrap_cfg.read()
except:
error('Cannot read file {}'.format(bootstrap_path))
content = content.split('\n')
new_content = []
for line in content:
if not line.startswith('--'):
strs = line.split('=')
if len(strs):
key = strs[0].strip(' ')
if key in replace_values:
line = '{}={}'.format(key, replace_values[key])
new_content.append(line)
try:
with open(bootstrap_path, 'w') as out:
out.write('\n'.join(new_content))
except:
error('Cannot write to file {}'.format(bootstrap_path))
print('{} updated with value {}'.format(bootstrap_path, replace_values))
def get_prohibited_file_mask(package_platform, engine_root):
sys.path.append(os.path.join(engine_root, 'Tools', 'build', 'JenkinsScripts', 'distribution', 'scrubbing'))
from validator_data_LEGAL_REVIEW_REQUIRED import get_prohibited_platforms_for_package
# The list of prohibited platforms is controlled by the validator on a per-package basis
prohibited_platforms = get_prohibited_platforms_for_package(package_platform)
prohibited_platforms.append('all')
excludes_list = []
for p in prohibited_platforms:
platform_excludes = glob_to_regex.generate_excludes_for_platform(engine_root, p)
excludes_list.extend(platform_excludes)
prohibited_file_mask = re.compile('|'.join(excludes_list), re.IGNORECASE)
return prohibited_file_mask
def scrub_files(package_env, prohibited_file_mask):
print('Perform the Code Scrubbing')
engine_root = package_env.get('ENGINE_ROOT')
success = True
for dirname, subFolders, files in os.walk(engine_root):
for filename in files:
full_path = os.path.join(dirname, filename)
if prohibited_file_mask.match(full_path):
try:
print('Deleting: {}'.format(full_path))
os.chmod(full_path, stat.S_IWRITE)
os.unlink(full_path)
except:
e = sys.exc_info()[0]
sys.stderr.write('Error: could not delete {} ... aborting.\n'.format(full_path))
sys.stderr.write('{}\n'.format(str(e)))
success = False
if not success:
sys.stderr.write('ERROR: scrub_files failed\n')
sys.exit(1)
def validate_restricted_files(package, package_env):
print('Perform the Code Scrubbing')
engine_root = package_env.get('ENGINE_ROOT')
# Run validator
success = True
validator_path = os.path.join(engine_root, 'Tools/build/JenkinsScripts/distribution/scrubbing/validator.py')
if sys.platform == 'win32':
python = os.path.join(engine_root, 'Tools', 'Python', 'python3.cmd')
else:
python = os.path.join(engine_root, 'Tools', 'Python', 'python3.sh')
args = [python, validator_path, '--package', package, engine_root]
return_code = safe_execute_system_call(args)
if return_code != 0:
success = False
if not success:
error('Restricted file validator failed.')
print('Restricted file validator completed successfully.')
def cmake_build(package_env):
build_targets = package_env.get('BUILD_TARGETS')
for build_target in build_targets:
build(build_target['BUILD_CONFIG_FILENAME'], build_target['PLATFORM'], build_target['TYPE'])
def create_packages(package_env):
package_targets = package_env.get('PACKAGE_TARGETS')
for package_target in package_targets:
print('Creating zipfile for package target {}'.format(package_target))
cur_dir = os.path.dirname(os.path.abspath(__file__))
filelist = os.path.join(cur_dir, 'package_filelists', '{}.json'.format(package_target['TYPE']))
with open(filelist, 'r') as source:
data = json.load(source)
lyengine = os.path.dirname(package_env.get('ENGINE_ROOT'))
print('Calculating filelists...')
files = {}
# We have to include 3rdParty in Mac/Console packages until LAD is available for those platforms
# Remove this when LAD is available for those platforms.
if package_target['TYPE'] in ['cmake_consoles', 'consoles']:
files.update(get_3rdparty_filelist(package_env, 'common'))
files.update(get_3rdparty_filelist(package_env, 'vc141'))
files.update(get_3rdparty_filelist(package_env, 'vc142'))
files.update(get_3rdparty_filelist(package_env, 'provo'))
elif package_target['TYPE'] in ['cmake_atom_pc']:
files.update(get_3rdparty_filelist(package_env, 'common'))
files.update(get_3rdparty_filelist(package_env, 'vc141'))
files.update(get_3rdparty_filelist(package_env, 'vc142'))
elif package_target['TYPE'] in ['cmake_all']:
if package_env.get_target_platform() == 'mac':
files.update(get_3rdparty_filelist(package_env, 'common'))
files.update(get_3rdparty_filelist(package_env, 'mac'))
elif package_env.get_target_platform() == 'consoles':
files.update(get_3rdparty_filelist(package_env, 'common'))
files.update(get_3rdparty_filelist(package_env, 'vc141'))
files.update(get_3rdparty_filelist(package_env, 'vc142'))
files.update(get_3rdparty_filelist(package_env, 'provo'))
if '@lyengine' in data:
if '@engine_root' in data['@lyengine']:
engine_root_basename = os.path.basename(package_env.get('ENGINE_ROOT'))
data['@lyengine'][engine_root_basename] = data['@lyengine']['@engine_root']
data['@lyengine'].pop('@engine_root')
files.update(filter_files(data['@lyengine'], lyengine))
if '@3rdParty' in data:
files.update(filter_files(data['@3rdParty'], package_env.get('THIRDPARTY_HOME')))
package_path = os.path.join(lyengine, package_target['PACKAGE_NAME'])
print('Creating zipfile at {}'.format(package_path))
start = timeit.default_timer()
with zipfile.ZipFile(package_path, 'w', compression=zipfile.ZIP_DEFLATED, allowZip64=True) as myzip:
for f in files:
if os.path.islink(f):
zipInfo = zipfile.ZipInfo(files[f])
zipInfo.create_system = 3
# long type of hex val of '0xA1ED0000L',
# say, symlink attr magic...
#zipInfo.external_attr = 0xA1ED0000L
zipInfo.external_attr |= 0xA0000000
myzip.writestr(zipInfo, os.readlink(f))
else:
myzip.write(f, files[f])
stop = timeit.default_timer()
total_time = int(stop - start)
print('{} is created. Total time: {} seconds.'.format(package_path, total_time))
def get_MD5(file_path):
from hashlib import md5
chunk_size = 200 * 1024
h = md5()
with open(file_path, 'rb') as f:
while True:
chunk = f.read(chunk_size)
if len(chunk):
h.update(chunk)
else:
break
return h.hexdigest()
md5_file = '{}.MD5'.format(package_path)
print('Creating MD5 file at {}'.format(md5_file))
start = timeit.default_timer()
with open(md5_file, 'w') as output:
output.write(get_MD5(package_path))
stop = timeit.default_timer()
total_time = int(stop - start)
print('{} is created. Total time: {} seconds.'.format(md5_file, total_time))
def filter_files(data, base, prefix='', support_symlinks=True):
includes = {}
excludes = set()
for key, value in data.items():
pattern = os.path.join(base, prefix, key)
if not isinstance(value, dict):
pattern = os.path.normpath(pattern)
result = glob(pattern, recursive=True)
files = [x for x in result if os.path.isfile(x) or (support_symlinks and os.path.islink(x))]
if value == "#exclude":
excludes.update(files)
elif value == "#include":
for file in files:
includes[file] = os.path.relpath(file, base)
else:
if value.startswith('#move:'):
for file in files:
file_name = os.path.relpath(file, os.path.join(base, prefix))
dst_dir = value.replace('#move:', '').strip(' ')
includes[file] = os.path.join(dst_dir, file_name)
elif value.startswith('#rename:'):
for file in files:
dst_file = value.replace('#rename:', '').strip(' ')
includes[file] = dst_file
else:
warn('Unknown directive {} for pattern {}'.format(value, pattern))
else:
includes.update(filter_files(value, base, os.path.join(prefix, key), support_symlinks))
for exclude in excludes:
try:
includes.pop(exclude)
except KeyError:
pass
return includes
def get_3rdparty_filelist(package_env, platform, support_symlinks=True):
engine_root = package_env.get('ENGINE_ROOT')
include_pattern_file = 'include_pattern_file'
if os.path.isfile(include_pattern_file):
os.remove(include_pattern_file)
exclude_pattern_file = 'exclude_pattern_file'
if os.path.isfile(exclude_pattern_file):
os.remove(exclude_pattern_file)
versions_file = 'versions_file'
if os.path.isfile(versions_file):
os.remove(versions_file)
# Generate 3rdParty version file
ly_dep_version_tool = os.path.join(engine_root, 'Tools/build/JenkinsScripts/distribution/ly_dep_version_tool.py')
setup_assistant_config = os.path.join(engine_root, 'SetupAssistantConfig.json')
if sys.platform == 'win32':
python = os.path.join(engine_root, 'Tools', 'Python', 'python.cmd')
else:
python = os.path.join(engine_root, 'Tools', 'Python', 'python.sh')
args = [python, ly_dep_version_tool, '-o', versions_file, '-s', setup_assistant_config]
execute_system_call(args)
# Generate 3rdParty include pattern and exclude pattern
generate_external_3rdparty_file_list = os.path.join(engine_root, 'Tools/build/JenkinsScripts/distribution/ThirdParty/generate_external_3rdparty_file_list.py')
package_config = os.path.join(engine_root, 'Tools/build/JenkinsScripts/distribution/ThirdParty/CMakePackageConfig.json')
args = [python, generate_external_3rdparty_file_list, '-s', versions_file, '-c', package_config, '-p', platform, '-i', include_pattern_file, '-e', exclude_pattern_file]
execute_system_call(args)
# Calculate filelist using include pattern and exclude pattern
thirdparty_home = package_env.get('THIRDPARTY_HOME')
filelist = {}
with open(include_pattern_file, 'r') as source:
include_patterns = source.readlines()
for include_pattern in include_patterns:
pattern = os.path.join(thirdparty_home, include_pattern.strip('\n'))
pattern = os.path.normpath(pattern)
result = glob(pattern, recursive=True)
files = [x for x in result if os.path.isfile(x) or (support_symlinks and os.path.islink(x))]
for file in files:
filelist[file] = os.path.join('3rdParty', os.path.relpath(file, thirdparty_home))
with open(exclude_pattern_file, 'r') as source:
exclude_patterns = source.readlines()
for exclude_pattern in exclude_patterns:
pattern = os.path.join(thirdparty_home, exclude_pattern.strip('\n'))
pattern = os.path.normpath(pattern)
result = glob(pattern, recursive=True)
files = [x for x in result if os.path.isfile(x) or (support_symlinks and os.path.islink(x))]
for file in files:
try:
filelist.pop(file)
except KeyError:
pass
return filelist
def parse_args():
cur_dir = os.path.dirname(os.path.abspath(__file__))
parser = OptionParser()
parser.add_option("--release", dest="release", default=False, action='store_true', help="Release build")
parser.add_option("--package_platform", dest="package_platform", default='consoles', help="Target platform to package")
parser.add_option("--package_env", dest="package_env", default=os.path.join(cur_dir, "cmake_package_env.json"),
help="JSON file that defines package environment variables")
parser.add_option("--package_build_configurations_json", dest="package_build_configurations_json",
default=os.path.join(cur_dir, "package_build_configurations.json"),
help="JSON file that defines build parameters")
(options, args) = parser.parse_args()
if options.package_platform is None:
error('No package platform specified')
return options, args
if __name__ == "__main__":
(options, args) = parse_args()
package(options)
Tools/build/JenkinsScripts/build/cmake_package_env.json
-106
View File
@@ -1,106 +0,0 @@
{
"global":{
"ENGINE_ROOT":"",
"THIRDPARTY_HOME":"",
"PACKAGE_NAME_PATTERN":"lumberyard-${MAJOR_VERSION}.${MINOR_VERSION}-${P4_CHANGELIST}",
"BUILD_NUMBER":"0",
"P4_CHANGELIST":"0",
"MAJOR_VERSION":"0",
"MINOR_VERSION":"0",
"LAD_PACKAGE_STORAGE_URL":"https://d7qxx8qkrwa8l.cloudfront.net"
},
"platforms":{
"consoles":{
"PACKAGE_TARGETS":[
{
"TYPE": "cmake_all",
"PACKAGE_NAME": "${PACKAGE_NAME_PATTERN}-consoles-${BUILD_NUMBER}.zip"
},
{
"TYPE": "symbols",
"PACKAGE_NAME": "${PACKAGE_NAME_PATTERN}-consoles-symbols-${BUILD_NUMBER}.zip"
}
],
"BOOTSTRAP_CFG_GAME_FOLDER":"AutomatedTesting",
"SKIP_BUILD": 1,
"BUILD_TARGETS":[
{
"BUILD_CONFIG_FILENAME": "build_config.json",
"PLATFORM": "Windows",
"TYPE": "profile_vs2017"
},
{
"BUILD_CONFIG_FILENAME": "build_config.json",
"PLATFORM": "Windows",
"TYPE": "profile_vs2019"
},
{
"BUILD_CONFIG_FILENAME": "build_config.json",
"PLATFORM": "Provo",
"TYPE": "profile"
}
]
},
"cmake_atom_pc":{
"PACKAGE_TARGETS":[
{
"TYPE": "cmake_atom_pc",
"PACKAGE_NAME": "${PACKAGE_NAME_PATTERN}-cmake_atom_pc-${BUILD_NUMBER}.zip"
}
],
"BOOTSTRAP_CFG_GAME_FOLDER":"AtomSampleViewer;AtomTest",
"SKIP_BUILD": 1,
"BUILD_TARGETS":[
{
"BUILD_CONFIG_FILENAME": "package_build_config.json",
"PLATFORM": "Windows",
"TYPE": "profile_vs2017_atom"
},
{
"BUILD_CONFIG_FILENAME": "package_build_config.json",
"PLATFORM": "Windows",
"TYPE": "profile_vs2019_atom"
}
]
},
"mac":{
"PACKAGE_TARGETS":[
{
"TYPE": "cmake_all",
"PACKAGE_NAME": "${PACKAGE_NAME_PATTERN}-cmake_mac_all-${BUILD_NUMBER}.zip"
}
],
"BOOTSTRAP_CFG_GAME_FOLDER":"AutomatedTesting",
"SKIP_BUILD": 1,
"BUILD_TARGETS":[
{
"BUILD_CONFIG_FILENAME": "build_config.json",
"PLATFORM": "Mac",
"TYPE": "profile"
},
{
"BUILD_CONFIG_FILENAME": "build_config.json",
"PLATFORM": "iOS",
"TYPE": "profile"
}
]
},
"linux":{
"PACKAGE_TARGETS":[
{
"TYPE": "cmake_all",
"PACKAGE_NAME": "${PACKAGE_NAME_PATTERN}-cmake_linux_all-${BUILD_NUMBER}.zip"
}
],
"BOOTSTRAP_CFG_GAME_FOLDER":"AutomatedTesting",
"SKIP_BUILD": 1,
"BUILD_TARGETS":[
{
"BUILD_CONFIG_FILENAME": "build_config.json",
"PLATFORM": "Linux",
"TYPE": "profile"
}
]
}
}
}
Tools/build/JenkinsScripts/build/download_latest_package_from_bucket.py
-80
View File
@@ -1,80 +0,0 @@
"""
All or portions of this file Copyright (c) Amazon.com, Inc. or its affiliates or
its licensors.
For complete copyright and license terms please see the LICENSE at the root of this
distribution (the "License"). All use of this software is governed by the License,
or, if provided, by the license below or the license accompanying this file. Do not
remove or modify any license notices. This file is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
Downloads the latest package from a S3 and unzips it to a desired location.
"""
import argparse
import boto3
import os
import re
import zipfile
def download_and_unzip_package(bucket_name, package_regex, build_number_regex, folder_path, destination_path):
"""
Downloads a given package from a S3 and unzips it.
:param bucket_name: S3 bucket
:param package_regex: Regex to find the desired package
:param build_number_regex: Regex to find the build number from the package name
:param folder_path: Folder path to the package
:param destination_path: Where to download the package to
:return:
"""
# Make sure the directory exists
if not os.path.isdir(destination_path):
os.makedirs(destination_path)
# Sorting function for latest package
def get_build_number(file_name_to_parse):
return re.search(build_number_regex, file_name_to_parse).group(0)[:-4] # [:-4] removes the .zip extension
s3 = boto3.resource('s3')
bucket = s3.Bucket(bucket_name)
largest_build_number = -1
latest_file = 'No file found!'
# Find the latest package
print 'Reading files from bucket...'
for bucket_file in bucket.objects.filter(Prefix=folder_path):
file_name = bucket_file.key
if re.search(package_regex, file_name) and get_build_number(file_name) > largest_build_number:
largest_build_number = get_build_number(file_name)
latest_file = file_name
package_name = latest_file.split('/')[-1]
# Download the package
print('Downloading package: {0} from bucket {1} to {2}'.format(latest_file, bucket_name, destination_path))
s3.Bucket(bucket_name).download_file(latest_file, os.path.join(destination_path, package_name))
# Unzip the package
with zipfile.ZipFile(os.path.join(destination_path, package_name), 'r') as zip_ref:
print('Unzipping package: {0} to {1}'.format(package_name, destination_path))
zip_ref.extractall(destination_path)
def main():
parser = argparse.ArgumentParser()
parser.add_argument('-b', '--bucket_name', required=True, help='Bucket that holds the package.')
parser.add_argument('-p', '--package_regex', required=True,
help='Regex to identify a package. Such as: lumberyard-0.0-[\d]{6,7}-pc-[\d]{4}.zip\s to find '
'the main pc package.')
parser.add_argument('-n', '--build_number_regex', required=True,
help='Regex to identify the build number. Such as [\d]{4,5}.zip$ to find the build number from '
'the name of the main pc package')
parser.add_argument('-d', '--destination_path', required=True, help='Destination for the contents of the packages.')
parser.add_argument('-f', '--folder_path', help='Folder that contains the package, must include /.')
args = parser.parse_args()
download_and_unzip_package(args.bucket_name, args.package_regex, args.build_number_regex, args.folder_path,
args.destination_path)
if __name__ == "__main__":
main()
Tools/build/JenkinsScripts/build/download_packages.py
-55
View File
@@ -1,55 +0,0 @@
"""
All or portions of this file Copyright (c) Amazon.com, Inc. or its affiliates or
its licensors.
For complete copyright and license terms please see the LICENSE at the root of this
distribution (the "License"). All use of this software is governed by the License,
or, if provided, by the license below or the license accompanying this file. Do not
remove or modify any license notices. This file is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
Downloads packages and unzips them.
"""
import argparse
import boto3
import os
import zipfile
def download_and_unzip_packages(bucket_name, package_key, folder_path, destination_path):
"""
Downloads a given package from a S3 and unzips it.
:param bucket_name: S3 bucket
:param package_key: Key for the package
:param folder_path: Folder path to the package
:param destination_path: Where to download the package to
:return:
"""
# Make sure the directory exists
if not os.path.isdir(destination_path):
os.makedirs(destination_path)
# Download the package
s3 = boto3.resource('s3')
print('Downloading package: {0} from bucket {1} to {2}'.format(package_key, bucket_name, destination_path))
s3.Bucket(bucket_name).download_file(folder_path + package_key, os.path.join(destination_path, package_key))
# Unzip the package
with zipfile.ZipFile(os.path.join(destination_path, package_key), 'r') as zip_ref:
print('Unzipping package: {0} to {1}'.format(package_key, destination_path))
zip_ref.extractall(destination_path)
def main():
parser = argparse.ArgumentParser()
parser.add_argument('-b', '--bucket_name', required=True, help='Bucket that holds the package.')
parser.add_argument('-p', '--package_key', required=True, help='Desired package\'s key.')
parser.add_argument('-d', '--destination_path', required=True, help='Destination for the contents of the packages.')
parser.add_argument('-f', '--folder_path', help='Folder that contains the package, must include /.')
args = parser.parse_args()
download_and_unzip_packages(args.bucket_name, args.package_key, args.folder_path, args.destination_path)
if __name__ == "__main__":
main()
Tools/build/JenkinsScripts/build/jenkins_scm_metrics.py
-57
View File
@@ -1,57 +0,0 @@
"""
All or portions of this file Copyright (c) Amazon.com, Inc. or its affiliates or
its licensors.
For complete copyright and license terms please see the LICENSE at the root of this
distribution (the "License"). All use of this software is governed by the License,
or, if provided, by the license below or the license accompanying this file. Do not
remove or modify any license notices. This file is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
"""
'''
All this script is doing is writing grabbing a file that was written previously marking the start of when the Perforce would run
and then getting the current time to find out how long we spent in Perforce
'''
import time
from utils.util import *
def write_metrics():
enable_build_metrics = os.environ.get('ENABLE_BUILD_METRICS')
metrics_namespace = os.environ.get('METRICS_NAMESPACE')
if enable_build_metrics == 'true':
scm_end = int(time.time())
workspace = os.environ.get('WORKSPACE')
metrics_file_name = 'scm_start.txt'
if workspace is None:
safe_exit_with_error('{} must be run in Jenkins job.'.format(os.path.basename(__file__)))
try:
with open(os.path.join(workspace, metrics_file_name), 'r') as f:
scm_start = int(f.readline())
except:
safe_exit_with_error('Failed to read from {}'.format(metrics_file_name))
scm_total = scm_end - scm_start
script_path = os.path.join(workspace, 'dev/Tools/build/waf-1.7.13/build_metrics/write_build_metric.py')
build_tag = os.environ.get('BUILD_TAG')
p4_changelist = os.environ.get('P4_CHANGELIST')
if build_tag is not None and p4_changelist is not None:
os.environ['BUILD_ID'] = '{0}.{1}'.format(build_tag, p4_changelist)
cwd = os.getcwd()
os.chdir(os.path.join(workspace, 'dev'))
cmd = 'python {} SCMTime {} Seconds --enable-build-metrics {} --metrics-namespace {} --project-spec None'.format(script_path, scm_total, True, metrics_namespace)
# metrics call shouldn't fail the job
safe_execute_system_call(cmd, shell=True)
os.chdir(cwd)
if __name__ == "__main__":
write_metrics()
Tools/build/JenkinsScripts/build/utils/__init__.py
-12
View File
@@ -1,12 +0,0 @@
"""
All or portions of this file Copyright (c) Amazon.com, Inc. or its affiliates or
its licensors.
For complete copyright and license terms please see the LICENSE at the root of this
distribution (the "License"). All use of this software is governed by the License,
or, if provided, by the license below or the license accompanying this file. Do not
remove or modify any license notices. This file is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
"""
Tools/build/JenkinsScripts/build/utils/copy_LAD_3rdParty.xml
-51
View File
@@ -1,51 +0,0 @@
<?xml version="1.0" encoding="utf-8"?>
<!--
Copyright (c) Amazon.com, Inc.
-->
<project name="CopyLadThirdParty" default="CopyLadThirdParty" basedir="../../../">
<fail message="Error: 3rdParty.home is not set">
<condition>
<not>
<isset property="3rdParty.home"/>
</not>
</condition>
</fail>
<fail message="Error: 3rdParty.destination is not set">
<condition>
<not>
<isset property="3rdParty.destination"/>
</not>
</condition>
</fail>
<fail message="Error: platform is not set">
<condition>
<not>
<isset property="platform"/>
</not>
</condition>
</fail>
<include file="../../distribution/package/3rdParty.xml" optional="false" />
<target name="CopyLadThirdParty">
<ThirdPartySDKsGeneratePlatformPatternSet platform="${platform}"/>
<copy todir="${3rdParty.destination}">
<fileset dir="${3rdParty.home}">
<patternset refid="include-3rdparty-patternset-common" />
</fileset>
</copy>
<copy todir="${3rdParty.destination}">
<fileset dir="${3rdParty.home}">
<patternset refid="include-3rdparty-patternset-${platform}" />
</fileset>
</copy>
<copy todir="${3rdParty.destination}">
<fileset dir="${3rdParty.home}">
<patternset id="include-3rdparty-patternset-non-shipped">
<include name="FbxSdk/**"/>
</patternset>
</fileset>
</copy>
</target>
</project>
Tools/build/JenkinsScripts/build/utils/download_from_s3.py
-102
View File
@@ -1,102 +0,0 @@
"""
All or portions of this file Copyright (c) Amazon.com, Inc. or its affiliates or
its licensors.
For complete copyright and license terms please see the LICENSE at the root of this
distribution (the "License"). All use of this software is governed by the License,
or, if provided, by the license below or the license accompanying this file. Do not
remove or modify any license notices. This file is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
"""
'''
Usage:
Use EC2 role to download files to %WORKSPACE% folder from bucket bucket_name:
python download_from_s3.py --base_dir %WORKSPACE% --files_to_download "file1,file2" --bucket bucket_name
Use profile to download files to %WORKSPACE% folder from bucket bucket_name:
python download_from_s3.py --base_dir %WORKSPACE% --profile profile --files_to_download "file1,file2" --bucket bucket_name
'''
import os
import json
import boto3
from optparse import OptionParser
from util import error
def parse_args():
parser = OptionParser()
parser.add_option("--base_dir", dest="base_dir", default=os.getcwd(), help="Base directory to download files, If not given, then current directory is used.")
parser.add_option("--files_to_download", dest="files_to_download", default=None, help="Files to download, separated by comma.")
parser.add_option("--profile", dest="profile", default=None, help="The name of a profile to use. If not given, then the default profile is used.")
parser.add_option("--bucket", dest="bucket", default=None, help="S3 bucket the files are downloaded from.")
parser.add_option("--key_prefix", dest="key_prefix", default='', help="Object key prefix.")
'''
ExtraArgs used to call s3.download_file(), should be in json format. extra_args key must be one of: ACL, CacheControl, ContentDisposition, ContentEncoding, ContentLanguage, ContentType, Expires,
GrantFullControl, GrantRead, GrantReadACP, GrantWriteACP, Metadata, RequestPayer, ServerSideEncryption, StorageClass,
SSECustomerAlgorithm, SSECustomerKey, SSECustomerKeyMD5, SSEKMSKeyId, WebsiteRedirectLocation
'''
parser.add_option("--extra_args", dest="extra_args", default=None, help="Additional parameters used to download file.")
parser.add_option("--max_retry", dest="max_retry", default=1, help="Maximum retry times to download file.")
(options, args) = parser.parse_args()
if not os.path.isdir(options.base_dir):
error('{} is not a valid directory'.format(options.base_dir))
if not options.files_to_download:
error('Use --files_to_download to specify files to download, separated by comma.')
if not options.bucket:
error('Use --bucket to specify bucket that the files are downloaded from.')
return options
def get_client(service_name, profile_name=None):
session = boto3.session.Session(profile_name=profile_name)
client = session.client(service_name)
return client
def s3_download_file(client, base_dir, file, bucket, key_prefix=None, extra_args=None, max_retry=1):
print 'Downloading file {} from bucket {}.'.format(file, bucket)
key = file if key_prefix is None else '{}/{}'.format(key_prefix, file)
for x in range(max_retry):
try:
client.download_file(
bucket, key, os.path.join(base_dir, file),
ExtraArgs=extra_args
)
print 'Download succeeded'
return True
except:
print 'Retrying download...'
print 'Download failed'
return False
def download_files(base_dir, files_to_download, bucket, key_prefix=None, profile=None, extra_args=None, max_retry=1):
client = get_client('s3', profile)
files_to_download = files_to_download.split(',')
extra_args = json.loads(extra_args) if extra_args else None
print 'Downloading {} files from bucket {}.'.format(len(files_to_download), bucket)
failure = []
success = []
for file in files_to_download:
if not s3_download_file(client, base_dir, file, bucket, key_prefix, extra_args, max_retry):
failure.append(file)
else:
success.append(file)
print '{} files are downloaded successfully:'.format(len(success))
print '\n'.join(success)
print '{} files failed to download:'.format(len(failure))
print '\n'.join(failure)
# Exit with error code 1 if any file is failed to download
if len(failure) > 0:
return False
return True
if __name__ == "__main__":
options = parse_args()
download_files(options.base_dir, options.files_to_download, options.bucket, options.key_prefix, options.profile, options.extra_args, options.max_retry)
Tools/build/JenkinsScripts/build/utils/email_to_lionbridge.py
-129
View File
@@ -1,129 +0,0 @@
"""
All or portions of this file Copyright (c) Amazon.com, Inc. or its affiliates or
its licensors.
For complete copyright and license terms please see the LICENSE at the root of this
distribution (the "License"). All use of this software is governed by the License,
or, if provided, by the license below or the license accompanying this file. Do not
remove or modify any license notices. This file is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
"""
"""
This script will be used in https://jenkins.agscollab.com/view/%7ESandbox/job/PACKAGE_COPY_S3/
PACKAGE_COPY_S3 is a downstream job of nightly packaging job, it copies the nightly packages from Infra S3 bucket to Lionbridge S3 bucket based on the INCLUDE_FILTER passed from packaging job
"""
import os
import re
import json
import requests
from requests.auth import HTTPBasicAuth
import boto3
from util import error, warn
# Write EMAIL_TEMPLATE to a file and inject it into the email sent to Lionbridge
EMAIL_TEMPLATE = '''Packages are uploaded to S3 bucket {}
Package List:
{}
Changelists:
{}
'''
def get_jenkins_env(key):
try:
return os.environ[key]
except KeyError:
print 'Error: Jenkins parameters {} is not set.'.format(key)
return None
JENKINS_USERNAME = get_jenkins_env('JENKINS_USERNAME')
JENKINS_API_TOKEN = get_jenkins_env('JENKINS_API_TOKEN')
JENKINS_URL = get_jenkins_env('JENKINS_URL')
WORKSPACE = get_jenkins_env('WORKSPACE')
S3_TARGET = get_jenkins_env('S3_TARGET')
INCLUDE_FILTER = get_jenkins_env('INCLUDE_FILTER')
EMAIL_TEMPLATE_FILE = get_jenkins_env('EMAIL_TEMPLATE_FILE')
if None in [JENKINS_USERNAME, JENKINS_API_TOKEN, JENKINS_URL, WORKSPACE, S3_TARGET, INCLUDE_FILTER, EMAIL_TEMPLATE_FILE]:
error('Please make sure all Jenkins parameters are set correctly.')
def parse_include_filter(include_filter):
try:
res = re.search('^(\w*)-*lumberyard-(\d+)\.(\d+)-(\d+)-(\w+).*\*(\d+)\.\*', include_filter)
branch = res.group(1)
major_version = int(res.group(2))
minor_version = int(res.group(3))
changelist_number = res.group(4)
platform = res.group(5)
build_number = res.group(6)
return branch, major_version, minor_version, changelist_number, platform, build_number
except (AttributeError, IndexError):
error('Unable to parse INCLUDE_FILTER, please make sure the INCLUDE_FILTER is set correctly')
# Get the changelists that trigger the build
def get_changelists(job_name, build_number):
changelists = []
headers = {'Content-type': 'application/json', 'Accept': 'application/json'}
try:
res = requests.get('{}/job/{}/{}/api/json'.format(JENKINS_URL, job_name, build_number),
auth=HTTPBasicAuth(JENKINS_USERNAME, JENKINS_API_TOKEN), headers=headers, verify=False)
res = json.loads(res.content)
changelists = res.get('changeSet').get('items')
return changelists
except:
warn('Error: Failed to get changes from build {} in job {}'.format(build_number, job_name))
return []
def get_packaging_job_name(branch, major_version, minor_version, platform):
if branch == '':
branch = 'ML' if major_version + minor_version == 0 else 'v{}_{}'.format(major_version, minor_version)
job_name = 'PKG_{}_{}'.format(branch, platform.capitalize())
return job_name
# Get package names by looking up S3 bucket
def get_package_names(branch, major_version, minor_version, include_filter, build_number):
package_names = []
prefix = include_filter[:include_filter.find('*')]
pattern = '.*{}.*{}..*'.format(prefix, build_number)
if branch == '':
bucket_name = 'ly-packages-mainline' if major_version + minor_version == 0 else 'ly-packages-release-candidate'
folder = 'lumberyard-packages'
else:
bucket_name = 'ly-packages-feature-branches'
folder = 'lumberyard-packages/{}'.format(branch)
s3 = boto3.resource('s3')
bucket = s3.Bucket(bucket_name)
for obj in bucket.objects.filter(Prefix='{}/{}'.format(folder, prefix)):
package_name = obj.key
if re.match(pattern, package_name):
package_names.append(package_name.replace('{}/'.format(folder), ''))
return package_names
if __name__ == "__main__":
branch, major_version, minor_version, changelist_number, platform, build_number = parse_include_filter(INCLUDE_FILTER)
packaging_job_name = get_packaging_job_name(branch, major_version, minor_version, platform)
changelists = get_changelists(packaging_job_name, build_number)
package_names = get_package_names(branch, major_version, minor_version, INCLUDE_FILTER, build_number)
with open(os.path.join(WORKSPACE, EMAIL_TEMPLATE_FILE), 'w+') as output:
if len(package_names) > 0:
package_list_str = '\n'.join(package_names)
changelists_str = ''
for item in changelists:
changelists_str += '---------------------------------------------------------------------------------------------\n'
try:
changelists_str += 'CL{} by {} on {}\n{}\n'.format(item['changeNumber'], item['author']['fullName'], item['changeTime'], item['msg'].encode('utf-8', 'ignore'))
except KeyError:
error('Internal error, check the output of Jenkins API.')
output.write(EMAIL_TEMPLATE.format(S3_TARGET, package_list_str, changelists_str))
Tools/build/JenkinsScripts/build/utils/incremental_build_util.py
-662
View File
@@ -1,662 +0,0 @@
"""
All or portions of this file Copyright (c) Amazon.com, Inc. or its affiliates or
its licensors.
For complete copyright and license terms please see the LICENSE at the root of this
distribution (the "License"). All use of this software is governed by the License,
or, if provided, by the license below or the license accompanying this file. Do not
remove or modify any license notices. This file is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
"""
import ast
import boto3
import datetime
import urllib2
import os
import time
import subprocess
import sys
import tempfile
import traceback
import shutil
import platform
import stat
IAM_ROLE_NAME = 'ec2-jenkins-node'
if os.name == 'nt':
import ctypes
import win32api
import collections
import locale
locale.setlocale(locale.LC_ALL, '') # set locale to default to get thousands separators
PULARGE_INTEGER = ctypes.POINTER(ctypes.c_ulonglong) # Pointer to large unsigned integer
kernel32 = ctypes.WinDLL('kernel32', use_last_error=True)
kernel32.GetDiskFreeSpaceExW.argtypes = (ctypes.c_wchar_p,) + (PULARGE_INTEGER,) * 3
class UsageTuple(collections.namedtuple('UsageTuple', 'total, used, free')):
def __str__(self):
# Add thousands separator to numbers displayed
return self.__class__.__name__ + '(total={:n}, used={:n}, free={:n})'.format(*self)
def is_dir_symlink(path):
FILE_ATTRIBUTE_REPARSE_POINT = 0x0400
return os.path.isdir(path) and (ctypes.windll.kernel32.GetFileAttributesW(unicode(path)) & FILE_ATTRIBUTE_REPARSE_POINT)
def get_free_space_mb(path):
if sys.version_info < (3,): # Python 2?
saved_conversion_mode = ctypes.set_conversion_mode('mbcs', 'strict')
else:
try:
path = os.fsdecode(path) # allows str or bytes (or os.PathLike in Python 3.6+)
except AttributeError: # fsdecode() not added until Python 3.2
pass
# Define variables to receive results when passed as "by reference" arguments
_, total, free = ctypes.c_ulonglong(), ctypes.c_ulonglong(), ctypes.c_ulonglong()
success = kernel32.GetDiskFreeSpaceExW(
path, ctypes.byref(_), ctypes.byref(total), ctypes.byref(free))
if not success:
error_code = ctypes.get_last_error()
if sys.version_info < (3,): # Python 2?
ctypes.set_conversion_mode(*saved_conversion_mode) # restore conversion mode
if not success:
windows_error_message = ctypes.FormatError(error_code)
raise ctypes.WinError(error_code, '{} {!r}'.format(windows_error_message, path))
used = total.value - free.value
return free.value / 1024 / 1024#for now
else:
def get_free_space_mb(dirname):
st = os.statvfs(dirname)
return st.f_bavail * st.f_frsize / 1024 / 1024
def get_iam_role_credentials(role_name):
security_metadata = None
try:
response = urllib2.urlopen(
'http://169.254.169.254/latest/meta-data/iam/security-credentials/{0}'.format(role_name)).read()
security_metadata = ast.literal_eval(response)
except:
print 'Unable to get iam role credentials'
print traceback.print_exc()
return security_metadata
def create_volume(ec2_client, availability_zone, project_name, volume_counter):
response = ec2_client.create_volume(
AvailabilityZone=availability_zone,
Size=300,
VolumeType='gp2',
TagSpecifications=
[
{
'ResourceType': 'volume',
'Tags':
[
{
'Key': 'Name',
'Value': '{0}'.format(project_name)
},
{
'Key': 'VolumeCounter',
'Value': str(volume_counter)
}
]
}
]
)
print response
volume_id = response['VolumeId']
# give some time for the creation call to complete
time.sleep(1)
response = ec2_client.describe_volumes(VolumeIds=[volume_id, ])
while (response['Volumes'][0]['State'] != 'available'):
time.sleep(1)
response = ec2_client.describe_volumes(VolumeIds=[volume_id, ])
return volume_id
def delete_volume(ec2_client, volume_id):
response = ec2_client.delete_volume(VolumeId=volume_id)
def unmount_build_volume_from_node():
if os.name == 'nt':
f = tempfile.NamedTemporaryFile(delete=False)
f.write("""
select disk 1
offline disk
""")
f.close()
subprocess.call('diskpart /s %s' % f.name)
os.unlink(f.name)
else:
subprocess.call(['umount', '/data'])
def detach_volume_from_node(ec2_client, volume, instance_id, force):
ec2_client.delete_tags(Resources=[volume.volume_id],
Tags=[
{
'Key': 'jenkins_attachment_node',
},
{
'Key': 'jenkins_attachment_time',
},
{
'Key': 'jenkins_attachment_build'
}
])
incremental_keys = ['jenkins_attachment_node', 'jenkins_attachment_time', 'jenkins_attachment_build']
volume.load()
print 'searching for keys adding during incremental build: {}'.format(incremental_keys)
while len(incremental_keys):
tag_keys = set()
for tag in volume.tags:
tag_keys.add(tag['Key'])
print 'found tags on instace {}'.format(tag_keys)
for incremental_key in list(incremental_keys):
if incremental_key not in tag_keys:
print 'incremental key {} has been successfully removed'.format(incremental_key)
incremental_keys.remove(incremental_key)
volume.load()
volume.detach_from_instance(Device='xvdf',
Force=force,
InstanceId=instance_id,
VolumeId=volume.volume_id)
while (len(volume.attachments) and volume.attachments[0]['State'] != 'detached'):
time.sleep(1)
volume.load()
volume.load()
if (len(volume.attachments)):
print 'Volume still has attachments'
for attachment in volume.attachments:
print 'Volume {} {} to instance {}'.format(attachment['VolumeId'], attachment['State'], attachment['InstanceId'])
def cleanup_node(workspace_name):
if os.name == 'nt':
jenkins_base = os.getenv('BASE')
dev_path = '{}\\workspace\\{}\\dev'.format(jenkins_base, workspace_name)
else:
dev_path = '/home/lybuilder/ly/workspace/{}/dev'.format(workspace_name)
if os.path.exists(dev_path):
if os.name == 'nt':
if is_dir_symlink(dev_path):
print "removing symlink path {}".format(dev_path)
os.rmdir(dev_path)
else:
# this shouldn't happen, but is here for sanity's sake, if we sync to the build node erroneously we want to clean it up if we can
print "given symlink path was not a symlink, deleting the full tree to prevent future build failures"
retcode = os.system('rmdir /S /Q {}'.format(dev_path))
if retcode != 0:
raise Exception("rmdir failed to remove directory: {}".format(dev_path))
return True
else:
if os.path.islink(dev_path):
print "unlinking symlink path {}".format(dev_path)
os.unlink(dev_path)
else:
print "given symlink path was not a symlink, deleting the full tree to prevent future build failures"
os.chmod(dev_path, stat.S_IWUSR)
shutil.rmtree(dev_path, ignore_errors=True)
return True
# check to make sure the directory was actually deleted
if os.path.exists(dev_path):
raise Exception("Failed to remove directory: {}".format(dev_path))
return False
def setup_volume(workspace_name, created):
if os.name == 'nt':
f = tempfile.NamedTemporaryFile(delete=False)
f.write("""
select disk 1
online disk
attribute disk clear readonly
""") # assume disk # for now
if created:
f.write("""create partition primary
select partition 1
format quick fs=ntfs
assign
active
""")
f.close()
subprocess.call(['diskpart', '/s', f.name])
time.sleep(2)
drives_after = win32api.GetLogicalDriveStrings()
drives_after = drives_after.split('\000')[:-1]
print drives_after
#drive_letter = next(item for item in drives_after if item not in drives_before)
drive_letter = 'D:\\'
os.unlink(f.name)
time.sleep(1)
dev_path = '{}ly\workspace\{}\dev'.format(drive_letter, workspace_name)
else:
subprocess.call(['file', '-s', '/dev/xvdf'])
if created:
subprocess.call(['mkfs', '-t', 'ext4', '/dev/xvdf'])
subprocess.call(['mount', '/dev/xvdf', '/data'])
dev_path = '/data/ly/workspace/{}/dev'.format(workspace_name)
return dev_path
def attach_volume_to_instance(volume, volume_id, instance_id, instance_name):
volume.attach_to_instance(Device='xvdf',
InstanceId=instance_id,
VolumeId=volume_id)
# give a little bit of time for the aws call to process
time.sleep(2)
# reload the volume just in case
volume.load()
while (len(volume.attachments) and volume.attachments[0]['State'] != 'attached'):
time.sleep(1)
volume.load()
volume.create_tags(Tags=[
{
'Key':'last_attachment_time',
'Value':datetime.datetime.utcnow().isoformat()
}
])
volume.create_tags(Tags=[
{
'Key':'jenkins_attachment_node',
'Value':instance_name,
},
{
'Key':'jenkins_attachment_time',
'Value':datetime.datetime.utcnow().isoformat()
},
{
'Key':'jenkins_attachment_build',
'Value':os.getenv('BUILD_TAG')
}
])
def prepare_incremental_build(workspace_name):
job_name = os.getenv('JOB_NAME', None)
clean_build = os.getenv('CLEAN_BUILD', 'false').lower() == 'true'
android_home = os.getenv('ANDROID_HOME', None)
if android_home is not None:
path = os.getenv('PATH').split(';')
print path
java_home = os.getenv('JAVA_HOME', None)
print java_home
os.environ['LY_NDK_PATH'] = 'C:\\ly\\3rdParty\\android-ndk\\r12'
print os.getenv('LY_NDK_PATH')
path = [x for x in path if not (java_home in x or android_home in x)]
print path
path.append(java_home)
path.append(android_home)
path.append(os.getenv('LY_NDK_PATH'))
print path
os.environ['PATH'] = ';'.join(path)
credentials = get_iam_role_credentials(IAM_ROLE_NAME)
aws_access_key_id = None
aws_secret_access_key = None
aws_session_token = None
if credentials is not None:
keys = ['AccessKeyId', 'SecretAccessKey', 'Token']
for key in keys:
if key not in credentials:
print 'Unable to find {0} in get_iam_role_credentials response {1}'.format(key, credentials)
return
aws_access_key_id = credentials['AccessKeyId']
aws_secret_access_key = credentials['SecretAccessKey']
aws_session_token = credentials['Token']
session = boto3.session.Session()
region = session.region_name
try:
instance_id = urllib2.urlopen('http://169.254.169.254/latest/meta-data/instance-id').read()
except:
# this likely means we're not an ec2 instance
raise Exception('No EC2 metadata!')
try:
availability_zone = urllib2.urlopen(
'http://169.254.169.254/latest/meta-data/placement/availability-zone').read()
except:
# also likely means we're not an ec2 instance
raise Exception('No EC2 metadata')
if region is None:
region = 'us-west-2'
client = boto3.client('ec2', region_name=region, aws_access_key_id=aws_access_key_id,
aws_secret_access_key=aws_secret_access_key,
aws_session_token=aws_session_token)
project_name = job_name
ec2_resource = boto3.resource('ec2', region_name=region)
instance = ec2_resource.Instance(instance_id)
volume_counter = 0
for volume in instance.volumes.all():
for attachment in volume.attachments:
print 'attachment device: {}'.format(attachment['Device'])
if 'xvdf' in attachment['Device'] and attachment['State'] != 'detached':
print 'A device is already attached to xvdf. This likely means a previous build failed to detach it\'s' \
'build volume. This volume is considered orphaned and will be force detached from this instance.'
unmount_build_volume_from_node()
detach_volume_from_node(client, volume, instance_id, True)
if cleanup_node(workspace_name):
clean_build = True
response = client.describe_volumes(Filters=
[
{
'Name': 'tag:Name',
'Values':
[
'{0}'.format(project_name)
]
}
])
created = False
if 'Volumes' in response and not len(response['Volumes']):
print 'Volume for {0} doesn\'t exist creating it...'.format(project_name)
# volume doesn't exist, create it
volume_id = create_volume(client, availability_zone, project_name, volume_counter)
created = True
elif len(response['Volumes']) > 1:
latest_volume = None
max_counter = 0
for volume in response['Volumes']:
for tag in volume['Tags']:
if tag['Key'] == 'VolumeCounter':
if int(tag['Value']) > max_counter:
max_counter = int(tag['Value'])
latest_volume = volume
volume_counter = max_counter
volume_id = latest_volume['VolumeId']
else:
volume = response['Volumes'][0]
if len(volume['Attachments']):
# this is bad we shouldn't be attached, we should have detached at the end of a build
attachment = volume['Attachments'][0]
print ('Volume already has attachment {}'.format(attachment))
print 'Creating new volume for {} and orphaning previous volume'.format(project_name)
for tag in volume['Tags']:
if tag['Key'] == 'VolumeCounter':
volume_counter = int(tag['Value']) + 1
break
volume_id = create_volume(client, availability_zone, project_name, volume_counter)
created = True
else:
volume_id = volume['VolumeId']
if clean_build and not created:
print 'CLEAN_BUILD option was set, deleting volume {0}'.format(volume_id)
revert_workspace(job_name)
delete_volume(client, volume_id)
volume_id = create_volume(client, availability_zone, project_name, volume_counter)
created = True
print 'attaching volume {} to instance {}'.format(volume_id, instance_id)
volume = ec2_resource.Volume(volume_id)
instance_name = next(tag['Value'] for tag in instance.tags if tag['Key'] == 'Name')
if os.name == 'nt':
drives_before = win32api.GetLogicalDriveStrings()
drives_before = drives_before.split('\000')[:-1]
print drives_before
attach_volume_to_instance(volume, volume_id, instance_id, instance_name)
dev_path = setup_volume(workspace_name, created)
dev_existed = True
if os.name == 'nt':
free_space_path = 'D:\\'
else:
free_space_path = '/data/'
if get_free_space_mb(free_space_path) < 1024:
print 'Volume is running low on disk space. Recreating volume and running clean build.'
unmount_build_volume_from_node()
detach_volume_from_node(client, volume, instance_id, False)
delete_volume(client, volume_id)
volume_id = create_volume(client, availability_zone, project_name, volume_counter)
volume = ec2_resource.Volume(volume_id)
attach_volume_to_instance(volume, volume_id, instance_id, instance_name)
setup_volume(workspace_name, True)
if not os.path.exists(dev_path):
print 'creating directory structure for {}'.format(dev_path)
os.makedirs(dev_path)
if os.name != 'nt':
print 'taking ownership of {}'.format(dev_path)
subprocess.call(['chown', '-R', 'lybuilder:root', dev_path])
dev_existed = False
if os.name == 'nt':
jenkins_base = os.getenv('BASE')
try:
symlink_path = '{}\\workspace\\{}\\dev'.format(jenkins_base, workspace_name)
print 'creating symlink to path: {}'.format(symlink_path)
subprocess.call(['cmd', '/c', 'mklink', '/J', symlink_path, dev_path])
#subprocess.call(['cmd', '/c', 'mklink', '/J', '{}\\3rdParty'.format(jenkins_base), 'E:\\3rdParty'])
except Exception as e:
print e
else:
subprocess.call(['ln', '-s', '-f', dev_path, '/home/lybuilder/ly/workspace/{}'.format(workspace_name)])
subprocess.call(['ln', '-s', '-f', '/home/lybuilder/ly/workspace/3rdParty', '/data/ly/workspace'])
if not dev_existed:
print 'flushing perforce #have revision'
subprocess.call(['p4', 'trust'])
subprocess.call(['p4', 'flush', '-f', '//ly_jenkins_{}/dev/...#none'.format(job_name)])
#subprocess.call(['p4', 'sync', '-f', '//ly_jenkins_{}/dev/...'.format(job_name)])
def revert_workspace(job_name):
try:
# Workaround for LY-86789: Revert bootstrap.cfg checkout.
print "REVERTING workspace {}".format(job_name)
subprocess.check_call(['p4', 'revert', '//ly_jenkins_{}/dev/...'.format(job_name)])
except subprocess.CalledProcessError as e:
print e.output
raise e
except Exception as e:
print e
raise e
def teardown_incremental_build(workspace_name):
job_name = os.getenv('JOB_NAME', None)
if os.path.isfile('envinject.properties'):
os.remove('envinject.properties')
credentials = get_iam_role_credentials(IAM_ROLE_NAME)
aws_access_key_id = None
aws_secret_access_key = None
aws_session_token = None
if credentials is not None:
keys = ['AccessKeyId', 'SecretAccessKey', 'Token']
for key in keys:
if key not in credentials:
raise Exception('Unable to find {0} in get_iam_role_credentials response {1}'.format(key, credentials))
aws_access_key_id = credentials['AccessKeyId']
aws_secret_access_key = credentials['SecretAccessKey']
aws_session_token = credentials['Token']
session = boto3.session.Session()
region = session.region_name
try:
instance_id = urllib2.urlopen('http://169.254.169.254/latest/meta-data/instance-id').read()
except:
# this likely means we're not an ec2 instance
raise Exception('No EC2 metadata!')
if region is None:
region = 'us-west-2'
client = boto3.client('ec2', region_name=region, aws_access_key_id=aws_access_key_id,
aws_secret_access_key=aws_secret_access_key,
aws_session_token=aws_session_token)
project_name = job_name
response = client.describe_volumes(Filters=
[
{
'Name': 'tag:Name',
'Values':
[
'{0}'.format(project_name)
]
}
])
ec2_resource = boto3.resource('ec2', region_name=region)
instance = ec2_resource.Instance(instance_id)
volume = None
for attached_volume in instance.volumes.all():
for attachment in attached_volume.attachments:
print 'attachment device: {}'.format(attachment['Device'])
if attachment['Device'] == 'xvdf':
volume = attached_volume
if volume is None:
# volume doesn't exist, do nothing
print 'Volume for {} does not exist or is not attached to the current instance. This probably isn\'t an issue but should be reported.'.format(project_name)
return
else:
revert_workspace(job_name)
unmount_build_volume_from_node()
detach_volume_from_node(client, volume, instance_id, False)
cleanup_node(workspace_name)
def prepare_incremental_build_mac(workspace_name):
job_name = os.getenv('JOB_NAME', None)
clean_build = os.getenv('CLEAN_BUILD', 'false').lower() == 'true'
subprocess.call(['mount', '-t', 'smbfs', '//lybuilder:Builder99@gt-sna11-nas-01.local/inc-build/ly', '/data/ly'])
dev_path = '/data/ly/workspace/{}/dev'.format(workspace_name)
dev_existed = True
if clean_build:
print 'cleaning {}'.format(dev_path)
subprocess.call(['rm', '-rf', dev_path])
if not os.path.exists(dev_path):
print 'creating directory structure for {}'.format(dev_path)
os.makedirs(dev_path)
dev_existed = False
#subprocess.call(['ln', '-s', '-f', '/data/ly/workspace', '/Users/lybuilder'])
#subprocess.call(['ln', '-s', '-f', '/Users/lybuilder/workspace/3rdParty', '/data/ly/workspace'])
if not dev_existed:
print 'flushing perforce #have revision'
subprocess.call(['p4', 'trust'])
subprocess.call(['p4', 'flush', '-f', '//ly_jenkins_{}/dev/...#none'.format(job_name)])
def main():
action = sys.argv[1]
workspace_name = sys.argv[2]
if action.lower() == 'prepare':
if platform.system().lower() == 'darwin':
prepare_incremental_build_mac(workspace_name)
else:
prepare_incremental_build(workspace_name)
elif action.lower() == 'teardown':
if platform.system().lower() == 'darwin':
pass
else:
teardown_incremental_build(workspace_name)
else:
'Invalid command. Valid actions are either "prepare" or "teardown."'
if __name__ == '__main__':
main()
Tools/build/JenkinsScripts/build/utils/jenkins_scm_metrics.py
-57
View File
@@ -1,57 +0,0 @@
"""
All or portions of this file Copyright (c) Amazon.com, Inc. or its affiliates or
its licensors.
For complete copyright and license terms please see the LICENSE at the root of this
distribution (the "License"). All use of this software is governed by the License,
or, if provided, by the license below or the license accompanying this file. Do not
remove or modify any license notices. This file is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
"""
'''
All this script is doing is writing grabbing a file that was written previously marking the start of when the Perforce would run
and then getting the current time to find out how long we spent in Perforce
'''
import time
from util import *
def write_metrics():
enable_build_metrics = os.environ.get('ENABLE_BUILD_METRICS')
metrics_namespace = os.environ.get('METRICS_NAMESPACE')
if enable_build_metrics == 'true':
scm_end = int(time.time())
workspace = os.environ.get('WORKSPACE')
metrics_file_name = 'scm_start.txt'
if workspace is None:
safe_exit_with_error('{} must be run in Jenkins job.'.format(os.path.basename(__file__)))
try:
with open(os.path.join(workspace, metrics_file_name), 'r') as f:
scm_start = int(f.readline())
except:
safe_exit_with_error('Failed to read from {}'.format(metrics_file_name))
scm_total = scm_end - scm_start
script_path = os.path.join(workspace, 'dev/Tools/build/waf-1.7.13/build_metrics/write_build_metric.py')
build_tag = os.environ.get('BUILD_TAG')
p4_changelist = os.environ.get('P4_CHANGELIST')
if build_tag is not None and p4_changelist is not None:
os.environ['BUILD_ID'] = '{0}.{1}'.format(build_tag, p4_changelist)
cwd = os.getcwd()
os.chdir(os.path.join(workspace, 'dev'))
cmd = 'python {} SCMTime {} Seconds --enable-build-metrics {} --metrics-namespace {} --project-spec None'.format(script_path, scm_total, True, metrics_namespace)
# metrics call shouldn't fail the job
safe_execute_system_call(cmd, shell=True)
os.chdir(cwd)
if __name__ == "__main__":
write_metrics()
Tools/build/JenkinsScripts/build/utils/lib/__init__.py
-12
View File
@@ -1,12 +0,0 @@
"""
All or portions of this file Copyright (c) Amazon.com, Inc. or its affiliates or
its licensors.
For complete copyright and license terms please see the LICENSE at the root of this
distribution (the "License"). All use of this software is governed by the License,
or, if provided, by the license below or the license accompanying this file. Do not
remove or modify any license notices. This file is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
"""
Tools/build/JenkinsScripts/build/utils/lib/glob3.py
-174
View File
@@ -1,174 +0,0 @@
"""
All or portions of this file Copyright (c) Amazon.com, Inc. or its affiliates or
its licensors.
For complete copyright and license terms please see the LICENSE at the root of this
distribution (the "License"). All use of this software is governed by the License,
or, if provided, by the license below or the license accompanying this file. Do not
remove or modify any license notices. This file is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
"""
"""
Filename globbing utility.
Modified using https://github.com/python/cpython/blob/3.7/Lib/glob.py to be compatible with Python2
Original file Copyright Python Software Foundation, used under license.
Modifications copyright Amazon.com, Inc. or its affiliates.
"""
import os
import re
import fnmatch
__all__ = ["glob", "iglob", "escape"]
def glob(pathname, recursive=False):
"""Return a list of paths matching a pathname pattern.
The pattern may contain simple shell-style wildcards a la
fnmatch. However, unlike fnmatch, filenames starting with a
dot are special cases that are not matched by '*' and '?'
patterns.
If recursive is true, the pattern '**' will match any files and
zero or more directories and subdirectories.
"""
return list(iglob(pathname, recursive=recursive))
def iglob(pathname, recursive=False):
"""Return an iterator which yields the paths matching a pathname pattern.
The pattern may contain simple shell-style wildcards a la
fnmatch. However, unlike fnmatch, filenames starting with a
dot are special cases that are not matched by '*' and '?'
patterns.
If recursive is true, the pattern '**' will match any files and
zero or more directories and subdirectories.
"""
it = _iglob(pathname, recursive, False)
if recursive and _isrecursive(pathname):
s = next(it) # skip empty string
assert not s
return it
def _iglob(pathname, recursive, dironly):
dirname, basename = os.path.split(pathname)
if not has_magic(pathname):
assert not dironly
if basename:
if os.path.lexists(pathname):
yield pathname
else:
# Patterns ending with a slash should match only directories
if os.path.isdir(dirname):
yield pathname
return
if not dirname:
if recursive and _isrecursive(basename):
yield _glob2(dirname, basename, dironly)
else:
yield _glob1(dirname, basename, dironly)
return
# `os.path.split()` returns the argument itself as a dirname if it is a
# drive or UNC path. Prevent an infinite recursion if a drive or UNC path
# contains magic characters (i.e. r'\\?\C:').
if dirname != pathname and has_magic(dirname):
dirs = _iglob(dirname, recursive, True)
else:
dirs = [dirname]
if has_magic(basename):
if recursive and _isrecursive(basename):
glob_in_dir = _glob2
else:
glob_in_dir = _glob1
else:
glob_in_dir = _glob0
for dirname in dirs:
for name in glob_in_dir(dirname, basename, dironly):
yield os.path.join(dirname, name)
# These 2 helper functions non-recursively glob inside a literal directory.
# They return a list of basenames. _glob1 accepts a pattern while _glob0
# takes a literal basename (so it only has to check for its existence).
def _glob1(dirname, pattern, dironly):
names = list(_iterdir(dirname, dironly))
return fnmatch.filter(names, pattern)
def _glob0(dirname, basename, dironly):
if not basename:
# `os.path.split()` returns an empty basename for paths ending with a
# directory separator. 'q*x/' should match only directories.
if os.path.isdir(dirname):
return [basename]
else:
if os.path.lexists(os.path.join(dirname, basename)):
return [basename]
return []
# Following functions are not public but can be used by third-party code.
def glob0(dirname, pattern):
return _glob0(dirname, pattern, False)
def glob1(dirname, pattern):
return _glob1(dirname, pattern, False)
# This helper function recursively yields relative pathnames inside a literal
# directory.
def _glob2(dirname, pattern, dironly):
assert _isrecursive(pattern)
return [pattern[:0]] + list(_rlistdir(dirname, dironly))
# If dironly is false, yields all file names inside a directory.
# If dironly is true, yields only directory names.
def _iterdir(dirname, dironly):
if not dirname:
if isinstance(dirname, bytes):
dirname = bytes(os.curdir, 'ASCII')
else:
dirname = os.curdir
try:
for entry in os.listdir(dirname):
yield entry
except OSError:
return
# Recursively yields relative pathnames inside a literal directory.
def _rlistdir(dirname, dironly):
if not os.path.islink(dirname):
names = list(_iterdir(dirname, dironly))
for x in names:
yield x
path = os.path.join(dirname, x) if dirname else x
for y in _rlistdir(path, dironly):
yield os.path.join(x, y)
magic_check = re.compile('([*?[])')
magic_check_bytes = re.compile(b'([*?[])')
def has_magic(s):
if isinstance(s, bytes):
match = magic_check_bytes.search(s)
else:
match = magic_check.search(s)
return match is not None
def _ishidden(path):
return path[0] in ('.', b'.'[0])
def _isrecursive(pattern):
if isinstance(pattern, bytes):
return pattern == b'**'
else:
return pattern == '**'
def escape(pathname):
"""Escape all special characters.
"""
# Escaping is done by wrapping any of "*?[" between square brackets.
# Metacharacters do not work in the drive part and shouldn't be escaped.
drive, pathname = os.path.splitdrive(pathname)
if isinstance(pathname, bytes):
pathname = magic_check_bytes.sub(br'[\1]', pathname)
else:
pathname = magic_check.sub(r'[\1]', pathname)
return drive + pathname
Tools/build/JenkinsScripts/build/utils/packaging_version.py
-75
View File
@@ -1,75 +0,0 @@
"""
All or portions of this file Copyright (c) Amazon.com, Inc. or its affiliates or
its licensors.
For complete copyright and license terms please see the LICENSE at the root of this
distribution (the "License"). All use of this software is governed by the License,
or, if provided, by the license below or the license accompanying this file. Do not
remove or modify any license notices. This file is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
"""
import os
import sys
from validate_ly_version import _read_version_from_branch_spec
from argparse import ArgumentParser
def main(args):
waf_branch_spec_file_directory = os.path.join(os.environ['WORKSPACE'], 'dev')
waf_branch_spec_file_name = 'waf_branch_spec.py'
if not os.path.exists(os.path.join(waf_branch_spec_file_directory, waf_branch_spec_file_name)):
raise Exception("Invalid workspace directory: {}".format(waf_branch_spec_file_directory))
waf_branch_spec_version = _read_version_from_branch_spec(waf_branch_spec_file_directory, waf_branch_spec_file_name)
if waf_branch_spec_version is None:
raise Exception("Unable to read branch spec version from {}.".format(waf_branch_spec_file_name))
if args.version:
waf_branch_spec_version = args.version
if waf_branch_spec_version=='0.0.0.0' and not args.allow_unversioned:
raise Exception('Version "{}" is invalid. Please specify a valid, non-zero LUMBERYARD_VERSION in {}.'.format(
waf_branch_spec_version,
os.path.join(waf_branch_spec_file_directory, waf_branch_spec_file_name)
))
versions = waf_branch_spec_version.split('.')
if len(versions) != 4:
raise Exception("Invalid branch spec version '{}'. Must use format 'X.X.X.X'".format(waf_branch_spec_version))
major_version = versions[0]
minor_version = versions[1]
env_inject_file_path = os.path.join(os.environ['WORKSPACE'], os.environ['ENV_INJECT_FILE'])
print major_version
print minor_version
with open(env_inject_file_path, 'w') as env_inject_file:
env_inject_file.write('MAJOR_VERSION={}\n'.format(major_version))
env_inject_file.write('MINOR_VERSION={}\n'.format(minor_version))
def check_env(*vars):
missing = []
for var in vars:
if var not in os.environ:
missing += (var,)
if missing:
raise Exception("Missing one or more environment variables: {}".format(", ".join(missing)))
if __name__ == "__main__":
parser = ArgumentParser()
parser.add_argument('--allow-unversioned', default=False, action='store_true',
help="Allow version '0.0.0.0'. Default is to fail if an invalid version is found")
parser.add_argument('--version', type=str,
help="Manually specify version to use, instead of scanning dev root.")
args = parser.parse_args()
check_env("WORKSPACE", "ENV_INJECT_FILE")
main(args)
Tools/build/JenkinsScripts/build/utils/scrubbing_test.py
-212
View File
@@ -1,212 +0,0 @@
#
# All or portions of this file Copyright (c) Amazon.com, Inc. or its affiliates or
# its licensors.
#
# For complete copyright and license terms please see the LICENSE at the root of this
# distribution (the "License"). All use of this software is governed by the License,
# or, if provided, by the license below or the license accompanying this file. Do not
# remove or modify any license notices. This file is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
#
import requests
from requests.auth import HTTPBasicAuth
from P4 import P4, P4Exception
from util import *
from zipfile import ZipFile
from download_from_s3 import s3_download_file, get_client
from botocore.exceptions import ClientError
from upload_to_s3 import s3_upload_file
import os
import shutil
import json
import urllib
PACKAGE_NAME_REGEX = r'^lumberyard-\d\.\d-\d+-\w+-\d+\.(zip|tgz)$'
P4_USER = 'lybuilder'
BUCKET = 'ly-scrubbing-test'
# Scrubber will fail if any of these files is missing, copy these files to scrubbing workspace before the test
SCRUBBING_REQUIRED_FILES = [
]
try:
JENKINS_USERNAME = os.environ['JENKINS_USERNAME']
JENKINS_API_TOKEN = os.environ['JENKINS_API_TOKEN']
JENKINS_SERVER = os.environ['JENKINS_URL']
P4_PORT = os.environ['ENV_P4_PORT']
JOB_NAME = os.environ['JOB_NAME']
BUILD_NUMBER = int(os.environ['BUILD_NUMBER'])
WORKSPACE = os.environ['WORKSPACE']
SCRUBBING_WORKSPACE = os.environ['SCRUBBING_WORKSPACE']
except KeyError:
error('This script has to run on Jenkins')
class File:
def __init__(self, path, action):
self.path = path
self.action = action
# Get the changelist numbers that trigger the build
def get_changelist_numbers():
changelist_numbers = []
changeset = []
headers = {'Content-type': 'application/json', 'Accept': 'application/json'}
try:
res = requests.get('{}/job/{}/{}/api/json'.format(JENKINS_SERVER, JOB_NAME, BUILD_NUMBER),
auth=HTTPBasicAuth(JENKINS_USERNAME, JENKINS_API_TOKEN), headers=headers, verify=False)
res = json.loads(res.content)
changeset = res.get('changeSet').get('items')
except:
print 'Error: Failed to get changes from build {} in job {}'.format(BUILD_NUMBER, JOB_NAME)
for item in changeset:
changelist_numbers.append(item.get('changeNumber'))
return changelist_numbers
# Get file list and actions that trigger the Jenkins job
def get_files():
p4 = P4()
p4.port = P4_PORT
p4.user = P4_USER
p4.connect()
files = []
changelist_numbers = get_changelist_numbers()
for changelist_number in changelist_numbers:
cmd = ['describe', '-s', changelist_number]
try:
res = p4.run(cmd)[0]
file_list = res.get('depotFile')
actions = res.get('action')
for action, file_path in zip(actions, file_list):
# P4 returns file paths that are url encoded
file_path = urllib.unquote(file_path).decode("utf8")
# Ignore files which are not in dev
p = file_path.find('dev')
if p != -1:
files.append(File(file_path[p:], action))
except P4Exception:
error('Internal error, please contact Build System')
return files
def copy_file(src, dst, overwrite=False):
if os.path.exists(dst) and not overwrite:
return
print 'Copying file from {} to {}'.format(src, dst)
dest_file_dir = os.path.dirname(dst)
if not os.path.exists(dest_file_dir):
os.makedirs(dest_file_dir)
shutil.copyfile(src, dst)
# Run scrubbing scripts
def scrub():
print 'Perform the Code Scrubbing'
scrubber_path = os.path.join(WORKSPACE, 'dev/Tools/build/JenkinsScripts/distribution/scrubbing/scrub_all.py')
scrub_params = ["-p", "-d", "-o"]
# Scrub code
args = ['python', scrubber_path, '-p', '-d', '-o', os.path.join(SCRUBBING_WORKSPACE, 'dev/Code'), os.path.join(SCRUBBING_WORKSPACE, 'dev')]
return_code = safe_execute_system_call(args)
if return_code != 0:
print 'ERROR: Code scrubbing failed.'
return False
print 'Code scrubbing complete successfully.'
return True
# Run scrubbing validator
def validate():
# Run validator
print 'Running validator'
validator_platforms = ["provo", "salem", "jasper"]
success = True
for validator_platform in validator_platforms:
validator_path = os.path.join(WORKSPACE, 'dev/Tools/build/JenkinsScripts/distribution/scrubbing/validator.py')
args = ['python', validator_path, '-p', validator_platform, os.path.join(SCRUBBING_WORKSPACE, 'dev')]
if safe_execute_system_call(args):
success = False
if not success:
print 'ERROR: Scrubbing validator failed.'
return False
print 'Scrubbing validator complete successfully.'
return True
def scrubbing_test():
if os.path.exists(SCRUBBING_WORKSPACE):
os.system('rmdir /s /q \"{}\"'.format(SCRUBBING_WORKSPACE))
os.mkdir(SCRUBBING_WORKSPACE)
client = get_client('s3')
zip_name = '{}.zip'.format(JOB_NAME)
# Check if zipfile exists in S3 bucket
try:
client.head_object(Bucket=BUCKET, Key=zip_name)
except ClientError as e:
if e.response['Error']['Code'] == '404':
print 'No previous zipfile found in S3 bucket {}'.format(BUCKET)
else:
raise
else:
# Download the zipfile from S3 bucket if the zipfile exists
if not s3_download_file(client, SCRUBBING_WORKSPACE, zip_name, BUCKET, max_retry=3):
warn('Failed to download {} from S3 bucket {}'.format(zip_name, BUCKET))
# Unzip the zipfile to SCRUBBING_WORKSPACE
zip_path = os.path.join(SCRUBBING_WORKSPACE, zip_name)
if os.path.exists(zip_path):
zip_file = ZipFile(os.path.join(SCRUBBING_WORKSPACE, zip_name), 'r')
zip_file.extractall(SCRUBBING_WORKSPACE)
zip_file.close()
# Copy scrubbing required files to SCRUBBING_WORKSPACE, no overwrite
for file in SCRUBBING_REQUIRED_FILES:
src_file = os.path.join(WORKSPACE, file)
dst_file = os.path.join(SCRUBBING_WORKSPACE, file)
copy_file(src_file, dst_file)
# Get file list and actions that trigger the Jenkins job
files = get_files()
# Copy or delete each file in SCRUBBING_WORKSPACE
for f in files:
dst_file = os.path.join(SCRUBBING_WORKSPACE, f.path)
if 'delete' in f.action:
if os.path.exists(dst_file):
print 'Deleting {}'.format(dst_file)
os.remove(dst_file)
else:
src_file = os.path.join(WORKSPACE, f.path)
copy_file(src_file, dst_file, overwrite=True)
# Backup the unmodified files and run scrubber and validator
backup_path = os.path.join(SCRUBBING_WORKSPACE, 'backup')
scrubbing_dev = os.path.join(SCRUBBING_WORKSPACE, 'dev')
success = True
if os.path.exists(scrubbing_dev):
shutil.copytree(scrubbing_dev, os.path.join(backup_path, 'dev'))
success = scrub() and validate()
if success:
# Delete zipfile from S3 if validator run successfully
try:
print 'Deleting {} from bucket {}'.format(zip_name, BUCKET)
client.delete_object(Bucket=BUCKET, Key=zip_name)
except:
warn('Failed to delete {} from bucket {}'.format(zip_name, BUCKET))
else:
# Upload backup files to S3 bucket
if os.path.exists(backup_path):
zip_path = os.path.join(SCRUBBING_WORKSPACE, JOB_NAME)
shutil.make_archive(zip_path, 'zip', backup_path)
if not s3_upload_file(client, SCRUBBING_WORKSPACE, zip_name, BUCKET, max_retry=3):
error('Failed to upload {} to S3 bucket {}'.format(zip_name, BUCKET))
exit(1)
if __name__ == "__main__":
scrubbing_test()
Tools/build/JenkinsScripts/build/utils/update_bootstrap_cfg.py
-82
View File
@@ -1,82 +0,0 @@
"""
All or portions of this file Copyright (c) Amazon.com, Inc. or its affiliates or
its licensors.
For complete copyright and license terms please see the LICENSE at the root of this
distribution (the "License"). All use of this software is governed by the License,
or, if provided, by the license below or the license accompanying this file. Do not
remove or modify any license notices. This file is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
"""
'''
This script is to update the configuration in dev/bootstrap.cfg
Usage: python update_bootstrap_cfg.py --bootstrap_cfg file_path --replace key1=value1,key2=value2
'''
from optparse import OptionParser
import os
import stat
def update_bootstrap_cfg(file, replace_values):
try:
with open(file, 'r') as bootstrap_cfg:
content = bootstrap_cfg.read()
except:
error('Cannot read file {}'.format(file))
content = content.split('\n')
new_content = []
for line in content:
if not line.startswith('--'):
strs = line.split('=')
if len(strs):
key = strs[0].strip(' ')
if key in replace_values:
line = '{}={}'.format(key, replace_values[key])
new_content.append(line)
try:
with open(file, 'w') as out:
out.write('\n'.join(new_content))
except:
error('Cannot write to file {}'.format(file))
print '{} updated with value {}'.format(file, replace_values)
def error(msg):
print msg
exit(1)
def parse_args():
parser = OptionParser()
parser.add_option("--bootstrap_cfg", dest="bootstrap_cfg", default=None, help="File path of bootstrap.cfg to be updated.")
parser.add_option("--replace", dest="replace", default=None, help="Target platform to package")
(options, args) = parser.parse_args()
bootstrap_cfg = options.bootstrap_cfg
replace = options.replace
if not bootstrap_cfg:
error('bootstrap.cfg is not specified.')
if not os.path.isfile(bootstrap_cfg):
error('File {} not found.'.format(bootstrap_cfg))
replace_values = {}
if replace:
try:
replace = replace.split(',')
for r in replace:
r = r.split('=')
key = r[0].strip(' ')
value = r[1].strip(' ')
replace_values[key] = value
except IndexError:
error('Please check the format of argument --replace.')
return bootstrap_cfg, replace_values
if __name__ == "__main__":
(file, replace_values) = parse_args()
update_bootstrap_cfg(file, replace_values)
Tools/build/JenkinsScripts/build/utils/upload_benchmarks.py
-174
View File
@@ -1,174 +0,0 @@
#!/usr/bin/env python
#
# All or portions of this file Copyright (c) Amazon.com, Inc. or its affiliates or
# its licensors.
#
# For complete copyright and license terms please see the LICENSE at the root of this
# distribution (the "License"). All use of this software is governed by the License,
# or, if provided, by the license below or the license accompanying this file. Do not
# remove or modify any license notices. This file is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
#
import argparse
from datetime import datetime, timezone
import hashlib
import os
import pathlib
import platform
import re
import sys
import subprocess
import zipfile
'''
Creates zip file of <BuildDir>/BenchmarkResults folder and sends the the zip via email to team email
'''
def compute_sha256(input_filepath):
'''
Computes a SHA-2 hash using a digest that is 256 bits
Args:
input_filepath: File whose content will be hashed using the SHA-2 hash function
Returns:
bytes: byte array containing hash digest in hex
'''
hasher = hashlib.sha256()
hash_result = None
CHUNK_SIZE = 128 * (1 << 10) # Chunk Size for Sha256 hashing reads file in chunks of 128 KiB
with open(input_filepath, 'rb') as hash_file:
buf = hash_file.read(CHUNK_SIZE)
hasher.update(buf)
hash_result = hasher.hexdigest()
return hash_result
def create_sha256sums_file(input_filepath):
'''
Create a sha256sums file from the contents of the input_filepath
The sha256sums file will be named using the input_filepath path with an added
extension of .sha256sums
Args:
input_filepath: File whose content will be hashed using the SHA-2 hash function
Returns:
string: Path to sha256sums file
'''
sha256_hash = compute_sha256(input_filepath)
if not sha256_hash:
print(f'Unable to compute sha256 hash for file {input_filepath}')
return None
hash_filepath = f'{input_filepath}.sha256sums'
with open(hash_filepath, "wb") as archive_hash_file:
new_hash_contents = f'{sha256_hash} *{os.path.basename(input_filepath)}\n'
archive_hash_file.write(new_hash_contents.encode("utf8"))
return hash_filepath
def get_files_to_archive(base_dir, regex):
'''
Gathers list of filepaths to add to archive file
Files are looked up with the base directory and cross checked against the supplied regular expression
which acts as an inclusion filter
Args:
base_dir: Directory to scan for files
regex: Regular expression that is matched against each filename to determine if the file should be
added to the archive
'''
# Get all file names in base directory
with os.scandir(base_dir) as dir_entry:
filepaths = [pathlib.PurePath(entry.path) for entry in dir_entry if entry.is_file()]
# Get all file names matching the regular expression, those file will be added to zip archive
archive_files = [str(filepath) for filepath in filepaths if re.match(regex, filepath.as_posix())]
return archive_files
return None
def create_archive_file(archive_file_prefix, input_filepaths, base_dir):
'''
Creates a zip file using the supplied input files
LZMA compression is used by default for the zip file compression
Args:
archive_file_prefix: Prefix to use as the name of the zip file that should be created
input_filepaths: List of input file paths that will be added to zip file
base_dir: Directory which is used to create relative paths for each input file path from
'''
try:
zipfile_name = '{}-{:%Y%m%d_%H%M%S}.zip'.format(archive_file_prefix,datetime.now(timezone.utc))
# The lzma shared library isn't installed by default on Mac.
compression_type = zipfile.ZIP_LZMA if platform.system() != 'Darwin' else zipfile.ZIP_BZIP2
with zipfile.ZipFile(zipfile_name, mode='w', compression=compression_type) as benchmark_archive:
zipfile_name = benchmark_archive.filename
for input_filepath in input_filepaths:
# Make input files relative to base_dir when storing them as archived names
input_filepath_relpath = os.path.relpath(input_filepath, start=base_dir)
benchmark_archive.write(input_filepath, input_filepath_relpath)
except OSError as err:
print(f'Failed to write benchmark files to zip archive with error {err}')
sys.exit(1)
except RuntimeError as zip_err:
print(f'Runtime Error in zipfile module {zip_err}')
sys.exit(1)
return zipfile_name
def upload_to_s3(upload_script_path, base_dir, path_regex, bucket, key_prefix):
'''
Uploads files which located within the base directory using the upload_to_s3.py script
Args:
base_dir: The directory to pass as the --base-dir value to the upload_to_s3.py script
path_regex: The regular expression to pass to the upload_to_s3.py script --file-regex parameter
bucket: The s3 bucket to use for the --bucket argument for upload_to_s3.py
key_prefix: The prefix to store the uploaded files to within the s3 bucket,
It is passed --key-prefix argument to upload_to_s3.py
'''
try:
subprocess.run(['python', upload_script_path, '--base_dir',
base_dir, '--file_regex', path_regex,
'--bucket', bucket, '--key_prefix', key_prefix],
check=True)
except subprocess.CalledProcessError as err:
print(f'{upload_script_path} failed with error {err}')
sys.exit(1)
def upload_benchmarks(args):
'''
Main function responsible for determine which files to add to the output zip file and uploading
the results to s3
Args:
args: Parse argument list of python command line parameters using the argparse module
'''
files_to_archive = get_files_to_archive(args.base_dir, args.file_regex)
archive_zip_path = create_archive_file(args.output_prefix, files_to_archive, args.base_dir)
# Create Sha256sum hash file of zip
create_sha256sums_file(archive_zip_path)
upload_dir = str(pathlib.Path(archive_zip_path).parent)
upload_regex = fr'{pathlib.Path(archive_zip_path).name}.*'
upload_to_s3(args.upload_to_s3_script_path, upload_dir, upload_regex, args.bucket, args.key_prefix)
def parse_args():
cur_dir = os.path.dirname(os.path.abspath(__file__))
parser = argparse.ArgumentParser()
parser.add_argument("--base_dir", default=os.getcwd(), help="Base directory to files which should be archived, If not given, then current directory is used.")
parser.add_argument("--upload-to-s3-script-path", default=os.path.join(cur_dir, 'upload_to_s3.py'), help="Path to upload_to_s3.py script. Script is used for uploading benchmarks to s3")
parser.add_argument("--file_regex", default=r'.*BenchmarkResults/.+\.json', help="Regular expression that used to match file names to archive.")
parser.add_argument("-o", "--output-prefix", default='benchmarks_results', help="Prefix to use to construct the name of the zip file where the benchmark results are zipped."
" A timestamp will be added to the end of filename")
parser.add_argument("--bucket", dest="bucket", default='ly-jenkins-cmake-benchmarks', help="S3 bucket the files are uploaded to.")
parser.add_argument("-k", "--key_prefix", default='user_build', dest="key_prefix", help="Object key prefix.")
args = parser.parse_args()
return args
if __name__ == '__main__':
args = parse_args();
upload_benchmarks(args)
Tools/build/JenkinsScripts/build/utils/upload_metrics_to_kinesis.py
-183
View File
@@ -1,183 +0,0 @@
########################################################################################
# All or portions of this file Copyright (c) Amazon.com, Inc. or its affiliates or
# its licensors.
#
# For complete copyright and license terms please see the LICENSE at the root of this
# distribution (the "License"). All use of this software is governed by the License,
# or, if provided, by the license below or the license accompanying this file. Do not
# remove or modify any license notices. This file is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
#
#
# Original file Copyright Crytek GMBH or its affiliates, used under license.
#
########################################################################################
import ast
import boto3
from botocore.exceptions import ClientError
from datetime import datetime
import logging
import os
import shutil
import sys
import time
import traceback
import urllib2
import uuid
KINESIS_STREAM_NAME = 'lumberyard-metrics-stream'
KINESIS_MAX_RECORD_SIZE = 1048576 # 1 MB
S3_BACKUP_BUCKET = 'infrastructure-build-metrics-backup'
IAM_ROLE_NAME = 'ec2-jenkins-node'
LOG_FILE_NAME = 'kinesis_upload.log'
MAX_RECORD_SIZE = KINESIS_MAX_RECORD_SIZE - 4 # to account for version header
MAX_RETRIES = 5
RETRY_EXCEPTIONS = ('ProvisionedThroughputExceededException',
'ThrottlingException')
# truncate the log file, eventually we need to send the logs to cloudwatch logs
with open(LOG_FILE_NAME, 'w'):
pass
logger = logging.getLogger('KinesisUploader')
fileHdlr = logging.FileHandler(LOG_FILE_NAME)
# uncomment this line and the two below to have logs go to stdout for debugging purposes
#streamHdlr = logging.StreamHandler(sys.stdout)
formatter = logging.Formatter('%(asctime)s %(levelname)s %(message)s')
fileHdlr.setFormatter(formatter)
#streamHdlr.setFormatter(formatter)
logger.addHandler(fileHdlr)
#logger.addHandler(streamHdlr)
logger.setLevel(logging.DEBUG)
def backup_file_to_s3(s3_client, bucket_name, file_location, s3_file_name):
try:
s3_client.meta.client.upload_file(file_location, bucket_name, s3_file_name)
os.remove(file_location)
except:
logger.error('Failed to upload backup file to S3. This is non-fatal!')
# logger.error(traceback.print_exc())
def get_iam_role_credentials(role_name):
security_metadata = None
try:
response = urllib2.urlopen(
'http://169.254.169.254/latest/meta-data/iam/security-credentials/{0}'.format(role_name)).read()
security_metadata = ast.literal_eval(response)
except:
logger.error('Unable to get iam role credentials')
logger.error(traceback.print_exc())
return security_metadata
def splitFileByRecord(stream, maxSize):
version = 1
# currently using random GUID for partition key, but in the future we may want to partition by some build id
# or by build host
partition_key = uuid.uuid4()
entry_size = 0
put_entries = []
current_entry = ''
for line in stream:
line_size_in_bytes = len(line.encode('utf-8'))
entry_size = entry_size + line_size_in_bytes
if (entry_size > MAX_RECORD_SIZE):
put_entries.append({
'Data': str(version) + '\n' + str(current_entry),
'PartitionKey': str(partition_key)
})
current_entry = line
entry_size = line_size_in_bytes
else:
current_entry = current_entry + line
if current_entry:
put_entries.append({
'Data': str(version) + '\n' + str(current_entry),
'PartitionKey': str(partition_key)
})
return put_entries
def main():
credentials = get_iam_role_credentials(IAM_ROLE_NAME)
aws_access_key_id = None
aws_secret_access_key = None
aws_session_token = None
if credentials is not None:
keys = ['AccessKeyId', 'SecretAccessKey', 'Token']
for key in keys:
if key not in credentials:
logger.error('Unable to find {0} in get_iam_role_credentials response {1}'.format(key, credentials))
return
aws_access_key_id = credentials['AccessKeyId']
aws_secret_access_key = credentials['SecretAccessKey']
aws_session_token = credentials['Token']
kinesis_client = boto3.client('kinesis', region_name='us-west-2', aws_access_key_id=aws_access_key_id,
aws_secret_access_key=aws_secret_access_key, aws_session_token=aws_session_token)
s3_client = boto3.resource('s3', aws_access_key_id=aws_access_key_id, aws_secret_access_key=aws_secret_access_key,
aws_session_token=aws_session_token)
file_location = sys.argv[1]
filename = os.path.basename(file_location)
backup_file_location = file_location + '.bak'
try:
if os.path.isfile(backup_file_location):
logger.info('Found pre-existing backup file. Uploading to S3.')
backup_file_to_s3(s3_client, S3_BACKUP_BUCKET, backup_file_location,
'{0}.{1}'.format(filename, datetime.now().isoformat()))
if os.path.isfile(file_location):
shutil.copyfile(file_location, backup_file_location)
backup_file_to_s3(s3_client, S3_BACKUP_BUCKET, backup_file_location,
'{0}.{1}'.format(filename, datetime.now().isoformat()))
logger.info('Opening metrics file {0}'.format(file_location))
with open(file_location, 'r+') as f:
records = splitFileByRecord(f, MAX_RECORD_SIZE)
i = 0
retries = 0
while i < len(records):
record = records[i]
try:
logger.info('Uploading {0} bytes of metrics to Kinesis...'.format(len(record)))
kinesis_client.put_record(StreamName=KINESIS_STREAM_NAME,
Data=record['Data'],
PartitionKey=record['PartitionKey'])
retries = 0
except ClientError as ex:
if ex.response['Error']['Code'] not in RETRY_EXCEPTIONS:
raise
sleep_time = 2 ** retries
logger.warn('Request throttled by Kinesis, '
'sleeping and retrying in {0} seconds'.format(2 ** retries))
time.sleep(sleep_time)
retries += 1
i -= 1
i += 1
f.truncate(0)
except:
logger.error(traceback.print_exc())
if __name__ == '__main__':
main()
Tools/build/JenkinsScripts/build/utils/upload_to_s3.py
-107
View File
@@ -1,107 +0,0 @@
#
# All or portions of this file Copyright (c) Amazon.com, Inc. or its affiliates or
# its licensors.
#
# For complete copyright and license terms please see the LICENSE at the root of this
# distribution (the "License"). All use of this software is governed by the License,
# or, if provided, by the license below or the license accompanying this file. Do not
# remove or modify any license notices. This file is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
#
'''
Usage:
Use EC2 role to upload all .zip and .MD5 files in %WORKSPACE% folder to bucket ly-packages-mainline:
python upload_to_s3.py --base_dir %WORKSPACE% --file_regex "(.*zip$|.*MD5$)" --bucket ly-packages-mainline
Use profile to upload all .zip and .MD5 files in %WORKSPACE% folder to bucket ly-packages-mainline:
python upload_to_s3.py --base_dir %WORKSPACE% --profile profile --file_regex "(.*zip$|.*MD5$)" --bucket ly-packages-mainline
'''
import os
import re
import json
import boto3
from optparse import OptionParser
from util import error
def parse_args():
parser = OptionParser()
parser.add_option("--base_dir", dest="base_dir", default=os.getcwd(), help="Base directory to upload files, If not given, then current directory is used.")
parser.add_option("--file_regex", dest="file_regex", default=None, help="Regular expression that used to match file names to upload.")
parser.add_option("--profile", dest="profile", default=None, help="The name of a profile to use. If not given, then the default profile is used.")
parser.add_option("--bucket", dest="bucket", default=None, help="S3 bucket the files are uploaded to.")
parser.add_option("--key_prefix", dest="key_prefix", default='', help="Object key prefix.")
'''
ExtraArgs used to call s3.upload_file(), should be in json format. extra_args key must be one of: ACL, CacheControl, ContentDisposition, ContentEncoding, ContentLanguage, ContentType, Expires,
GrantFullControl, GrantRead, GrantReadACP, GrantWriteACP, Metadata, RequestPayer, ServerSideEncryption, StorageClass,
SSECustomerAlgorithm, SSECustomerKey, SSECustomerKeyMD5, SSEKMSKeyId, WebsiteRedirectLocation
'''
parser.add_option("--extra_args", dest="extra_args", default=None, help="Additional parameters used to upload file.")
parser.add_option("--max_retry", dest="max_retry", default=1, help="Maximum retry times to upload file.")
(options, args) = parser.parse_args()
if not os.path.isdir(options.base_dir):
error('{} is not a valid directory'.format(options.base_dir))
if not options.file_regex:
error('Use --file_regex to specify regular expression that used to match file names to upload.')
if not options.bucket:
error('Use --bucket to specify bucket that the files are uploaded to.')
return options
def get_client(service_name, profile_name):
session = boto3.session.Session(profile_name=profile_name)
client = session.client(service_name)
return client
def get_files_to_upload(base_dir, regex):
# Get all file names in base directory
files = [x for x in os.listdir(base_dir) if os.path.isfile(os.path.join(base_dir, x))]
# Get all file names matching the regular expression, those file will be uploaded to S3
files_to_upload = [x for x in files if re.match(regex, x)]
return files_to_upload
def s3_upload_file(client, base_dir, file, bucket, key_prefix=None, extra_args=None, max_retry=1):
print('Uploading file {} to bucket {}.'.format(file, bucket))
key = file if key_prefix is None else '{}/{}'.format(key_prefix, file)
for x in range(max_retry):
try:
client.upload_file(
os.path.join(base_dir, file), bucket, key,
ExtraArgs=extra_args
)
print('Upload succeeded')
return True
except Exception as err:
print('exception while uploading: {}'.format(err))
print('Retrying upload...')
print('Upload failed')
return False
if __name__ == "__main__":
options = parse_args()
client = get_client('s3', options.profile)
files_to_upload = get_files_to_upload(options.base_dir, options.file_regex)
extra_args = json.loads(options.extra_args) if options.extra_args else None
print('Uploading {} files to bucket {}.'.format(len(files_to_upload), options.bucket))
failure = []
success = []
for file in files_to_upload:
if not s3_upload_file(client, options.base_dir, file, options.bucket, options.key_prefix, extra_args, 2):
failure.append(file)
else:
success.append(file)
print('Upload finished.')
print('{} files are uploaded successfully:'.format(len(success)))
print('\n'.join(success))
if len(failure) > 0:
print('{} files failed to upload:'.format(len(failure)))
print('\n'.join(failure))
# Exit with error code 1 if any file is failed to upload
exit(1)
Tools/build/JenkinsScripts/build/utils/util.py
-65
View File
@@ -1,65 +0,0 @@
"""
All or portions of this file Copyright (c) Amazon.com, Inc. or its affiliates or
its licensors.
For complete copyright and license terms please see the LICENSE at the root of this
distribution (the "License"). All use of this software is governed by the License,
or, if provided, by the license below or the license accompanying this file. Do not
remove or modify any license notices. This file is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
"""
import json
import os
import re
import subprocess
class LyBuildError(Exception):
def __init__(self, message):
super(LyBuildError, self).__init__(message)
def __str__(self):
return str(self.message)
def ly_build_error(message):
raise LyBuildError(message)
def error(message):
print('Error: {}'.format(message))
exit(1)
# Exit with status code 0 means it won't fail the whole build process
def safe_exit_with_error(message):
print('Error: {}'.format(message))
exit(0)
def warn(message):
print('Warning: {}'.format(message))
def execute_system_call(command, **kwargs):
print('Executing subprocess.check_call({})'.format(command))
try:
subprocess.check_call(command, **kwargs)
except subprocess.CalledProcessError as e:
print(e.output)
error('Executing subprocess.check_call({}) failed with error {}'.format(command, e))
except FileNotFoundError as e:
error("File Not Found - Failed to call {} with error {}".format(command, e))
def safe_execute_system_call(command, **kwargs):
print('Executing subprocess.check_call({})'.format(command))
try:
subprocess.check_call(command, **kwargs)
except subprocess.CalledProcessError as e:
print(e.output)
warn('Executing subprocess.check_call({}) failed'.format(command))
return e.returncode
return 0
Tools/build/JenkinsScripts/distribution/AWS_PyTools/LyChecksum.py
-64
View File
@@ -1,64 +0,0 @@
"""
All or portions of this file Copyright (c) Amazon.com, Inc. or its affiliates or
its licensors.
For complete copyright and license terms please see the LICENSE at the root of this
distribution (the "License"). All use of this software is governed by the License,
or, if provided, by the license below or the license accompanying this file. Do not
remove or modify any license notices. This file is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
"""
import hashlib
import re
def generateFilesetChecksum(filePaths):
filesetHash = hashlib.sha512()
for filePath in filePaths:
updateHashWithFileChecksum(filePath, filesetHash)
return filesetHash
def getChecksumForSingleFile(filePath, openMode='rb'):
filesetHash = hashlib.sha512()
updateHashWithFileChecksum(filePath, filesetHash, openMode)
return filesetHash
def getMD5ChecksumForSingleFile(filePath, openMode='rb'):
filesetHash = hashlib.md5()
updateHashWithFileChecksum(filePath, filesetHash, openMode)
return filesetHash
def updateHashWithFileChecksum(filePath, filesetHash, openMode='rb'):
BLOCKSIZE = 65536
with open(filePath.strip('\n'), openMode) as file:
buf = file.read(BLOCKSIZE)
while len(buf) > 0:
filesetHash.update(buf)
buf = file.read(BLOCKSIZE)
def is_valid_hash_sha1(checksum):
# sha1 hashes are 40 hex characters long.
if len(checksum) is not 40:
return False
sha1_re = re.compile("(^[0-9A-Fa-f]{40}$)")
result = sha1_re.match(checksum)
if not result:
return False
return True
def is_valid_hash_sha512(checksum):
# sha512 hashes are 128 hex characters long.
if len(checksum) is not 128:
return False
sha512_re = re.compile("(^[0-9A-Fa-f]{128}$)")
result = sha512_re.match(checksum)
if not result:
return False
return True
Tools/build/JenkinsScripts/distribution/AWS_PyTools/LyCloudfrontOps.py
-76
View File
@@ -1,76 +0,0 @@
"""
All or portions of this file Copyright (c) Amazon.com, Inc. or its affiliates or
its licensors.
For complete copyright and license terms please see the LICENSE at the root of this
distribution (the "License"). All use of this software is governed by the License,
or, if provided, by the license below or the license accompanying this file. Do not
remove or modify any license notices. This file is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
"""
import os.path
import boto3
try:
import urllib.parse as urllib_compat
except:
import urlparse as urllib_compat
def getCloudfrontDistPath(uploadURL):
pathFromDomainName = urllib_compat.urlparse(uploadURL)[2]
# need to remove the first slash, otherwise it will create a nameless directory on S3
return pathFromDomainName[1:]
def getCloudfrontDistribution(cloudfrontUrl, awsCredentialProfileName):
session = boto3.session.Session(profile_name=awsCredentialProfileName)
cloudfront = session.client('cloudfront')
distributionList = cloudfront.list_distributions()
targetDistId = None
for distribution in distributionList["DistributionList"]["Items"]:
if distribution["DomainName"] == urllib_compat.urlparse(cloudfrontUrl)[1]:
targetDistId = distribution["Id"]
pass
assert (targetDistId is not None), "No distribution with the domain name {} found.".format(cloudfrontUrl)
targetDist = cloudfront.get_distribution(Id=targetDistId)
return targetDist
def getBucket(cloudfrontDistribution, awsCredentialProfileName):
bucketName = getBucketName(cloudfrontDistribution)
session = boto3.session.Session(profile_name=awsCredentialProfileName)
s3 = session.resource('s3')
return s3.Bucket(bucketName)
def getBucketName(cloudfrontDistribution):
s3Info = cloudfrontDistribution["Distribution"]["DistributionConfig"]["Origins"]["Items"][0]
bucketDomainName = s3Info["DomainName"]
return bucketDomainName.split('.')[0] # first part of the domain name is the bucket name
def buildBucketPath(cloudfrontUrl, cloudfrontDistribution):
s3Info = cloudfrontDistribution["Distribution"]["DistributionConfig"]["Origins"]["Items"][0]
originPath = s3Info["OriginPath"]
bucketPath = None
if originPath:
# Start originPath after the first character (presumed to be '/') to avoid nameless directory in S3.
bucketPath = str.format("{0}/{1}", originPath[1:], getCloudfrontDistPath(cloudfrontUrl))
else:
bucketPath = getCloudfrontDistPath(cloudfrontUrl)
return bucketPath
def uploadFileToCloudfrontURL(absFilePath, cloudfrontBaseUrl, awsCredentialProfileName, overwrite):
cloudfrontDist = getCloudfrontDistribution(cloudfrontBaseUrl, awsCredentialProfileName)
s3Bucket = getBucket(cloudfrontDist, awsCredentialProfileName)
s3BucketPath = buildBucketPath(cloudfrontBaseUrl, cloudfrontDist)
targetBucketPath = urllib_compat.urljoin(s3BucketPath, os.path.basename(absFilePath))
# Check if file already exists in the S3 bucket.
file_exists = len(list(s3Bucket.objects.filter(Prefix=targetBucketPath))) > 0
if not file_exists or overwrite:
s3Bucket.upload_file(absFilePath, targetBucketPath)
return targetBucketPath
Tools/build/JenkinsScripts/distribution/AWS_PyTools/__init__.py
-12
View File
@@ -1,12 +0,0 @@
"""
All or portions of this file Copyright (c) Amazon.com, Inc. or its affiliates or
its licensors.
For complete copyright and license terms please see the LICENSE at the root of this
distribution (the "License"). All use of this software is governed by the License,
or, if provided, by the license below or the license accompanying this file. Do not
remove or modify any license notices. This file is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
"""
Tools/build/JenkinsScripts/distribution/AWS_WAF_Updater/update_internal_whitelist.py
-303
View File
@@ -1,303 +0,0 @@
"""
All or portions of this file Copyright (c) Amazon.com, Inc. or its affiliates or
its licensors.
For complete copyright and license terms please see the LICENSE at the root of this
distribution (the "License"). All use of this software is governed by the License,
or, if provided, by the license below or the license accompanying this file. Do not
remove or modify any license notices. This file is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
"""
import argparse
import boto3
import hashlib
import json
import math
import netaddr
import time
import urllib2
# Order of operations
# 1) Get list of internal IPs from dogfish
# 2) Translate IP ranges into WAF-vaild CIDR Subnets
# 3) Get InternalIPWhitelist rule from WAF
# 4) Get all list of all IP Sets on that rule
# 5) Loop through all IP ranges in all IP sets
# a) If the IP address is not in the set of WAF-valid CIDR Subnets, remove it from the IPSet lists
# b) If the IP address is in the set of WAF-valid CIDR Subnets, remove it from the WAF-valid CIDR subnet list
# 6) Anything left in the WAF-valid CIDR Subnet list needs to be added to IP Sets
# a) create a new IP set if necessary (max 1000 ranges per IP Set)
# 7) Push updates to WAF
rule_name = "Internal_IP_Whitelist"
ip_set_basename = "Amazon_Internal_IPs"
# Maximum number of IP descriptors per IP Set
max_ranges_per_ip_set = 1000
# Maximum number of IP descriptors updates per call
max_ranges_per_update = 1000
ip_version = "IPV4"
def parse_args():
"""Handle argument parsing and validate destination folder exists before returning argparse args"""
parser = argparse.ArgumentParser(description='Use AWS CLI to update the Internal IP Whitelist in dev or prod')
parser.add_argument('-l', '--list', action='store_true',
help="List the WAF Valid IPs that the list of IPs translates to instead of running the update.")
parser.add_argument('-p', '--profile', default='default', help="The name of the AWS CLI profile to use.")
args = parser.parse_args()
return args
def translate_and_apply_to_waf(ip_ranges, args):
subnet_list = list()
for ip_range in ip_ranges:
subnet_list.extend(translate_range_to_subnet_list(ip_range))
session = boto3.Session(profile_name=args.profile)
waf = session.client('waf')
change_tokens = apply_updates_to_waf(subnet_list, waf)
# wait for every change to finish before ending
waiting_on_changes = True
while waiting_on_changes:
change_in_progress = False
time.sleep(.3)
for token in change_tokens:
status = waf.get_change_token_status(ChangeToken=token["ChangeToken"])
change_in_progress = change_in_progress and (status == "PENDING")
if change_in_progress:
break
waiting_on_changes = change_in_progress
return not waiting_on_changes
def apply_updates_to_waf(ip_list, waf):
white_list_rule = get_ip_whitelist_rule(waf)
ip_set_list, negate_conditions = get_ip_sets_from_rule(white_list_rule, waf)
ip_to_add = list(ip_list)
adds = list()
change_tokens = list()
# for each ip set, get the ip ranges in each descriptor and compare against the ip list
# remove duplicates from the list to add
for ip_set in ip_set_list:
for ip_range_descriptor in ip_set["IPSetDescriptors"]:
ip = netaddr.IPNetwork(ip_range_descriptor["Value"])
# if an ip in the list is already in an IPSet, then remove it from the list of stuff to add
if ip in ip_list:
ip_to_add.remove(ip)
# create add operations
for ip in ip_to_add:
adds.append(
{
'Action': 'INSERT',
'IPSetDescriptor': {
'Type': ip_version,
'Value': str(ip)
}
}
)
# make delete operations for anything in a set that isn't in the list of IPs, then populate the rest of the list
# with adds until we hit max of the IPSet
for ip_set in ip_set_list:
removes = list()
for ip_range_descriptor in ip_set["IPSetDescriptors"]:
ip = netaddr.IPNetwork(ip_range_descriptor["Value"])
if ip not in ip_list:
removes.append(
{
'Action': 'DELETE',
'IPSetDescriptor': ip_range_descriptor
}
)
# perform the updates to this set
perform_updates_to_existing_ipset(ip_set, removes, adds, waf, change_tokens)
# we've done all the removes, and filled all existing ip_sets with adds. if we have leftovers, we need to make a
# new ip_set and add it to the rule
if len(adds) > 0:
num_ip_sets = len(ip_set_list)
rule_updates = list()
while len(adds) > 0:
create_token = waf.get_change_token()
change_tokens.append(create_token)
ip_set = waf.create_ip_set(Name="{0}_{1}".format(ip_set_basename, num_ip_sets),
ChangeToken=create_token["ChangeToken"])["IPSet"]
num_ip_sets += 1
ip_set_id = ip_set["IPSetId"]
rule_updates.append(
{
'Action': 'INSERT',
'Predicate': {
'Negated': negate_conditions,
'Type': 'IPMatch',
'DataId': ip_set_id
}
}
)
# populate the new ip_set
batch = move_updates_to_batch(adds)
update_set_token = waf.get_change_token()
change_tokens.append(update_set_token)
waf.update_ip_set(IPSetId=ip_set_id,
ChangeToken=update_set_token["ChangeToken"],
Updates=batch)
# update the rule with the new ip_lists
update_rule_token = waf.get_change_token()
change_tokens.append(update_rule_token)
waf.update_rule(RuleId=white_list_rule["RuleId"],
ChangeToken=update_rule_token["ChangeToken"],
Updates=rule_updates)
return change_tokens
def perform_updates_to_existing_ipset(ip_set, removes, adds, waf, change_tokens):
# figure out how many adds can be done on this ip_set after all of the removes
space_remaining_in_set = max_ranges_per_ip_set - (len(ip_set["IPSetDescriptors"]) - len(removes))
# fill the list of updates to perform with removes and adds until the end result is either a full set or
# all operations have been performed.
updates = list(removes)
updates.extend(list(adds[0:space_remaining_in_set]))
adds[0:space_remaining_in_set] = []
# make batches of updates (max per batch = max_ranges_per_update)
update_batches = list()
while len(updates) > 0:
batch = move_updates_to_batch(updates)
update_batches.append(batch)
# submit an update set request for each batch
for batch in update_batches:
change_token = waf.get_change_token()
change_tokens.append(change_token)
waf.update_ip_set(IPSetId=ip_set["IPSetId"],
ChangeToken=change_token["ChangeToken"],
Updates=batch)
def move_updates_to_batch(original_list):
items_to_batch = min(len(original_list), max_ranges_per_update)
batch = list(original_list[0:items_to_batch])
original_list[0:items_to_batch] = []
return batch
def get_ip_whitelist_rule(waf_client):
rules_list = waf_client.list_rules(Limit=100)
whitelist_rule_id = None
for rule in rules_list["Rules"]:
if rule["Name"] == rule_name:
whitelist_rule_id = rule["RuleId"]
if whitelist_rule_id is None:
return None
return waf_client.get_rule(RuleId=whitelist_rule_id)["Rule"]
def get_ip_sets_from_rule(rule, waf_client):
ip_sets = list()
negate_conditions = False
for condition in rule["Predicates"]:
if condition["Type"] == 'IPMatch':
ip_set_id = condition["DataId"]
ip_set = waf_client.get_ip_set(IPSetId=ip_set_id)
if ip_set is not None and ip_set_basename in ip_set["IPSet"]["Name"]:
ip_sets.append(ip_set["IPSet"])
else:
print "No IPSet with ID {0}".format(ip_set_id)
negate_conditions = condition["Negated"]
return ip_sets, negate_conditions
def translate_range_to_subnet_list(ip_range):
"""
:param ip_range: The IP address + CIDR prefix (IPNetwork object)
:return: A list of all WAF-valid subnets that compose the given IP range
"""
prefix = ip_range.prefixlen
waf_valid_prefix = find_closest_waf_range(prefix)
return list(ip_range.subnet(waf_valid_prefix))
def find_closest_waf_range(cidr_prefix_length):
"""
AWS WAF only accepts CIDR ranges of /8, /16, /24, /32. figure out what the closest safe range we need to convert to.
This will always return a smaller range than what is passed in for security reasons.
:param cidr_prefix_length: The arbitrary CIDR range to convert to a WAF-valid range
:return: The closest WAF-valid range. i.e. if cidr_prefix_length = 18, this fuction will return 24
"""
multiple = math.trunc(cidr_prefix_length / 8)
return (multiple + 1) * 8 # needs to be 1 based to get accurate range
def list_all_waf_valid_subnets(ip_list):
num_ips = 0
for ip_range in ip_list:
subnets = translate_range_to_subnet_list(ip_range)
num_ips += len(subnets)
for ip in subnets:
print ip
return num_ips
def main():
# IPs taken from https://w.amazon.com/index.php/PublicIPRanges
ips = ["207.171.176.0/20", # SEA
"205.251.224.0/22",
"176.32.120.0/22",
"54.240.196.0/24",
"54.231.244.0/22",
"52.95.52.0/22",
"205.251.232.0/22", # PDX
"54.240.230.0/23",
"54.240.248.0/21",
"54.231.160.0/19",
"54.239.2.0/23",
"54.239.48.0/22",
"52.93.12.0/22",
"52.94.208.0/21",
"52.218.128.0/17",
"204.246.160.0/22", # SFO
"205.251.228.0/22",
"176.32.112.0/21",
"54.240.198.0/24",
"54.231.232.0/21",
"52.219.20.0/22",
"52.219.24.0/21"]
args = parse_args()
ip_objects = list(netaddr.IPNetwork(ip) for ip in ips)
if args.list:
print list_all_waf_valid_subnets(ip_objects)
else:
return translate_and_apply_to_waf(ip_objects, args)
if __name__ == '__main__':
main()
Tools/build/JenkinsScripts/distribution/Installer/BootstrapperLogo.png
-3
View File
@@ -1,3 +0,0 @@
version https://git-lfs.github.com/spec/v1
oid sha256:45c80fc02c64747d3eb8fe76d89979b573401b92482c27d8c95bd01df22c525f
size 6582
Tools/build/JenkinsScripts/distribution/Installer/BuildInstaller.py
-289
View File
@@ -1,289 +0,0 @@
#
# All or portions of this file Copyright (c) Amazon.com, Inc. or its affiliates or
# its licensors.
#
# For complete copyright and license terms please see the LICENSE at the root of this
# distribution (the "License"). All use of this software is governed by the License,
# or, if provided, by the license below or the license accompanying this file. Do not
# remove or modify any license notices. This file is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
#
import glob
import shutil
import InstallerParams
from Insignia import *
from InstallerArgs import *
from InstallerPackaging import *
from Light import *
from SignTool import *
# Per Installer:
# Heat the includes to build the fragment
# candle + light to build the installer / merge module / whatever
# Collect the results
# If Signing:
# Copy installer and cabs to a clean folder
# Sign the CAB files
# insignia the installer to update its cab file references
# Sign the installer
# For the Bootstrapper:
# Generate S3 links or whatever data we need to shove into the final installer.
# Generate a WXS file if we need to based on the above data
# candle + light generated WXS + pre-built WXS to build the installer (from
# the unsigned files or, if signing, the signed files of the installers)
# If Signing:
# Copy bootstrapper and cabs to a clean folder
# Detach the burn engine from the bootstrapper, and sign it
# Reattach the burn engine to the bootstrapper, and sign the bootstrapper
# After completion, clean up temp files if necessary.
class OperationMode:
StepCounting, BuildInstaller = range(2)
# It's good practice to define an init for a class in Python, but this
# class only exists to serve as an enum for the operation mode.
# Defining the __init__ with just a pass is a way to stub out this function so
# Python IDEs don't get upset that there is a class with no init.
def __init__(self):
pass
args = createArgs()
try:
params = InstallerParams.InstallerParams(args)
validateArgs(args, params)
except InstallerParams.InstallerParamError as error:
raise Exception('Installer Params failed to be created with error:\n{}'.format(error))
# Tracking this like this to simplify calls to performStep.
# Otherwise, it has to look like this:
# def performStep(mode, stepsTaken, maxSteps, message, operation, *operationArgs):
# return stepsTaken+1, result
# and calls to performStep have to also capture the stepsTaken.
stepsTaken = 0
maxSteps = 0
operationMode = OperationMode.StepCounting
# The goal of authoring this function is to simplify calls in, which reduces friction when using it.
# Call this when you want to only call a function during full operational mode, and not during step counting.
# This will use the function's name as the message for the printout.
def performStep(operation, *operationArgs):
return performStepWithMessage(operation.__name__, operation, *operationArgs)
# Call when you want a message that does not match the function's name.
def performStepWithMessage(message, operation, *operationArgs):
global stepsTaken
result = None
if operationMode == OperationMode.BuildInstaller:
printProgress(message, stepsTaken, maxSteps)
result = operation(*operationArgs)
stepsTaken += 1
return result
# Call when you want to handle branching yourself. Can be used to trigger a branch for the operation mode.
def describeStep(message):
global stepsTaken
if operationMode == OperationMode.BuildInstaller:
printProgress(message, stepsTaken, maxSteps)
stepsTaken += 1
def buildInstaller():
global stepsTaken
stepsTaken = 0
# CREATE PACKAGES
if not params.skipMsiAndCabCreation:
performStep(createThirdPartyPackages, args, params)
performStep(createDevPackage, args, params)
performStep(createRootPackage, args, params)
else:
describeStep("Skipping Create Packages (MSIs and Cabs) step, re-using existing packages.")
if not params.doSigning or not args.bootstrapOnly:
params.msiFileNameList = performStepWithMessage("Gathering MSIs",
get_file_names_in_directory,
params.intermediateInstallerPath,
".msi")
params.cabFileNameList = performStepWithMessage("Gathering CABs",
get_file_names_in_directory,
params.intermediateInstallerPath,
".cab")
# SIGN CABs AND MSIs
if params.doSigning and not args.bootstrapOnly:
# we don't want to modify the original metrics exe, so copy it to where the
# other clean files are, and update the path to it
if params.metricsPath is not params.intermediateInstallerPath:
describeStep("Copying metrics to signing path")
if operationMode == OperationMode.BuildInstaller:
params.metricsPath = params.intermediateInstallerPath
safe_shutil_file_copy(params.fullPathToMetrics, os.path.join(params.metricsPath, params.metricsExe))
# copy the clean cab and msi files to a new directory to sign them.
if params.installerPath is not params.intermediateInstallerPath:
describeStep("Copying clean MSI, CAB, and EXE files to signing path")
if operationMode == OperationMode.BuildInstaller:
if os.path.exists(params.installerPath):
verbose_print(args.verbose, "Removing old files from signing path.")
shutil.rmtree(params.installerPath)
shutil.copytree(params.intermediateInstallerPath, params.installerPath)
# don't need the wixpdb files in the signing folder, so get rid of them
for filepath in glob.glob(os.path.join(params.installerPath, "*.wixpdb")):
os.remove(filepath)
# Sign the Cab files, verify signing was successful
performStep(signtoolSignAndVerifyFiles,
params.cabFileNameList,
params.installerPath,
params.intermediateInstallerPath,
params.signingType,
args.timestampServer,
args.verbose)
# Run Insignia on the MSI files
performStep(insigniaMSIs,
params.installerPath,
args.verbose,
params.msiFileNameList)
# Sign the MSIs, verify signing was successful
performStepWithMessage("Signing and verifying MSIs",
signtoolSignAndVerifyFiles,
params.msiFileNameList,
params.installerPath,
params.intermediateInstallerPath,
params.signingType,
args.timestampServer,
args.verbose)
# Sign the Metrics executable
performStepWithMessage("Signing and verifying metrics.exe",
signtoolSignAndVerifyFile,
params.metricsExe,
params.installerPath,
params.intermediateInstallerPath,
params.signingType,
args.timestampServer,
args.verbose)
# make sure that the bootstrapper will get the metrics exe from the right place
params.metricsPath = params.installerPath
# CREATE BOOTSTRAP
packageNameList = ""
if operationMode is OperationMode.BuildInstaller:
for msiName in params.msiFileNameList:
packageName = os.path.splitext(msiName)[0]
packageNameList += '{};'.format(packageName)
# Remove the last semi-colon from the list.
packageNameList = packageNameList[:-1]
# CANDLE BOOTSTRAP
success = performStep(candleBootstrap,
params.bootstrapWixObjDir,
params.installerPath,
packageNameList,
"LumberyardBootstrapper.wxs Redistributables.wxs",
args.hostURL,
args.verbose,
args.lyVersion,
params.metricsPath,
params.metricsExe,
params.pathTo2015Thru2019Redist,
params.redist2015Thru2019Exe,
create_id('LumberyardBootstrapper', 'BOOTSTRAPPER', args.lyVersion, args.buildId))
assert (operationMode is not OperationMode.BuildInstaller or success == 0), \
"Failed to generate wixobj file for bootstrapper."
# LIGHT BOOTSTRAP
success = performStep(lightBootstrap,
params.bootstrapOutputPath,
os.path.join(params.bootstrapWixObjDir, "*.wixobj"),
args.verbose,
args.cabCachePath)
assert (operationMode is not OperationMode.BuildInstaller or success == 0), \
"Failed to generate executable file for bootstrapper."
# SIGN ENGINE AND BOOTSTRAPPER
if params.doSigning:
# make sure the c++ redist gets copied from the temp directory to the actual
# installer output path with the bootstrapper.
if operationMode == OperationMode.BuildInstaller:
safe_shutil_file_copy(os.path.join(params.tempBootstrapOutputDir, params.redist2015Thru2019Exe),
os.path.join(params.installerPath, params.redist2015Thru2019Exe))
unsignedBootstrapPath = os.path.join(params.installerPath, params.tempBootstrapName)
signingBootstrapPath = os.path.join(params.installerPath, params.bootstrapName)
signingEnginePath = os.path.join(params.installerPath, "engine.exe")
if operationMode == OperationMode.BuildInstaller:
shutil.copy(params.bootstrapOutputPath, unsignedBootstrapPath)
# copy bootstrapper to installerPath?
if operationMode == OperationMode.BuildInstaller and \
params.installerPath is not params.intermediateInstallerPath and \
os.path.exists(signingEnginePath):
os.remove(signingEnginePath)
# extract the engine from the bootstrapper with Insignia
success = performStep(insigniaDetachBurnEngine,
unsignedBootstrapPath,
signingEnginePath,
args.verbose)
assert (operationMode is not OperationMode.BuildInstaller or success == 0), \
"Failed to detach burn engine from bootstrapper."
# sign the engine, verify signing was successful
performStep(signtoolSignAndVerifyFile,
signingEnginePath,
params.installerPath,
params.intermediateInstallerPath,
params.signingType,
args.timestampServer,
args.verbose)
# attach the engine back to the bootstrapper with Insignia
success = performStep(insigniaAttachBurnEngine,
unsignedBootstrapPath,
signingEnginePath,
signingBootstrapPath,
args.verbose)
assert (operationMode is not OperationMode.BuildInstaller or success == -1 or success == 0), \
"Failed to reattach burn engine to bootstrapper with error {}.".format(success)
# delete the stray engine file since it has been reattached to the installer
if operationMode is OperationMode.BuildInstaller:
os.remove(signingEnginePath)
os.remove(unsignedBootstrapPath)
# sign the bootstrapper, verify the signing was successful
performStep(signtoolSignAndVerifyFile,
signingBootstrapPath,
params.installerPath,
params.intermediateInstallerPath,
params.signingType,
args.timestampServer,
args.verbose)
if args.buildId is not None:
performStep(create_version_file, args.buildId, params.installerPath)
if not args.keep:
performStep(cleanTempFiles, params)
describeStep("All steps completed")
operationMode = OperationMode.StepCounting
maxSteps = 0
buildInstaller()
operationMode = OperationMode.BuildInstaller
maxSteps = stepsTaken
buildInstaller()
Tools/build/JenkinsScripts/distribution/Installer/BuildInstallerUtils.py
-206
View File
@@ -1,206 +0,0 @@
#
# All or portions of this file Copyright (c) Amazon.com, Inc. or its affiliates or
# its licensors.
#
# For complete copyright and license terms please see the LICENSE at the root of this
# distribution (the "License"). All use of this software is governed by the License,
# or, if provided, by the license below or the license accompanying this file. Do not
# remove or modify any license notices. This file is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
#
import json
import os
import re
import shutil
import uuid
from urllib.parse import urlparse
download_url_base = "http://gamedev.amazon.com/lumberyard/releases/"
def set_download_url_base(url):
global download_url_base
download_url_base = url
def is_url(potential_url):
return urlparse(potential_url)[0] == 'https'
def get_package_name(package_path):
path_to_file = package_path
if is_url(package_path):
# index 2 is everything that isn't a parameter in the URL after the high level domain
# see https://docs.python.org/2/library/urlparse.html#urlparse.urlparse for more info
path_to_file = urlparse(package_path)[2]
return os.path.basename(path_to_file)
def get_ly_version_from_package(args, unpacked_location):
version = None
path_to_default_settings = os.path.join(unpacked_location, 'dev/_WAF_/default_settings.json')
verbose_print(args.verbose, 'Searching for Lumberyard version in {}'.format(path_to_default_settings))
with open(path_to_default_settings) as default_settings_file:
default_settings_data = json.load(default_settings_file)
default_settings_file.close()
for build_option in default_settings_data['Build Options']:
if 'attribute' in build_option and 'default_value' in build_option:
if build_option['attribute'] == 'version':
version = build_option['default_value']
if version is None:
verbose_print(args.verbose, 'Version not found in package')
raise Exception('Version was not available in default settings.json')
return version
def append_trailing_slash_to_url(url):
if not url.endswith(tuple(['/', '\\'])):
url += '/'
return url
def generate_target_url(base_target_url, version, build_id, suppress_version_in_path, append_build_id):
output_url = base_target_url
if append_build_id:
output_url = append_trailing_slash_to_url(output_url)
output_url += build_id
if not suppress_version_in_path:
output_url = append_trailing_slash_to_url(output_url)
output_url += '{}/installer'.format(version)
return output_url
# PRODUCT & UPGRADE/PATCH GUID CREATION
def create_id(name, seed, version, build_id):
"""
Generate the Product GUID using the name, the version, and a changelist value.
@param name - Name of the product.
@param seed - String used to create a unique GUID.
@param version - The version of the product in the form "PRODUCT.MAJOR.MINOR.PATCH".
@param build_id - An optional identifier for the build to be used in GUID generation.
@return - A GUID for this version of the product.
"""
# Temporary. Replace the download_url_base with wherever we pass in to the public host or host url.
uuid_seed = download_url_base + seed + name + version
if build_id is not None:
uuid_seed += build_id
return str(uuid.uuid3(uuid.NAMESPACE_URL, uuid_seed)).upper()
# END PRODUCT & UPGRADE/PATCH GUID CREATION
def replace_leading_numbers(source_string):
pattern = re.compile(r'^[0-9]')
return pattern.sub('N', source_string)
def strip_special_characters(source_string):
"""
Remove all non-alphanumeric characters from the given sourceString.
@return - A new string that is a copy of the original, containing only
letters and numbers.
"""
if source_string.isalnum():
return source_string
pattern = re.compile(r'[\W_]+')
return pattern.sub('', source_string)
def check_for_empty_subfolders(package_root, allowed_empty_folders):
# find empty folders
empty_folders_found = []
for root, dirs, files in os.walk(package_root):
if dirs == [] and files == []:
empty_folders_found.append(os.path.normpath(os.path.relpath(root, package_root)))
if allowed_empty_folders is not None:
whitelist_root = 'Whitelist'
whitelist_folders = []
assert (os.path.exists(allowed_empty_folders)), 'The whitelist file specified at {} does not exist.'.format(allowed_empty_folders)
with open(allowed_empty_folders, 'r') as source:
json_whitelist = json.load(source)
try:
for allowed_folder in json_whitelist[whitelist_root]:
whitelist_folders.append(os.path.normpath(allowed_folder))
except KeyError:
print('Unknown json root {}, please check the json root specified.'.format(whitelist_root))
exit(1)
assert (len(whitelist_folders) > 0), 'The whitelist in the file specified at {} is empty. Either populate the list or omit the argument.'.format(allowed_empty_folders)
for folder in empty_folders_found:
assert (folder in whitelist_folders), 'The empty folder {} could not be found in the whitelist of empty folders.'.format(folder)
def get_immediate_subdirectories(root_dir):
"""
Create a list of all directories that exist in the given directory, without
recursing through the subdirectories' children.
@param root_dir - The directory to search for subdirectories.
@return - A list of subdirectories in this directory, excluding their children.
"""
directories = []
for directory in os.listdir(root_dir):
if os.path.isdir(os.path.join(root_dir, directory)):
directories.append(directory)
return directories
def get_file_names_in_directory(directory, file_extension=None):
"""
Create a list of all files in a directory. If given a file extension, it
will list all files with that extension.
@param directory - The directory to gather files from.
@param file_extension - (Optional) The type of files to find. Must be a
string in the ".extension" format. (Default None)
@retun - A list of names of all files in this directory (that match the given
extension if provided).
"""
file_list = []
if file_extension:
for file in os.listdir(directory):
if file.endswith(file_extension):
file_list.append(os.path.basename(file))
else:
for file in os.listdir(directory):
file_list.append(os.path.basename(file))
return file_list
# VERBOSE RELATED FUNCTIONS
def verbose_print(isVerbose, message):
if isVerbose:
print(message)
def find_file_in_package(packageRoot, fileToFind, pathFilters=None):
"""
Searchs the package path for a file.
@param packageRoot: Path to the root of content.
@param fileToFind: Name of the file to find.
@param pathFilters: Path filter to apply to find.
@return: The full path to the file if found, otherwise None.
"""
for root, dirs, files in os.walk(packageRoot):
if fileToFind in files:
if pathFilters is None or any(pathFilter in root for pathFilter in pathFilters):
return os.path.join(root, fileToFind)
return None
def safe_shutil_file_copy(src, dst):
# need to remove the old version of dst if it already exists due to a bug
# in shutil.copy that causes both the src and dst files to be zeroed out
# if they are identical files.
if os.path.exists(dst):
os.remove(dst)
shutil.copy(src, dst)
def create_version_file(buildId, installerPath):
with open(os.path.join(installerPath, 'version.txt'), 'w') as versionFile:
versionFile.write(buildId)
Tools/build/JenkinsScripts/distribution/Installer/BuildInstallerWixUtils.py
-85
View File
@@ -1,85 +0,0 @@
#
# All or portions of this file Copyright (c) Amazon.com, Inc. or its affiliates or
# its licensors.
#
# For complete copyright and license terms please see the LICENSE at the root of this
# distribution (the "License"). All use of this software is governed by the License,
# or, if provided, by the license below or the license accompanying this file. Do not
# remove or modify any license notices. This file is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
#
import glob
import os
import shutil
from BuildInstallerUtils import *
def boolToWixBool(value):
"""
WiX uses strings "yes" and "no" for their boolean values.
@return - "yes" if value == True, otherwise "no".
"""
if value:
return "yes"
else:
return "no"
def createPackageInfo(directoryName, rootDirectory, outputDirectory, outputPrefix=""):
"""
Generate commonly used information about a package that is used for WiX functions.
@param directoryName - The name of the directory that will be packaged.
@param rootDirectory - The path to the given package.
@param outputDirectory - The full path (including file name) to the wxs file
generated for this package.
@param outputPrefix - A string to append to the beginning of the name of the
package when creating the output file. (Default = "").
@return - A dictionary containing the package name, source information, and
output information.
"""
safeDirectoryName = directoryName
if not safeDirectoryName:
safeDirectoryName = "packageRoot"
moduleName = strip_special_characters(safeDirectoryName)
sourceDirectory = os.path.join(rootDirectory, directoryName)
wxsName = '{}{}'.format(outputPrefix, moduleName)
outputPath = os.path.join(outputDirectory, '{}.wxs'.format(wxsName))
componentGroupRef = '{}CG'.format(replace_leading_numbers(wxsName))
packageInfo = {
'name': moduleName,
'wxsName': wxsName,
'wxsPath': outputPath,
'sourceName': safeDirectoryName,
'sourcePath': sourceDirectory,
'componentGroupRefs': componentGroupRef
}
return packageInfo
def getVerboseCommand(verboseMode):
if verboseMode:
return " -v"
else:
return ""
def printProgress(message, stepCount, maxSteps):
# We want the step count to line up at the end to the total steps, so add one.
print('{}/{}: {}'.format(stepCount + 1, maxSteps, message))
def cleanTempFiles(params):
dirs_to_delete = [
params.wxsRoot,
params.wixObjOutput,
params.bootstrapWixObjDir,
params.tempBootstrapOutputDir,
]
for del_dir in dirs_to_delete:
if os.path.exists(del_dir):
shutil.rmtree(del_dir)
for filepath in glob.glob(os.path.join(params.intermediateInstallerPath, "*.wixpdb")):
os.remove(filepath)
Tools/build/JenkinsScripts/distribution/Installer/Candle.py
-145
View File
@@ -1,145 +0,0 @@
#
# All or portions of this file Copyright (c) Amazon.com, Inc. or its affiliates or
# its licensors.
#
# For complete copyright and license terms please see the LICENSE at the root of this
# distribution (the "License"). All use of this software is governed by the License,
# or, if provided, by the license below or the license accompanying this file. Do not
# remove or modify any license notices. This file is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
#
import os
from BuildInstallerWixUtils import *
# CANDLE COMMANDLINE TEMPLATES
candleCommandBase = "candle.exe -nologo -o {outputDirectory} {verbose} {preprocessorParams} {wxsFile}"
candlePackagePreprocessor = " -dProductGUID={productGUID} -dProductUpgradeGUID={upgradeCodeGUID}" \
" -dLumberyardVersion={lumberyardVersion}" \
" -dCabPrefix={cabPrefix}" \
" -dComponentGroupRefs={packageComponentGroup}" \
' -dComponentRefs={componentRefs} -dPackageName="{packageName}"'
candleDownloadPreprocessor = " -dROOTURL={downloadURL}"
candleBootstrapPreprocessor = "-dLYInstallerPath={installerPath} -dLYInstallerNameList={installerList}" \
" -dLumberyardVersion={lumberyardVersion}" \
" -dUseStreaming={useStreaming}" \
' -dMetricsSourcePath="{metricsSourcePath}"' \
' -dMetricsExeName="{metricsExe}"' \
' -dPathTo2015Thru2019Redist="{pathTo2015Thru2019Redist}"' \
' -dFilename2015Thru2019Redist="{redist2015Thru2019Exe}"' \
" -dUpgradeCode={upgradeGUID} -ext WixBalExtension -ext WixUtilExtension"
usedCabPrefixList = set()
def candlePackageContent(outputDir, wxsPath, verboseMode):
verboseCmd = getVerboseCommand(verboseMode)
candleCommand = candleCommandBase.format(outputDirectory=os.path.join(outputDir, ''),
verbose=verboseCmd, preprocessorParams="", wxsFile=wxsPath)
verbose_print(verboseMode, '\n{}\n'.format(candleCommand))
return os.system(candleCommand)
def candleAllPackagesContent(packageInfoMap, wixobjDir, verboseMode):
for packageInfo in packageInfoMap.values():
outputDir = os.path.join(wixobjDir, packageInfo['wxsName'])
success = candlePackageContent(outputDir, packageInfo['wxsPath'], verboseMode)
assert (success == 0), 'Failed to generate wixobj file for {} content.'.format(packageInfo['name'])
# Cab prefixes have to be less than 8 characters, including the sequential numbers for multiple cabs.
# To give room for multiple cabs for an MSI, we're dropping down to 5 characters.
# When we strip some paths in 3rd party to 5 characters, they collide, so we do a little extra
# Logic to help with collisions.
def generateCabPrefix(packageName):
# We want a cab prefix length of five to give room for 100+ cabs for an MSI
cabPrefixLength = 5
uniquenessIndex = -1
uniquenessValue = 0
cabName = packageName[:cabPrefixLength]
# If there is a cab name collision when two packages truncate to the same five digit
# value, then we want to fiddle with the cab prefix to get something unique.
# This simple logic starts at the last character in the prefix and replaces it with a numeral
# it keeps trying that, and moving earlier in the string as it does so.
while cabName in usedCabPrefixList:
cabName = cabName[:cabPrefixLength+uniquenessIndex] + str(uniquenessValue) + cabName[cabPrefixLength+uniquenessIndex+1:]
uniquenessValue += 1
if uniquenessValue > 9:
uniquenessValue = 0
uniquenessIndex -= 1
if uniquenessIndex <= -cabPrefixLength:
raise Exception("Could not generate unique cab name for {}".format(packageName))
usedCabPrefixList.add(cabName)
return cabName
def candlePackage(outputDir, wxsTemplatePath, packageInfo, verboseMode, lyVersion, buildId):
verboseCmd = getVerboseCommand(verboseMode)
cabPrefix = generateCabPrefix(packageInfo['name'])
productGUID = create_id(packageInfo['name'], 'PRODUCT', lyVersion, buildId)
productUpgradeGUID = create_id(packageInfo['name'], 'PRODUCTUPDATE', lyVersion, buildId)
componentRefs = packageInfo.get('componentRefs', '')
candlePreprocessor = candlePackagePreprocessor.format(productGUID=productGUID,
upgradeCodeGUID=productUpgradeGUID,
lumberyardVersion=lyVersion,
componentRefs=componentRefs,
cabPrefix=cabPrefix,
packageComponentGroup=packageInfo['componentGroupRefs'],
packageName=packageInfo['sourceName'])
candleCommand = candleCommandBase.format(outputDirectory=outputDir,
verbose=verboseCmd,
preprocessorParams=candlePreprocessor,
wxsFile=wxsTemplatePath)
verbose_print(verboseMode, '\n{}\n'.format(candleCommand))
return os.system(candleCommand)
def candlePackages(packageInfoMap, wixobjDir, wxsTemplatePath, verboseMode, lyVersion, buildId):
for packageInfo in packageInfoMap.values():
outputDir = os.path.join(wixobjDir, packageInfo['wxsName'], 'HeatPackage{}.wixobj'.format(packageInfo['wxsName']))
success = candlePackage(outputDir, wxsTemplatePath, packageInfo, verboseMode, lyVersion, buildId)
assert (success == 0), 'Failed to generate wixobj file for {}.'.format(packageInfo['name'])
def candleBootstrap(outputDir,
installersPath,
installerNameList,
wxsFileName,
downloadURL,
verboseMode,
lyVersion,
metricsSourcePath,
metricsExe,
pathTo2015Thru2019Redist,
redist2015Thru2019Exe,
upgradeCodeGUID):
verboseCmd = getVerboseCommand(verboseMode)
useStreaming = downloadURL is not None
useStreamingText = boolToWixBool(useStreaming)
candlePreprocessor = candleBootstrapPreprocessor.format(installerPath=os.path.join(installersPath, ''),
installerList=installerNameList,
lumberyardVersion=lyVersion,
useStreaming=useStreamingText,
upgradeGUID=upgradeCodeGUID,
metricsSourcePath=metricsSourcePath,
metricsExe=metricsExe,
pathTo2015Thru2019Redist=pathTo2015Thru2019Redist,
redist2015Thru2019Exe=redist2015Thru2019Exe)
if useStreaming:
# The WXS file appends the trailing slash before the package name: "$(var.ROOTURL)/{2}"
# We need to strip the trailing slash here if it was passed in with one.
downloadURL = downloadURL.rstrip('/')
candlePreprocessor += candleDownloadPreprocessor.format(downloadURL=downloadURL)
candleCommand = candleCommandBase.format(outputDirectory=outputDir, verbose=verboseCmd,
preprocessorParams=candlePreprocessor, wxsFile=wxsFileName)
verbose_print(verboseMode, '\n{}\n'.format(candleCommand))
return os.system(candleCommand)
Tools/build/JenkinsScripts/distribution/Installer/Heat.py
-257
View File
@@ -1,257 +0,0 @@
#
# All or portions of this file Copyright (c) Amazon.com, Inc. or its affiliates or
# its licensors.
#
# For complete copyright and license terms please see the LICENSE at the root of this
# distribution (the "License"). All use of this software is governed by the License,
# or, if provided, by the license below or the license accompanying this file. Do not
# remove or modify any license notices. This file is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
#
import os
import sys
import traceback
import xml.etree.ElementTree as ET
from BuildInstallerWixUtils import *
# HEAT COMMANDLINE TEMPLATES
heatCommandBase = 'heat.exe {harvestType} "{harvestSource}" -nologo -sreg -gg'
heatCommandRequiredArgs = " -dr {directoryRef} -cg {componentGroup} -out {outputPath}"
def convertToForwardSlashAndLower(pathString):
"""
Used to ensure path strings are formatted in a consistent manner.
"""
return pathString.replace('\\', '/').lower()
def lowerDictValues(dictToLower):
"""
Used to lower-case all values in the given dictionary.
"""
for key in dictToLower.keys():
if isinstance(dictToLower[key], list):
dictToLower[key][:] = [convertToForwardSlashAndLower(valueStr) for valueStr in dictToLower[key]]
def removeWxsEntriesWithJsonDirFilelist(wxsFilepath, jsonDirFilelist, directoryKey):
"""
Removes entries from the given *.wxs file with the given jsonDirFilelist.
@param wxsFilepath - Path to *.wxs file to operate the given jsonDirFilelist against
@param jsonDirFilelist - Dictionary populated from given "dirFilelist" JSON. Files
in the *.wxs files that don't have corresponding entries in dirFilelist will be
removed from the *.wxs XML content.
"""
# Without registering the namespace, the ET XML serialized output
# is pretty funky, and WiX will not be happy.
namespaceStr = 'http://schemas.microsoft.com/wix/2006/wi'
ET.register_namespace('', namespaceStr)
tree = ET.ElementTree()
tree.parse(wxsFilepath)
xmlRoot = tree.getroot()
# Find all Component tags that have File tags
ns = {'wixns': namespaceStr}
componentGroupList = xmlRoot.findall('.//wixns:ComponentGroup', ns)
for componentGroup in componentGroupList:
componentList = componentGroup.findall('.//wixns:Component[wixns:File]', ns)
for component in componentList:
for childFileTag in component:
# Entries are typically prefixed with an unnecessary 'SourceDir\' string
sourceValue = convertToForwardSlashAndLower(childFileTag.attrib['Source']).replace('sourcedir/', '')
if sourceValue not in jsonDirFilelist[directoryKey]:
component.remove(childFileTag)
# WXS files generated by our installer typically only have one File tag per
# ComponentGroup, but we'll check that it's empty, just in case.
if len(component.getchildren()) < 1:
componentGroup.remove(component)
tree.write(wxsFilepath)
def heatDirectory(wxsName, sourceDirectory, outputPath, componentGroup, directoryRefName, verbose):
"""
Generate a .wxs file for a given directory, including all subdirectories,
and place it at the given outputPath.
@remarks - Intentionally hardcoding harvest type to "dir" here. For other
types of harvests, a new function should be created. Any logic not
directly related to executing the heat command should exist outside this
function.
"""
verboseCmd = getVerboseCommand(verbose)
# Intentionally hardcoding harvest type to "dir" here. For other types of harvests, a new
# function should be created. Any logic not directly related to executing
# the heat command should exist outside this function.
heatCommand = heatCommandBase.format(harvestType="dir",
harvestSource=sourceDirectory)
heatCommand += verboseCmd
heatCommand += heatCommandRequiredArgs.format(directoryRef=directoryRefName,
componentGroup=componentGroup, outputPath=outputPath)
verbose_print(verbose, '\n{}\n'.format(heatCommand))
return os.system(heatCommand)
def heatDirectories(rootDirectory, outputDirectory, directoryRefName, verbose, outputPrefix="", dirFilelist=None):
"""
Generates package info for every directory in the given rootDirectory, and
gathers each package's content information into a .wxs file.
@param rootDirectory - The directory containing the source of many packages.
@param outputDirectory - The directory to put all generated .wxs files.
@param directoryRefName - The ID of the directory for these source files to
be placed when installed. (Must match HeatPackageBase directory ID.)
@param verbose - Running in verbose mode?
@param outputPrefix - A string to append to the beginning of the name of the
package when creating the output file. (Default = "").
@param dirFilelist - A string that gives a path to a JSON file containing a
list of directories, and for each directory, a list of files. Only the
files listed in the JSON file will be included in the installer output
for the directory specified. This can be used to remove unnecessary
files that aren't needed on a customer's machine, but are included in a
packaged build created by Jenkins, for example.
@return - A dictionary of package names to their associated package information (dictionaries).
"""
# Attempt to parse jsonDirFilelist JSON.
jsonDirFilelist = {}
if dirFilelist is not None:
assert (os.path.exists(dirFilelist)), 'The "dirFilelist" argument was provided but the JSON file at {} does not exist.'.format(dirFilelist)
with open(dirFilelist, 'r') as source:
try:
jsonDirFilelist = json.load(source)
except ValueError as e:
print(traceback.format_exc())
print('Error parsing the given JSON file at {} with exception: {}'.format(dirFilelist, e))
sys.exit()
except:
print(traceback.format_exc())
print('Unexpected error parsing the given JSON file at {}. Please verify that the file is correctly formatted.'.format(dirFilelist))
sys.exit()
combinedWxsResults = {}
jsonValuesLowered = False
for directoryName in get_immediate_subdirectories(rootDirectory):
packageInfo = createPackageInfo(directoryName, rootDirectory, outputDirectory, outputPrefix)
moduleName = packageInfo['name']
sourceDirectory = packageInfo['sourcePath']
wxsName = packageInfo['wxsName']
outputPath = packageInfo['wxsPath']
# There will only be one component group in the reference list at this point.
componentGroup = packageInfo['componentGroupRefs']
# check for existence of name collision
if moduleName in combinedWxsResults:
print('ERROR when creating module "{0}" from "{1}". A module with the name "{0}" already exists.'.format(moduleName, sourceDirectory))
# Passing let us rapidly iterate on this tool, feel free to upgrade to raising an exception.
pass
combinedWxsResults[moduleName] = packageInfo
success = heatDirectory(wxsName, sourceDirectory, outputPath, componentGroup, directoryRefName, verbose)
assert (success == 0), 'Failed to generate WXS file for {}.'.format(moduleName)
sourceDirFormatted = convertToForwardSlashAndLower(sourceDirectory)
for key in jsonDirFilelist.keys():
if sourceDirFormatted.endswith(convertToForwardSlashAndLower(key)):
# Lower-case all entries to allow case-insensitive compare
if not jsonValuesLowered:
lowerDictValues(jsonDirFilelist)
jsonValuesLowered = True
# Alter XML contents of WXS file by filtering it against the JSON
# directory list of files.
removeWxsEntriesWithJsonDirFilelist(outputPath, jsonDirFilelist, key)
return combinedWxsResults
def heatFile(file,
directoryRefName,
rootDirectory,
verbose,
componentGroup,
outputPath):
"""
Generates a WXS file for an individual file.
@param file: Full path to the file to heat.
@param directoryRefName: The ID of the directory for these source files to
be placed when installed. (Must match HeatPackageBase directory ID.)
@param verbose: Running in verbose mode?
@param componentGroup: The component group to set in the file.
@param outputPath: Where to output the generated WXS file.
@return:
"""
heatCommand = heatCommandBase.format(harvestType="file", harvestSource=file)
heatCommand += getVerboseCommand(verbose)
# According to Wix's docs ( http://wixtoolset.org/documentation/manual/v3/overview/heat.html )
# SRD's description implies that it suppresses root directory harvesting, which seems to imply it works only
# in directory harvesting mode. It also implies that it takes in no parameters.
# This is either poor documentation, or incorrect (I'm assuming poor documentation).
# The actual behavior is:
# Normally when harvesting with Heat (directory or file), directories and directory references are generated
# based on the path to the root directory. By calling suppress root directory harvesting and passing in a directory,
# then the generated nested directory path in the generated wxs file will not include pathing based on this.
# This is necessary when harvesting the loose files in a folder's root: Heat's harvesting does not support
# whitelist / blacklist functionality, and all subfolders of the package have been harvested in other ways.
# This means that all of the loose files in the directory roots that weren't included elsewhere need to be
# included in some other installers. If the root directory is not suppressed, then they generate a directory
# hierarchy that collides with each other.
heatCommand += " -srd " + rootDirectory
heatCommand += heatCommandRequiredArgs.format(directoryRef=directoryRefName,
componentGroup=componentGroup,
outputPath=outputPath)
verbose_print(verbose, '\n{}\n'.format(heatCommand))
return os.system(heatCommand)
def heatFiles(fileList,
rootDirectory,
outputDirectory,
directoryRefName,
verbose,
outputPrefix = ""):
"""
Generates WXS files for every file in the file list, and returns a mapping containing the associated
WXS files and component groups.
@param fileList: The list of files to generate WXS files for.
@param outputDirectory: The directory to put generated WXS files.
@param directoryRefName: The ID of the directory for these source files to
be placed when installed. (Must match HeatPackageBase directory ID.)
@param verbose: Running in verbose mode?
@param outputPrefix: A string to append to the beginning of the name of the
package when creating the output file. (Default = "").
@return: A dictionary of input files to their associated WXS information (component group, WXS file location).
"""
combinedWxsResults = {}
for file in fileList:
moduleName = strip_special_characters(file)
wxsName = '{}{}'.format(outputPrefix, moduleName)
outputPath = os.path.join(outputDirectory, '{}.wxs'.format(wxsName))
componentGroup = '{}CG'.format(replace_leading_numbers(strip_special_characters(file)))
wxsInfo = {
'name': moduleName,
'wxsName': wxsName,
'wxsPath': outputPath,
'componentGroupRefs': componentGroup
}
combinedWxsResults[file] = wxsInfo
success = heatFile(os.path.join(rootDirectory,file),
directoryRefName,
rootDirectory,
verbose,
componentGroup,
outputPath)
assert (success == 0), 'Failed to generate WXS file for {}.'.format(moduleName)
return combinedWxsResults
Tools/build/JenkinsScripts/distribution/Installer/HeatDevPackageBase.wxs
-3
View File
@@ -1,3 +0,0 @@
version https://git-lfs.github.com/spec/v1
oid sha256:92fb31b309b613d1f47feb3a728d4528d11d27d9a5e539b109800e209cab0580
size 5754
Tools/build/JenkinsScripts/distribution/Installer/HeatPackageBase.wxs
-3
View File
@@ -1,3 +0,0 @@
version https://git-lfs.github.com/spec/v1
oid sha256:e708bd9a6e9191431ecc5840e6adaf3f4e5ca52f3f994acefa454bb908476c97
size 5435

Some files were not shown because too many files have changed in this diff Show More