Merge pull request #11 from aws-lumberyard/main

Merging from main to 1.0

.lfsconfig

@@ -1,2 +1,2 @@
 [lfs]
-url=https://d111jhwumogxnd.cloudfront.net/api/v1
+url=https://dsasbkzux9giw.cloudfront.net/api/v1

AutomatedReview/Jenkinsfile

@@ -16,14 +16,15 @@ INCREMENTAL_BUILD_SCRIPT_PATH = 'scripts/build/bootstrap/incremental_build_util.
 
 EMPTY_JSON = readJSON text: '{}'
 
-ENGINE_REPOSITORY_NAME = env.DEFAULT_REPOSITORY_NAME
+ENGINE_REPOSITORY_NAME = 'o3de'
 
 def pipelineProperties = []
 
 def pipelineParameters = [
     // Build/clean Parameters
     // The CLEAN_OUTPUT_DIRECTORY is used by ci_build scripts. Creating the parameter here passes it as an environment variable to jobs and is consumed that way
-    booleanParam(defaultValue: false, description: 'Deletes the contents of the output directory before building. This will cause a \"clean\" build', name: 'CLEAN_OUTPUT_DIRECTORY'),
+    booleanParam(defaultValue: false, description: 'Deletes the contents of the output directory before building. This will cause a \"clean\" build. NOTE: does not imply CLEAN_ASSETS', name: 'CLEAN_OUTPUT_DIRECTORY'),
+    booleanParam(defaultValue: false, description: 'Deletes the contents of the output directories of the AssetProcessor before building.', name: 'CLEAN_ASSETS'),
     booleanParam(defaultValue: false, description: 'Deletes the contents of the workspace and forces a complete pull.', name: 'CLEAN_WORKSPACE'),
     booleanParam(defaultValue: false, description: 'Recreates the volume used for the workspace. The volume will be created out of a snapshot taken from main.', name: 'RECREATE_VOLUME'),
     string(defaultValue: '', description: 'Filters and overrides the list of jobs to run for each of the below platforms (comma-separated). Can\'t be used during a pull request.', name: 'JOB_LIST_OVERRIDE'),
@@ -105,9 +106,9 @@ def GetRunningPipelineName(JENKINS_JOB_NAME) {
     // If the job name has an underscore
     def job_parts = JENKINS_JOB_NAME.tokenize('/')[0].tokenize('_')
     if (job_parts.size() > 1) {
-        return job_parts[job_parts.size()-1]
+        return [job_parts.take(job_parts.size() - 1).join('_'), job_parts[job_parts.size()-1]]
     }
-    return 'default'
+    return [job_parts[0], 'default']
 }
 
 @NonCPS
@@ -222,45 +223,39 @@ def PullFilesFromGit(String filenamePath, String branchName, boolean failIfNotFo
     folderPathParts.remove(folderPathParts.size()-1) // remove the filename
     def folderPath = folderPathParts.join('/')
     if (folderPath.contains('*')) {
-
-        try {
-            def currentPath = ''
-            for (int i = 0; i < folderPathParts.size(); i++) {
-                if (folderPathParts[i] == '*') {
-                    palMkdir(currentPath)
-                    retry(3) { palSh("aws codecommit get-folder --repository-name ${repositoryName} --commit-specifier ${branchName} --folder-path ${currentPath} > ${currentPath}/.codecommit", "GetFolder ${currentPath}") }
-                    def folderInfo = readJSON file: "${currentPath}/.codecommit"
-                    folderInfo.subFolders.each { folder ->
-                        def newSubPath = currentPath + '/' + folder.relativePath
-                        for (int j = i+1; j < folderPathParts.size(); j++) {
-                            newSubPath = newSubPath + '/' + folderPathParts[j]
-                        }
-                        newSubPath = newSubPath + '/' + filename
-                        PullFilesFromGit(newSubPath, branchName, false, repositoryName)
+        
+        def currentPath = ''
+        for (int i = 0; i < folderPathParts.size(); i++) {
+            if (folderPathParts[i] == '*') {
+                palMkdir(currentPath)
+                retry(3) { palSh("aws codecommit get-folder --repository-name ${repositoryName} --commit-specifier ${branchName} --folder-path ${currentPath} > ${currentPath}/.codecommit", "GetFolder ${currentPath}") }
+                def folderInfo = readJSON file: "${currentPath}/.codecommit"
+                folderInfo.subFolders.each { folder ->
+                    def newSubPath = currentPath + '/' + folder.relativePath
+                    for (int j = i+1; j < folderPathParts.size(); j++) {
+                        newSubPath = newSubPath + '/' + folderPathParts[j]
                     }
-                    palRm("${currentPath}/.codecommit")
-                }
-                if (i == 0) {
-                    currentPath = folderPathParts[i]
-                } else {
-                    currentPath = currentPath + '/' + folderPathParts[i]
+                    newSubPath = newSubPath + '/' + filename
+                    PullFilesFromGit(newSubPath, branchName, false, repositoryName)
                 }
+                palRm("${currentPath}/.codecommit")
+            }
+            if (i == 0) {
+                currentPath = folderPathParts[i]
+            } else {
+                currentPath = currentPath + '/' + folderPathParts[i]
             }
-        } catch(Exception e) {
         }
 
     } else if (filename.contains('*')) {
 
-        try {
-            palMkdir(folderPath)
-            retry(3) { palSh("aws codecommit get-folder --repository-name ${repositoryName} --commit-specifier ${branchName} --folder-path ${folderPath} > ${folderPath}/.codecommit", "GetFolder ${folderPath}") }
-            def folderInfo = readJSON file: "${folderPath}/.codecommit"
-            folderInfo.files.each { file ->
-                PullFilesFromGit("${folderPath}/${filename}", branchName, false, repositoryName)
-            }
-            palRm("${folderPath}/.codecommit")
-        } catch(Exception e) {
+        palMkdir(folderPath)
+        retry(3) { palSh("aws codecommit get-folder --repository-name ${repositoryName} --commit-specifier ${branchName} --folder-path ${folderPath} > ${folderPath}/.codecommit", "GetFolder ${folderPath}") }
+        def folderInfo = readJSON file: "${folderPath}/.codecommit"
+        folderInfo.files.each { file ->
+            PullFilesFromGit("${folderPath}/${filename}", branchName, false, repositoryName)
         }
+        palRm("${folderPath}/.codecommit")
 
     } else {
 
@@ -403,7 +398,7 @@ def CheckoutRepo(boolean disableSubmodules = false) {
     }
 }
 
-def PreBuildCommonSteps(String pipeline, String branchName, String platform, String buildType, String workspace, boolean mount = true, boolean disableSubmodules = false) {
+def PreBuildCommonSteps(Map pipelineConfig, String projectName, String pipeline, String branchName, String platform, String buildType, String workspace, boolean mount = true, boolean disableSubmodules = false) {
     echo 'Starting pre-build common steps...'
 
     if (mount) {
@@ -413,11 +408,11 @@ def PreBuildCommonSteps(String pipeline, String branchName, String platform, Str
         if(env.IS_UNIX) pythonCmd = 'sudo -E python -u '
         else pythonCmd = 'python -u '
 
-        if(params.RECREATE_VOLUME) {
-            palSh("${pythonCmd} ${INCREMENTAL_BUILD_SCRIPT_PATH} --action delete --pipeline ${pipeline} --branch ${branchName} --platform ${platform} --build_type ${buildType}", 'Deleting volume')
+        if(env.RECREATE_VOLUME.toBoolean()) {
+            palSh("${pythonCmd} ${INCREMENTAL_BUILD_SCRIPT_PATH} --action delete --project ${projectName} --pipeline ${pipeline} --branch ${branchName} --platform ${platform} --build_type ${buildType}", 'Deleting volume')
         }
         timeout(5) {
-            palSh("${pythonCmd} ${INCREMENTAL_BUILD_SCRIPT_PATH} --action mount --pipeline ${pipeline} --branch ${branchName} --platform ${platform} --build_type ${buildType}", 'Mounting volume')
+            palSh("${pythonCmd} ${INCREMENTAL_BUILD_SCRIPT_PATH} --action mount --project ${projectName} --pipeline ${pipeline} --branch ${branchName} --platform ${platform} --build_type ${buildType}", 'Mounting volume')
         }
 
         if(env.IS_UNIX) {
@@ -434,7 +429,7 @@ def PreBuildCommonSteps(String pipeline, String branchName, String platform, Str
 
     // Cleanup previous repo location, we are currently at the root of the workspace, if we have a .git folder
     // we need to cleanup. Once all branches take this relocation, we can remove this
-    if(params.CLEAN_WORKSPACE || fileExists("${workspace}/.git")) {
+    if(env.CLEAN_WORKSPACE.toBoolean() || fileExists("${workspace}/.git")) {
         if(fileExists(workspace)) {
             palRmDir(workspace)
         }
@@ -453,6 +448,17 @@ def PreBuildCommonSteps(String pipeline, String branchName, String platform, Str
                 bat label: 'Getting python',
                     script: 'python/get_python.bat'
             }
+
+            if(env.CLEAN_OUTPUT_DIRECTORY.toBoolean() || env.CLEAN_ASSETS.toBoolean()) {
+                def command = "${pipelineConfig.BUILD_ENTRY_POINT} --platform ${platform} --type clean"
+                if (env.IS_UNIX) {
+                    sh label: "Running ${platform} clean",
+                       script: "${pipelineConfig.PYTHON_DIR}/python.sh -u ${command}"
+                } else {
+                    bat label: "Running ${platform} clean",
+                        script: "${pipelineConfig.PYTHON_DIR}/python.cmd -u ${command}".replace('/','\\')
+                }
+            }
         }
     }
 }
@@ -476,13 +482,12 @@ def TestMetrics(Map options, String workspace, String branchName, String repoNam
         dir("${workspace}/${ENGINE_REPOSITORY_NAME}") {
             checkout scm: [
                 $class: 'GitSCM',
+                branches: [[name: '*/main']],
                 extensions: [[$class: 'RelativeTargetDirectory', relativeTargetDir: 'mars']],
-                userRemoteConfigs: [[url: "${env.MARS_REPO}", name: 'mars']]
+                userRemoteConfigs: [[url: "${env.MARS_REPO}", name: 'mars', credentialsId: "${env.GITHUB_USER}"]]
             ]
             withCredentials([usernamePassword(credentialsId: "${env.SERVICE_USER}", passwordVariable: 'apitoken', usernameVariable: 'username')]) {
                 def command = "${options.PYTHON_DIR}/python.cmd -u mars/scripts/python/ctest_test_metric_scraper.py -e jenkins.creds.user ${username} -e jenkins.creds.pass ${apitoken} ${cmakeBuildDir} ${branchName} %BUILD_NUMBER% AR ${configuration} ${repoName} "
-                if (params.DESTINATION_BRANCH)
-                    command += "--destination-branch ${params.DESTINATION_BRANCH} "
                 bat label: "Publishing ${buildJobName} Test Metrics",
                 script: command
             }
@@ -516,10 +521,10 @@ def PostBuildCommonSteps(String workspace, boolean mount = true) {
     }
 }
 
-def CreateSetupStage(String pipelineName, String branchName, String platformName, String jobName, Map environmentVars) {
+def CreateSetupStage(Map pipelineConfig, String projectName, String pipelineName, String branchName, String platformName, String jobName, Map environmentVars) {
     return {
         stage("Setup") {
-            PreBuildCommonSteps(pipelineName, branchName, platformName, jobName,  environmentVars['WORKSPACE'], environmentVars['MOUNT_VOLUME'])
+            PreBuildCommonSteps(pipelineConfig, projectName, pipelineName, branchName, platformName, jobName,  environmentVars['WORKSPACE'], environmentVars['MOUNT_VOLUME'])
         }
     }
 }
@@ -548,6 +553,7 @@ def CreateTeardownStage(Map environmentVars) {
     }
 }
 
+def projectName = ''
 def pipelineName = ''
 def branchName = ''
 def pipelineConfig = {}
@@ -562,7 +568,7 @@ try {
             }
             withEnv(envVarList) {
                 timestamps {
-                    pipelineName = GetRunningPipelineName(env.JOB_NAME) // env.JOB_NAME is the name of the job given by Jenkins
+                    (projectName, pipelineName) = GetRunningPipelineName(env.JOB_NAME) // env.JOB_NAME is the name of the job given by Jenkins
                     scmType = GetSCMType()
 
                     if(env.BRANCH_NAME) {
@@ -628,7 +634,7 @@ try {
                                 try {
                                     def build_job_name = build_job.key
                                     
-                                    CreateSetupStage(pipelineName, branchName, platform.key, build_job.key, envVars).call()
+                                    CreateSetupStage(pipelineConfig, projectName, pipelineName, branchName, platform.key, build_job.key, envVars).call()
 
                                     if(build_job.value.steps) { //this is a pipe with many steps so create all the build stages
                                         build_job.value.steps.each { build_step ->
@@ -686,7 +692,7 @@ finally {
             snsPublish(
                 topicArn: env.SNS_TOPIC, 
                 subject:'Build Result', 
-                message:"${currentBuild.currentResult}:${params.REPOSITORY_NAME}:${params.SOURCE_BRANCH}:${params.SOURCE_COMMIT}:${params.DESTINATION_COMMIT}:${params.PULL_REQUEST_ID}:${BUILD_URL}:${params.RECREATE_VOLUME}:${params.CLEAN_OUTPUT_DIRECTORY}"
+                message:"${currentBuild.currentResult}:${params.REPOSITORY_NAME}:${params.SOURCE_BRANCH}:${params.SOURCE_COMMIT}:${params.DESTINATION_COMMIT}:${params.PULL_REQUEST_ID}:${BUILD_URL}:${env.RECREATE_VOLUME}:${env.CLEAN_OUTPUT_DIRECTORY}:${env.CLEAN_ASSETS}"
             )
         }
         step([

scripts/build/Jenkins/Jenkinsfile

@@ -0,0 +1,568 @@
+#!/usr/bin/env groovy
+/*
+* All or portions of this file Copyright (c) Amazon.com, Inc. or its affiliates or
+* its licensors.
+*
+* For complete copyright and license terms please see the LICENSE at the root of this
+* distribution (the "License"). All use of this software is governed by the License,
+* or, if provided, by the license below or the license accompanying this file. Do not
+* remove or modify any license notices. This file is distributed on an "AS IS" BASIS,
+* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+*
+*/
+
+PIPELINE_CONFIG_FILE = 'scripts/build/Jenkins/lumberyard.json'
+INCREMENTAL_BUILD_SCRIPT_PATH = 'scripts/build/bootstrap/incremental_build_util.py'
+
+EMPTY_JSON = readJSON text: '{}'
+
+ENGINE_REPOSITORY_NAME = 'o3de'
+
+def pipelineProperties = []
+
+def pipelineParameters = [
+    // Build/clean Parameters
+    // The CLEAN_OUTPUT_DIRECTORY is used by ci_build scripts. Creating the parameter here passes it as an environment variable to jobs and is consumed that way
+    booleanParam(defaultValue: false, description: 'Deletes the contents of the output directory before building. This will cause a \"clean\" build. NOTE: does not imply CLEAN_ASSETS', name: 'CLEAN_OUTPUT_DIRECTORY'),
+    booleanParam(defaultValue: false, description: 'Deletes the contents of the output directories of the AssetProcessor before building.', name: 'CLEAN_ASSETS'),
+    booleanParam(defaultValue: false, description: 'Deletes the contents of the workspace and forces a complete pull.', name: 'CLEAN_WORKSPACE'),
+    booleanParam(defaultValue: false, description: 'Recreates the volume used for the workspace. The volume will be created out of a snapshot taken from main.', name: 'RECREATE_VOLUME'),
+    string(defaultValue: '', description: 'Filters and overrides the list of jobs to run for each of the below platforms (comma-separated). Can\'t be used during a pull request.', name: 'JOB_LIST_OVERRIDE')
+]
+
+def palSh(cmd, lbl = '', winSlashReplacement = true) {
+    if (env.IS_UNIX) {
+        sh label: lbl,
+           script: cmd
+    } else if (winSlashReplacement) {
+        bat label: lbl,
+            script: cmd.replace('/','\\')
+    } else {
+        bat label: lbl,
+            script: cmd
+    }
+}
+
+def palMkdir(path) {
+    if (env.IS_UNIX) {
+        sh label: "Making directories ${path}",
+           script: "mkdir -p ${path}"
+    } else {
+        def win_path = path.replace('/','\\')
+        bat label: "Making directories ${win_path}",
+            script: "mkdir ${win_path}."
+    }
+}
+
+def palRm(path) {
+    if (env.IS_UNIX) {
+        sh label: "Removing ${path}",
+           script: "rm ${path}"
+    } else {
+        def win_path = path.replace('/','\\')
+        bat label: "Removing ${win_path}",
+            script: "del ${win_path}"
+    }
+}
+
+def palRmDir(path) {
+    if (env.IS_UNIX) {
+        sh label: "Removing ${path}",
+           script: "rm -rf ${path}"
+    } else {
+        def win_path = path.replace('/','\\')
+        bat label: "Removing ${win_path}",
+            script: "rd /s /q ${win_path}"
+    }
+}
+
+def IsJobEnabled(buildTypeMap, pipelineName, platformName) {
+    def job_list_override = params.JOB_LIST_OVERRIDE.tokenize(',')
+    if (!job_list_override.isEmpty()) {
+        return params[platformName] && job_list_override.contains(buildTypeMap.key);
+    } else {
+        if (params[platformName]) {
+            if(buildTypeMap.value.TAGS) {
+                return buildTypeMap.value.TAGS.contains(pipelineName)
+            }    
+        }
+    }
+    return false
+}
+
+def GetRunningPipelineName(JENKINS_JOB_NAME) {
+    // If the job name has an underscore
+    def job_parts = JENKINS_JOB_NAME.tokenize('/')[0].tokenize('_')
+    if (job_parts.size() > 1) {
+        return [job_parts.take(job_parts.size() - 1).join('_'), job_parts[job_parts.size()-1]]
+    }
+    return [job_parts[0], 'default']
+}
+
+@NonCPS
+def RegexMatcher(str, regex) {
+    def matcher = (str =~ regex)
+    return matcher ? matcher.group(1) : null
+}
+
+def LoadPipelineConfig(String pipelineName, String branchName) {
+    echo 'Loading pipeline config'
+    def pipelineConfig = {}
+    pipelineConfig = readJSON file: PIPELINE_CONFIG_FILE
+    palRm(PIPELINE_CONFIG_FILE)
+    pipelineConfig.platforms = EMPTY_JSON
+
+    // Load the pipeline configs per platform
+    pipelineConfig.PIPELINE_CONFIGS.each { pipeline_config ->
+        def platform_regex = pipeline_config.replace('.','\\.').replace('*', '(.*)')
+        if (!env.IS_UNIX) {
+            platform_regex = platform_regex.replace('/','\\\\')
+        }
+        echo "Searching platform pipeline configs in ${pipeline_config} using ${platform_regex}"
+        for (pipeline_config_path in findFiles(glob: pipeline_config)) {
+            echo "\tFound platform pipeline config ${pipeline_config_path}"
+            def platform = RegexMatcher(pipeline_config_path, platform_regex)
+            if(platform) {
+                pipelineConfig.platforms[platform] = EMPTY_JSON
+                pipelineConfig.platforms[platform].PIPELINE_ENV = readJSON file: pipeline_config_path.toString()
+            }
+            palRm(pipeline_config_path.toString())
+        }
+    }
+
+    // Load the build configs
+    pipelineConfig.BUILD_CONFIGS.each { build_config ->
+        def platform_regex = build_config.replace('.','\\.').replace('*', '(.*)')
+        if (!env.IS_UNIX) {
+            platform_regex = platform_regex.replace('/','\\\\')
+        }
+        echo "Searching configs in ${build_config} using ${platform_regex}"
+        for (build_config_path in findFiles(glob: build_config)) {
+            echo "\tFound config ${build_config_path}"
+            def platform = RegexMatcher(build_config_path, platform_regex)
+            if(platform) {
+                pipelineConfig.platforms[platform].build_types = readJSON file: build_config_path.toString()
+            }
+        }
+    }
+    return pipelineConfig
+}
+
+def GetBuildEnvVars(Map platformEnv, Map buildTypeEnv, String pipelineName) {
+    def envVarMap = [:]
+    platformPipelineEnv = platformEnv['ENV'] ?: [:]
+    platformPipelineEnv.each { var ->
+        envVarMap[var.key] = var.value
+    }
+    platformEnvOverride = platformEnv['PIPELINE_ENV_OVERRIDE'] ?: [:]
+    platformPipelineEnvOverride = platformEnvOverride[pipelineName] ?: [:]
+    platformPipelineEnvOverride.each { var ->
+        envVarMap[var.key] = var.value
+    }
+    buildTypeEnv.each { var ->
+        // This may override the above one if there is an entry defined by the job
+        envVarMap[var.key] = var.value 
+    }
+    
+    // Environment that only applies to to Jenkins tweaks.
+    // For 3rdParty downloads, we store them in the EBS volume so we can reuse them across node
+    // instances. This allow us to scale up and down without having to re-download 3rdParty
+    envVarMap['LY_PACKAGE_DOWNLOAD_CACHE_LOCATION'] = "${envVarMap['WORKSPACE']}/3rdParty/downloaded_packages"
+    envVarMap['LY_PACKAGE_UNPACK_LOCATION'] = "${envVarMap['WORKSPACE']}/3rdParty/packages"
+
+    return envVarMap
+}
+
+def GetEnvStringList(Map envVarMap) {
+    def strList = []
+    envVarMap.each { var ->
+        strList.add("${var.key}=${var.value}")
+    }
+    return strList
+}
+
+def SetLfsCredentials(cmd, lbl = '') {
+    if (env.IS_UNIX) {
+        sh label: lbl,
+           script: cmd
+    } else { 
+        bat label: lbl,
+            script: cmd
+    }
+}
+
+def CheckoutBootstrapScripts(String branchName) {
+    checkout([$class: "GitSCM",
+            branches: [[name: "*/${branchName}"]],
+            doGenerateSubmoduleConfigurations: false,
+            extensions: [
+              [
+                $class: "SparseCheckoutPaths",
+                sparseCheckoutPaths: [
+                  [ $class: "SparseCheckoutPath", path: "scripts/build/Jenkins/" ],
+                  [ $class: "SparseCheckoutPath", path: "scripts/build/bootstrap/" ],
+                  [ $class: "SparseCheckoutPath", path: "Tools/build/JenkinsScripts/build/Platform" ]
+                ]
+              ],
+              [
+                $class: "CloneOption", depth: 1, noTags: false, reference: "", shallow: true
+              ]
+            ],
+            submoduleCfg: [],
+            userRemoteConfigs: scm.userRemoteConfigs
+          ])
+}
+
+def CheckoutRepo(boolean disableSubmodules = false) {
+    dir(ENGINE_REPOSITORY_NAME) {
+        palSh('git lfs uninstall', 'Git LFS Uninstall') // Prevent git from pulling lfs objects during checkout
+
+        if(fileExists('.git')) {
+            // If the repository after checkout is locked, likely we took a snapshot while git was running,
+            // to leave the repo in a usable state, garbagecollect. This also helps in situations where 
+            def indexLockFile = '.git/index.lock'
+            if(fileExists(indexLockFile)) {
+                palSh('git gc', 'Git GarbageCollect')
+            }
+            if(fileExists(indexLockFile)) { // if it is still there, remove it
+                palRm(indexLockFile)
+            }
+        }
+    }
+
+    def random = new Random()
+    def retryAttempt = 0
+    retry(5) {
+        if (retryAttempt > 0) {
+            sleep random.nextInt(60 * retryAttempt)  // Stagger checkouts to prevent HTTP 429 (Too Many Requests) response from CodeCommit
+        }
+        retryAttempt = retryAttempt + 1
+        dir(ENGINE_REPOSITORY_NAME) {
+            checkout scm: [
+                $class: 'GitSCM',
+                branches: scm.branches,
+                extensions: [
+                    [$class: 'SubmoduleOption', disableSubmodules: disableSubmodules, recursiveSubmodules: true],
+                    [$class: 'CheckoutOption', timeout: 60]
+                ],
+                userRemoteConfigs: scm.userRemoteConfigs
+           ]
+        }
+    }
+
+    // Add folder where we will store the 3rdParty downloads and packages
+    if(!fileExists('3rdParty')) {
+        palMkdir('3rdParty')
+    }    
+
+    dir(ENGINE_REPOSITORY_NAME) {
+        // Run lfs in a separate step. Jenkins is unable to load the credentials for the custom LFS endpoint
+        withCredentials([usernamePassword(credentialsId: "${env.GITHUB_USER}", passwordVariable: 'accesstoken', usernameVariable: 'username')]) {
+            SetLfsCredentials("git config -f .lfsconfig lfs.url https://${username}:${accesstoken}@${env.LFS_URL}", 'Set credentials')
+        }
+        palSh('git lfs install', 'Git LFS Install')
+        palSh('git lfs pull', 'Git LFS Pull')
+
+        // CHANGE_ID is used by some scripts to identify uniquely the current change (usually metric jobs)
+        palSh('git rev-parse HEAD > commitid', 'Getting commit id')
+        env.CHANGE_ID = readFile file: 'commitid'
+        env.CHANGE_ID = env.CHANGE_ID.trim()
+        palRm('commitid')
+    }
+}
+
+def PreBuildCommonSteps(Map pipelineConfig, String projectName, String pipeline, String branchName, String platform, String buildType, String workspace, boolean mount = true, boolean disableSubmodules = false) {
+    echo 'Starting pre-build common steps...'
+
+    if (mount) {
+        unstash name: 'incremental_build_script'
+
+        def pythonCmd = ''
+        if(env.IS_UNIX) pythonCmd = 'sudo -E python -u '
+        else pythonCmd = 'python -u '
+
+        if(env.RECREATE_VOLUME.toBoolean()) {
+            palSh("${pythonCmd} ${INCREMENTAL_BUILD_SCRIPT_PATH} --action delete --project ${projectName} --pipeline ${pipeline} --branch ${branchName} --platform ${platform} --build_type ${buildType}", 'Deleting volume')
+        }
+        timeout(5) {
+            palSh("${pythonCmd} ${INCREMENTAL_BUILD_SCRIPT_PATH} --action mount --project ${projectName} --pipeline ${pipeline} --branch ${branchName} --platform ${platform} --build_type ${buildType}", 'Mounting volume')
+        }
+
+        if(env.IS_UNIX) {
+            sh label: 'Setting volume\'s ownership',
+               script: """
+                if sudo test ! -d "${workspace}"; then
+                    sudo mkdir -p ${workspace} 
+                    cd ${workspace}/..
+                    sudo chown -R lybuilder:root .
+                fi
+               """
+        }
+    }
+
+    // Cleanup previous repo location, we are currently at the root of the workspace, if we have a .git folder
+    // we need to cleanup. Once all branches take this relocation, we can remove this
+    if(env.CLEAN_WORKSPACE.toBoolean() || fileExists("${workspace}/.git")) {
+        if(fileExists(workspace)) {
+            palRmDir(workspace)
+        }
+    }
+
+    dir(workspace) {
+
+        CheckoutRepo(disableSubmodules)
+
+        // Get python
+        dir(ENGINE_REPOSITORY_NAME) {
+            if(env.IS_UNIX) {
+                sh label: 'Getting python',
+                   script: 'python/get_python.sh'
+            } else {
+                bat label: 'Getting python',
+                    script: 'python/get_python.bat'
+            }
+
+            if(env.CLEAN_OUTPUT_DIRECTORY.toBoolean() || env.CLEAN_ASSETS.toBoolean()) {
+                def command = "${pipelineConfig.BUILD_ENTRY_POINT} --platform ${platform} --type clean"
+                if (env.IS_UNIX) {
+                    sh label: "Running ${platform} clean",
+                       script: "${pipelineConfig.PYTHON_DIR}/python.sh -u ${command}"
+                } else {
+                    bat label: "Running ${platform} clean",
+                        script: "${pipelineConfig.PYTHON_DIR}/python.cmd -u ${command}".replace('/','\\')
+                }
+            }
+        }
+    }
+}
+
+def Build(Map options, String platform, String type, String workspace) {
+    def command = "${options.BUILD_ENTRY_POINT} --platform ${platform} --type ${type}"
+    dir("${workspace}/${ENGINE_REPOSITORY_NAME}") {
+        if (env.IS_UNIX) {
+            sh label: "Running ${platform} ${type}",
+               script: "${options.PYTHON_DIR}/python.sh -u ${command}"
+        } else {
+            bat label: "Running ${platform} ${type}",
+                script: "${options.PYTHON_DIR}/python.cmd -u ${command}".replace('/','\\')
+        }
+    }
+}
+
+def TestMetrics(Map options, String workspace, String branchName, String repoName, String buildJobName, String outputDirectory, String configuration) {
+    catchError(buildResult: null, stageResult: null) {
+        def cmakeBuildDir = [workspace, ENGINE_REPOSITORY_NAME, outputDirectory].join('/')
+        dir("${workspace}/${ENGINE_REPOSITORY_NAME}") {
+            checkout scm: [
+                $class: 'GitSCM',
+                branches: [[name: '*/main']],
+                extensions: [[$class: 'RelativeTargetDirectory', relativeTargetDir: 'mars']],
+                userRemoteConfigs: [[url: "${env.MARS_REPO}", name: 'mars', credentialsId: "${env.GITHUB_USER}"]]
+            ]
+            withCredentials([usernamePassword(credentialsId: "${env.SERVICE_USER}", passwordVariable: 'apitoken', usernameVariable: 'username')]) {
+                def command = "${options.PYTHON_DIR}/python.cmd -u mars/scripts/python/ctest_test_metric_scraper.py -e jenkins.creds.user ${username} -e jenkins.creds.pass ${apitoken} ${cmakeBuildDir} ${branchName} %BUILD_NUMBER% AR ${configuration} ${repoName} "
+                bat label: "Publishing ${buildJobName} Test Metrics",
+                    script: command
+            }
+        }
+    }
+}
+
+def PostBuildCommonSteps(String workspace, boolean mount = true) {
+    echo 'Starting post-build common steps...'
+
+    if (mount) {
+        def pythonCmd = ''
+        if(env.IS_UNIX) pythonCmd = 'sudo -E python -u '
+        else pythonCmd = 'python -u '
+
+        try {
+            timeout(5) {
+                palSh("${pythonCmd} ${INCREMENTAL_BUILD_SCRIPT_PATH} --action unmount", 'Unmounting volume')
+            }
+        } catch (Exception e) {
+            echo "Unmount script error ${e}"
+        }
+    }
+}
+
+def CreateSetupStage(Map pipelineConfig, String projectName, String pipelineName, String branchName, String platformName, String jobName, Map environmentVars) {
+    return {
+        stage("Setup") {
+            PreBuildCommonSteps(pipelineConfig, projectName, pipelineName, branchName, platformName, jobName,  environmentVars['WORKSPACE'], environmentVars['MOUNT_VOLUME'])
+        }
+    }
+}
+
+def CreateBuildStage(Map pipelineConfig, String platformName, String jobName, Map environmentVars) {
+    return {
+        stage("${jobName}") {
+            Build(pipelineConfig, platformName, jobName, environmentVars['WORKSPACE'])
+        }
+    }
+}
+
+def CreateTestMetricsStage(Map pipelineConfig, String branchName, Map environmentVars, String buildJobName, String outputDirectory, String configuration) {
+    return {
+        stage("${buildJobName}_metrics") {
+            TestMetrics(pipelineConfig, environmentVars['WORKSPACE'], branchName, env.DEFAULT_REPOSITORY_NAME, buildJobName, outputDirectory, configuration)
+        }
+    }
+}
+
+def CreateTeardownStage(Map environmentVars) {
+    return {
+        stage("Teardown") {
+            PostBuildCommonSteps(environmentVars['WORKSPACE'], environmentVars['MOUNT_VOLUME'])
+        }
+    }
+}
+
+def projectName = ''
+def pipelineName = ''
+def branchName = ''
+def pipelineConfig = {}
+
+// Start Pipeline
+try {
+    stage('Setup Pipeline') {
+        node('controller') {
+            def envVarList = []
+            if(isUnix()) {
+                envVarList.add('IS_UNIX=1')
+            }
+            withEnv(envVarList) {
+                timestamps {
+                    (projectName, pipelineName) = GetRunningPipelineName(env.JOB_NAME) // env.JOB_NAME is the name of the job given by Jenkins
+
+                    if(env.BRANCH_NAME) {
+                        branchName = env.BRANCH_NAME
+                    } else {
+                        branchName = scm.branches[0].name // for non-multibranch pipelines
+                        env.BRANCH_NAME = branchName // so scripts that read this environment have it (e.g. incremental_build_util.py)
+                    }
+                    pipelineProperties.add(disableConcurrentBuilds())
+
+                    echo "Running \"${pipelineName}\" for \"${branchName}\"..."
+
+                    CheckoutBootstrapScripts(branchName)
+ 
+                    // Load configs
+                    pipelineConfig = LoadPipelineConfig(pipelineName, branchName)
+     
+                    // Add each platform as a parameter that the user can disable if needed
+                    pipelineConfig.platforms.each { platform ->
+                        pipelineParameters.add(booleanParam(defaultValue: true, description: '', name: platform.key))
+                    }
+                    pipelineProperties.add(parameters(pipelineParameters))
+                    properties(pipelineProperties)
+
+                    // Stash the INCREMENTAL_BUILD_SCRIPT_PATH since all nodes will use it
+                    stash name: 'incremental_build_script',
+                          includes: INCREMENTAL_BUILD_SCRIPT_PATH
+                 }    
+            }
+        }
+    }
+
+    if(env.BUILD_NUMBER == '1') {
+        // Exit pipeline early on the intial build. This allows Jenkins to load the pipeline for the branch and enables users 
+        // to select build parameters on their first actual build. See https://issues.jenkins.io/browse/JENKINS-41929
+        currentBuild.result = 'SUCCESS'
+        return
+    }
+
+    // Build and Post-Build Testing Stage
+    def buildConfigs = [:]
+
+    // Platform Builds run on EC2
+    pipelineConfig.platforms.each { platform ->
+        platform.value.build_types.each { build_job ->
+            if (IsJobEnabled(build_job, pipelineName, platform.key)) {   // User can filter jobs, jobs are tagged by pipeline
+                def envVars = GetBuildEnvVars(platform.value.PIPELINE_ENV ?: EMPTY_JSON, build_job.value.PIPELINE_ENV ?: EMPTY_JSON, pipelineName)
+                envVars['JOB_NAME'] = "${branchName}_${platform.key}_${build_job.key}" // backwards compatibility, some scripts rely on this
+                def nodeLabel = envVars['NODE_LABEL']
+
+                buildConfigs["${platform.key} [${build_job.key}]"] = { 
+                    node("${nodeLabel}") {
+                        if(isUnix()) { // Has to happen inside a node
+                            envVars['IS_UNIX'] = 1
+                        }                 
+                        withEnv(GetEnvStringList(envVars)) {
+                            timeout(time: envVars['TIMEOUT'], unit: 'MINUTES', activity: true) {
+                                try {
+                                    def build_job_name = build_job.key
+                                    
+                                    CreateSetupStage(pipelineConfig, projectName, pipelineName, branchName, platform.key, build_job.key, envVars).call()
+
+                                    if(build_job.value.steps) { //this is a pipe with many steps so create all the build stages
+                                        build_job.value.steps.each { build_step ->
+                                            build_job_name = build_step
+                                            CreateBuildStage(pipelineConfig,  platform.key, build_step, envVars).call()
+                                        }
+                                    } else {
+                                        CreateBuildStage(pipelineConfig,  platform.key, build_job.key, envVars).call()
+                                    }
+                                    
+                                    if (env.MARS_REPO && platform.key == 'Windows' && build_job_name.startsWith('test')) {
+                                        def output_directory = platform.value.build_types[build_job_name].PARAMETERS.OUTPUT_DIRECTORY
+                                        def configuration = platform.value.build_types[build_job_name].PARAMETERS.CONFIGURATION
+                                        CreateTestMetricsStage(pipelineConfig, branchName, envVars, build_job_name, output_directory, configuration).call()
+                                    }
+                                }
+                                catch(Exception e) {
+                                    //  https://github.com/jenkinsci/jenkins/blob/master/core/src/main/java/hudson/model/Result.java
+                                    //  {SUCCESS,UNSTABLE,FAILURE,NOT_BUILT,ABORTED}
+                                    def currentResult = envVars['ON_FAILURE_MARK'] ?: 'FAILURE'
+                                    if (currentResult == 'FAILURE') {
+                                        currentBuild.result = 'FAILURE'
+                                        error "FAILURE: ${e}"
+                                    } else if (currentResult == 'UNSTABLE') {
+                                        currentBuild.result = 'UNSTABLE'
+                                        unstable(message: "UNSTABLE: ${e}")
+                                    }
+                                }
+                                finally {
+                                    CreateTeardownStage(envVars).call()
+                                }
+                            }
+                        }
+                    }
+                }
+            }
+        }
+    }
+
+    timestamps {
+
+        stage('Build') {
+            parallel buildConfigs // Run parallel builds
+        }
+
+        echo 'All builds successful'
+    }
+}
+catch(Exception e) {
+    error "Exception: ${e}"
+}
+finally {
+    try {
+        if(env.SNS_TOPIC) {
+            snsPublish(
+                topicArn: env.SNS_TOPIC, 
+                subject:'Build Result', 
+                message:"${currentBuild.currentResult}:${BUILD_URL}:${env.RECREATE_VOLUME}:${env.CLEAN_OUTPUT_DIRECTORY}:${env.CLEAN_ASSETS}"
+            )
+        }
+        step([
+            $class: 'Mailer',
+            notifyEveryUnstableBuild: true,
+            sendToIndividuals: true,
+            recipients: emailextrecipients([
+                [$class: 'CulpritsRecipientProvider'],
+                [$class: 'RequesterRecipientProvider']
+            ])
+        ])
+    } catch(Exception e) {
+    }
+}

scripts/build/Jenkins/lumberyard.json

@@ -0,0 +1,12 @@
+{
+    "BUILD_ENTRY_POINT": "Tools/build/JenkinsScripts/build/ci_build.py",
+    "PIPELINE_CONFIGS": [
+        "Tools/build/JenkinsScripts/build/Platform/*/pipeline.json",
+        "restricted/*/Tools/build/JenkinsScripts/build/pipeline.json"
+    ],
+    "BUILD_CONFIGS": [
+        "Tools/build/JenkinsScripts/build/Platform/*/build_config.json",
+        "restricted/*/Tools/build/JenkinsScripts/build/build_config.json"
+    ],
+    "PYTHON_DIR": "python"
+}

scripts/build/bootstrap/incremental_build_util.py

@@ -94,6 +94,7 @@ def error(message):
 def parse_args():
     parser = argparse.ArgumentParser()
     parser.add_argument('-a', '--action', dest="action", help="Action (mount|unmount|delete)")
+    parser.add_argument('-proj', '--project', dest="project", help="Project")
     parser.add_argument('-pipe', '--pipeline', dest="pipeline", help="Pipeline")
     parser.add_argument('-b', '--branch', dest="branch", help="Branch")
     parser.add_argument('-plat', '--platform', dest="platform", help="Platform")
@@ -107,6 +108,8 @@ def parse_args():
         error('No action specified')
     args.action = args.action.lower()
     if args.action != 'unmount':
+        if args.project is None:
+            error('No project specified')
         if args.pipeline is None:
             error('No pipeline specified')
         if args.branch is None:
@@ -118,8 +121,8 @@ def parse_args():
     
     return args
 
-def get_mount_name(pipeline, branch, platform, build_type):
-    mount_name = "{}_{}_{}_{}".format(pipeline, branch, platform, build_type)
+def get_mount_name(project, pipeline, branch, platform, build_type):
+    mount_name = "{}_{}_{}_{}_{}".format(project, pipeline, branch, platform, build_type)
     mount_name = mount_name.replace('/','_').replace('\\','_')
     return mount_name
 
@@ -171,8 +174,8 @@ def delete_volume(ec2_client, volume_id):
     response = ec2_client.delete_volume(VolumeId=volume_id)
     print 'Volume {} deleted'.format(volume_id)
 
-def find_snapshot_id(ec2_client, pipeline, platform, build_type, disk_size):
-    mount_name = get_mount_name(pipeline, 'main', platform, build_type) # we take snapshots out of main
+def find_snapshot_id(ec2_client, project, pipeline, platform, build_type, disk_size):
+    mount_name = get_mount_name(project, pipeline, 'main', platform, build_type) # we take snapshots out of main
     response = ec2_client.describe_snapshots(Filters= [{
         'Name': 'tag:Name', 'Values': [mount_name]
     }])
@@ -188,9 +191,9 @@ def find_snapshot_id(ec2_client, pipeline, platform, build_type, disk_size):
                     snapshot_id = snapshot['SnapshotId']
     return snapshot_id
 
-def create_volume(ec2_client, availability_zone, pipeline, branch, platform, build_type, disk_size, disk_type):   
+def create_volume(ec2_client, availability_zone, project, pipeline, branch, platform, build_type, disk_size, disk_type):   
     # The actual EBS default calculation for IOps is a floating point number, the closest approxmiation is 4x of the disk size for simplicity
-    mount_name = get_mount_name(pipeline, branch, platform, build_type)
+    mount_name = get_mount_name(project, pipeline, branch, platform, build_type)
     pipeline_and_branch = get_pipeline_and_branch(pipeline, branch) 
     parameters = dict(
         AvailabilityZone = availability_zone,
@@ -199,6 +202,7 @@ def create_volume(ec2_client, availability_zone, pipeline, branch, platform, bui
             'ResourceType': 'volume',
             'Tags': [
                 { 'Key': 'Name', 'Value': mount_name },
+                { 'Key': 'Project', 'Value': project },
                 { 'Key': 'Pipeline', 'Value': pipeline },
                 { 'Key': 'BranchName', 'Value': branch },
                 { 'Key': 'Platform', 'Value': platform },
@@ -210,7 +214,7 @@ def create_volume(ec2_client, availability_zone, pipeline, branch, platform, bui
     if 'io1' in disk_type.lower(): 
         parameters['Iops'] = (4 * disk_size)
 
-    snapshot_id = find_snapshot_id(ec2_client, pipeline, platform, build_type, disk_size)
+    snapshot_id = find_snapshot_id(ec2_client, project, pipeline, platform, build_type, disk_size)
     if snapshot_id:
         parameters['SnapshotId'] = snapshot_id
         created = False
@@ -230,8 +234,8 @@ def create_volume(ec2_client, availability_zone, pipeline, branch, platform, bui
         time.sleep(1)
         response = ec2_client.describe_volumes(VolumeIds=[volume_id, ])
 
-    print("Volume {} created\n\tSnapshot: {}\n\tPipeline {}\n\tBranch {}\n\tPlatform: {}\n\tBuild type: {}"
-        .format(volume_id, snapshot_id, pipeline, branch, platform, build_type))
+    print("Volume {} created\n\tSnapshot: {}\n\tProject {}\n\tPipeline {}\n\tBranch {}\n\tPlatform: {}\n\tBuild type: {}"
+        .format(volume_id, snapshot_id, project, pipeline, branch, platform, build_type))
     return volume_id, created
 
 
@@ -355,7 +359,7 @@ def attach_ebs_and_create_partition_with_retry(volume, volume_id, ec2_instance_i
                 mount_volume(created)
         attempt += 1
 
-def mount_ebs(pipeline, branch, platform, build_type, disk_size, disk_type):
+def mount_ebs(project, pipeline, branch, platform, build_type, disk_size, disk_type):
     session = boto3.session.Session()
     region = session.region_name
     if region is None:
@@ -375,7 +379,7 @@ def mount_ebs(pipeline, branch, platform, build_type, disk_size, disk_type):
                 unmount_volume()
                 detach_volume(volume, ec2_instance_id, False) # Force unmounts should not be used, as that will cause the EBS block device driver to fail the remount
 
-    mount_name = get_mount_name(pipeline, branch, platform, build_type)
+    mount_name = get_mount_name(project, pipeline, branch, platform, build_type)
     response = ec2_client.describe_volumes(Filters=[{
         'Name': 'tag:Name', 'Values': [mount_name]
         }])
@@ -384,7 +388,7 @@ def mount_ebs(pipeline, branch, platform, build_type, disk_size, disk_type):
     if 'Volumes' in response and not len(response['Volumes']):
         print 'Volume for {} doesn\'t exist creating it...'.format(mount_name)
         # volume doesn't exist, create it
-        volume_id, created = create_volume(ec2_client, ec2_availability_zone, pipeline, branch, platform, build_type, disk_size, disk_type)
+        volume_id, created = create_volume(ec2_client, ec2_availability_zone, project, pipeline, branch, platform, build_type, disk_size, disk_type)
     else:
         volume = response['Volumes'][0]
         volume_id = volume['VolumeId']
@@ -392,7 +396,7 @@ def mount_ebs(pipeline, branch, platform, build_type, disk_size, disk_type):
         if (volume['Size'] != disk_size or volume['VolumeType'] != disk_type):
             print 'Override disk attributes does not match the existing volume, deleting {} and replacing the volume'.format(volume_id)
             delete_volume(ec2_client, volume_id)
-            volume_id, created = create_volume(ec2_client, ec2_availability_zone, pipeline, branch, platform, build_type, disk_size, disk_type)
+            volume_id, created = create_volume(ec2_client, ec2_availability_zone, project, pipeline, branch, platform, build_type, disk_size, disk_type)
         if len(volume['Attachments']):
             # this is bad we shouldn't be attached, we should have detached at the end of a build
             attachment = volume['Attachments'][0]
@@ -422,7 +426,7 @@ def mount_ebs(pipeline, branch, platform, build_type, disk_size, disk_type):
             print 'Error: EBS disk size reached to the allowed maximum disk size {}MB, please contact ly-infra@ and ly-build@ to investigate.'.format(MAX_EBS_DISK_SIZE)
             exit(1)
         print 'Recreating the EBS with disk size {}'.format(new_disk_size)
-        volume_id, created = create_volume(ec2_client, ec2_availability_zone, pipeline, branch, platform, build_type, new_disk_size, disk_type)
+        volume_id, created = create_volume(ec2_client, ec2_availability_zone, project, pipeline, branch, platform, build_type, new_disk_size, disk_type)
         volume = ec2_resource.Volume(volume_id)
         attach_ebs_and_create_partition_with_retry(volume, volume_id, ec2_instance_id, created)
 
@@ -454,7 +458,7 @@ def unmount_ebs():
         unmount_volume()
         detach_volume(volume, ec2_instance_id, False)
 
-def delete_ebs(pipeline, branch, platform, build_type):
+def delete_ebs(project, pipeline, branch, platform, build_type):
     unmount_ebs()
 
     session = boto3.session.Session()
@@ -466,7 +470,7 @@ def delete_ebs(pipeline, branch, platform, build_type):
     ec2_resource = boto3.resource('ec2', region_name=region)
     ec2_instance = ec2_resource.Instance(ec2_instance_id)
 
-    mount_name = get_mount_name(pipeline, branch, platform, build_type)
+    mount_name = get_mount_name(project, pipeline, branch, platform, build_type)
     response = ec2_client.describe_volumes(Filters=[
         { 'Name': 'tag:Name', 'Values': [mount_name] }
     ])
@@ -477,15 +481,15 @@ def delete_ebs(pipeline, branch, platform, build_type):
         delete_volume(ec2_client, volume_id)
 
 
-def main(action, pipeline, branch, platform, build_type, disk_size, disk_type):
+def main(action, project, pipeline, branch, platform, build_type, disk_size, disk_type):
     if action == 'mount':
-        mount_ebs(pipeline, branch, platform, build_type, disk_size, disk_type)
+        mount_ebs(project, pipeline, branch, platform, build_type, disk_size, disk_type)
     elif action == 'unmount':
         unmount_ebs()
     elif action == 'delete':
-        delete_ebs(pipeline, branch, platform, build_type)
+        delete_ebs(project, pipeline, branch, platform, build_type)
 
 if __name__ == "__main__":
     args = parse_args()
-    ret = main(args.action, args.pipeline, args.branch, args.platform, args.build_type, args.disk_size, args.disk_type)
+    ret = main(args.action, args.project, args.pipeline, args.branch, args.platform, args.build_type, args.disk_size, args.disk_type)
     sys.exit(ret)

scripts/build/package/package.py

@@ -10,10 +10,8 @@
 #
 import os
 import sys
-import glob_to_regex
 import zipfile
 import timeit
-import stat
 import progressbar
 from optparse import OptionParser
 from PackageEnv import PackageEnv
@@ -26,18 +24,6 @@ from glob3 import glob
 
 def package(options):
     package_env = PackageEnv(options.platform, options.type, options.package_env)
-    engine_root = package_env.get('ENGINE_ROOT')
-
-    if not package_env.get('SKIP_SCRUBBING'):
-        # Ask the validator code to tell us which files need to be removed from the package
-        prohibited_file_mask = get_prohibited_file_mask(options.platform, engine_root)
-
-        # Scrub files. This is destructive, but is necessary to allow the current file existance checks to work properly. Better to copy and then build, or to
-        # mask on sync, but this is what we have for now
-        scrub_files(package_env, prohibited_file_mask)
-
-        # validate files
-        validate_restricted_files(options.platform, options.type, package_env)
 
     # Override values in bootstrap.cfg for PC package
     override_bootstrap_cfg(package_env)
@@ -93,61 +79,6 @@ def override_bootstrap_cfg(package_env):
     print('{} updated with value {}'.format(bootstrap_path, replace_values))
 
 
-def get_prohibited_file_mask(platform, engine_root):
-    sys.path.append(os.path.join(engine_root, 'Tools', 'build', 'JenkinsScripts', 'distribution', 'scrubbing'))
-    from validator_data_LEGAL_REVIEW_REQUIRED import get_prohibited_platforms_for_package
-
-    # The list of prohibited platforms is controlled by the validator on a per-package basis
-    prohibited_platforms = get_prohibited_platforms_for_package(platform)
-    prohibited_platforms.append('all')
-    excludes_list = []
-    for p in prohibited_platforms:
-        platform_excludes = glob_to_regex.generate_excludes_for_platform(engine_root, p)
-        excludes_list.extend(platform_excludes)
-    prohibited_file_mask = re.compile('|'.join(excludes_list), re.IGNORECASE)
-    return prohibited_file_mask
-
-
-def scrub_files(package_env, prohibited_file_mask):
-    print('Perform the Code Scrubbing')
-    engine_root = package_env.get('ENGINE_ROOT')
-
-    success = True
-    for dirname, subFolders, files in os.walk(engine_root):
-        for filename in files:
-            full_path = os.path.join(dirname, filename)
-            if prohibited_file_mask.match(full_path):
-                try:
-                    print('Deleting: {}'.format(full_path))
-                    os.chmod(full_path, stat.S_IWRITE)
-                    os.unlink(full_path)
-                except:
-                    e = sys.exc_info()[0]
-                    sys.stderr.write('Error: could not delete {} ... aborting.\n'.format(full_path))
-                    sys.stderr.write('{}\n'.format(str(e)))
-                    success = False
-    if not success:
-        sys.stderr.write('ERROR: scrub_files failed\n')
-        sys.exit(1)
-
-
-def validate_restricted_files(package_platform, package_type, package_env):
-    print('Perform the Code Scrubbing')
-    engine_root = package_env.get('ENGINE_ROOT')
-
-    # Run validator
-    success = True
-    validator_path = os.path.join(engine_root, 'Tools/build/JenkinsScripts/distribution/scrubbing/validator.py')
-    python = get_python_path(package_env)
-    args = [python, validator_path, '--package_platform', package_platform, '--package_type', package_type, engine_root]
-    return_code = safe_execute_system_call(args)
-    if return_code != 0:
-        success = False
-    if not success:
-        error('Restricted file validator failed.')
-    print('Restricted file validator completed successfully.')
-
-
 def cmake_build(package_env):
     build_targets = package_env.get('BUILD_TARGETS')
     for build_target in build_targets:
@@ -161,15 +92,7 @@ def create_package(package_env, package_target):
     if file_list_type == 'All':
         filelist = os.path.join(cur_dir, 'package_filelists', package_target['FILE_LIST'])
     else:
-        # Search non-restricted platform first
         filelist = os.path.join(cur_dir, 'Platform', file_list_type, 'package_filelists', package_target['FILE_LIST'])
-        if not os.path.exists(filelist):
-            engine_root = package_env.get('ENGINE_ROOT')
-            # Use real path in case engine root is a symlink path
-            if os.name == 'posix' and os.path.islink(engine_root):
-                engine_root = os.readlink(engine_root)
-            rel_path = os.path.relpath(cur_dir, engine_root)
-            filelist = os.path.join(engine_root, 'restricted', file_list_type, rel_path, 'package_filelists', package_target['FILE_LIST'])
     with open(filelist, 'r') as source:
         data = json.load(source)
     lyengine = package_env.get('ENGINE_ROOT')
@@ -296,7 +219,3 @@ def parse_args():
 if __name__ == "__main__":
     (options, args) = parse_args()
     package(options)
-
-
-
-