From 8279b65622e39c1b546b13aeb58884c85adbf1e2 Mon Sep 17 00:00:00 2001 From: scottr Date: Fri, 18 Jun 2021 08:14:47 -0700 Subject: [PATCH 01/22] [cpack/stabilization/2106-jenkins] updated installer job temp dir to use WORKSPACE_TMP and added log file dump on error --- scripts/build/Platform/Windows/installer_windows.cmd | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/scripts/build/Platform/Windows/installer_windows.cmd b/scripts/build/Platform/Windows/installer_windows.cmd index a6ce15d59e..576e810fb7 100644 --- a/scripts/build/Platform/Windows/installer_windows.cmd +++ b/scripts/build/Platform/Windows/installer_windows.cmd @@ -21,7 +21,7 @@ IF NOT EXIST %OUTPUT_DIRECTORY% ( PUSHD %OUTPUT_DIRECTORY% REM Override the temporary directory used by wix to the workspace -SET "WIX_TEMP=!WORKSPACE!/temp/wix" +SET "WIX_TEMP=!WORKSPACE_TMP!/wix" IF NOT EXIST "%WIX_TEMP%" ( MKDIR "%WIX_TEMP%" ) @@ -52,7 +52,13 @@ IF ERRORLEVEL 1 ( ECHO [ci_build] "!CPACK_PATH!" -C %CONFIGURATION% "!CPACK_PATH!" -C %CONFIGURATION% -IF NOT %ERRORLEVEL%==0 GOTO :popd_error +IF NOT %ERRORLEVEL%==0 ( + rem dump the log file generated by cpack specifically for WIX + ECHO **************************************************************** + TYPE "_CPack_Packages\\win64\\WIX\\wix.log" + ECHO **************************************************************** + GOTO :popd_error +) POPD EXIT /b 0 From 059324add979e38b4ea44fce1b08219974bd6d96 Mon Sep 17 00:00:00 2001 From: scottr Date: Fri, 18 Jun 2021 08:15:59 -0700 Subject: [PATCH 02/22] [cpack/stabilization/2106-jenkins] updated installer job tags to run nightly --- scripts/build/Platform/Windows/build_config.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/build/Platform/Windows/build_config.json b/scripts/build/Platform/Windows/build_config.json index 552ef2c6fd..5b4915939c 100644 --- a/scripts/build/Platform/Windows/build_config.json +++ b/scripts/build/Platform/Windows/build_config.json @@ -308,7 +308,7 @@ }, "windows_installer": { "TAGS": [ - "package" + "nightly-clean" ], "COMMAND": "build_installer_windows.cmd", "PARAMETERS": { From 1d8fb2a7f7d30a061b908945a0592dc7621fb614 Mon Sep 17 00:00:00 2001 From: scottr Date: Fri, 18 Jun 2021 08:20:40 -0700 Subject: [PATCH 03/22] [cpack/stabilization/2106-jenkins] fixed installer framework include in jenkins job --- scripts/build/Platform/Windows/build_config.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/build/Platform/Windows/build_config.json b/scripts/build/Platform/Windows/build_config.json index 5b4915939c..feab627d96 100644 --- a/scripts/build/Platform/Windows/build_config.json +++ b/scripts/build/Platform/Windows/build_config.json @@ -314,7 +314,7 @@ "PARAMETERS": { "CONFIGURATION": "profile", "OUTPUT_DIRECTORY": "build\\windows_vs2019", - "CMAKE_OPTIONS": "-G \"Visual Studio 16 2019\" -DCMAKE_SYSTEM_VERSION=10.0 -DLY_UNITY_BUILD=TRUE -DLY_DISABLE_TEST_MODULES=TRUE -DCPACK_WIX_ROOT=\"!WIX!\"", + "CMAKE_OPTIONS": "-G \"Visual Studio 16 2019\" -DCMAKE_SYSTEM_VERSION=10.0 -DLY_UNITY_BUILD=TRUE -DLY_DISABLE_TEST_MODULES=TRUE -DCPACK_WIX_ROOT=\"!WIX! \"", "CMAKE_LY_PROJECTS": "", "CMAKE_TARGET": "ALL_BUILD", "CMAKE_NATIVE_BUILD_ARGS": "/m /nologo" From 6e727e6ab0d7c942ad6183639014ac770e685f82 Mon Sep 17 00:00:00 2001 From: scottr Date: Fri, 18 Jun 2021 08:25:03 -0700 Subject: [PATCH 04/22] [cpack/stabilization/2106-jenkins] various changes to shorten cpack install path on windows --- cmake/Packaging.cmake | 3 +-- cmake/Platform/Windows/PackagingPostBuild.cmake | 2 +- cmake/Platform/Windows/Packaging_windows.cmake | 6 ++++++ 3 files changed, 8 insertions(+), 3 deletions(-) diff --git a/cmake/Packaging.cmake b/cmake/Packaging.cmake index 2f001cf7ee..e7782ac284 100644 --- a/cmake/Packaging.cmake +++ b/cmake/Packaging.cmake @@ -32,8 +32,7 @@ set(CPACK_PACKAGE_VENDOR "TBD") set(CPACK_PACKAGE_VERSION "${LY_VERSION_STRING}") set(CPACK_PACKAGE_DESCRIPTION_SUMMARY "Installation Tool") -string(TOLOWER ${PROJECT_NAME} _project_name_lower) -set(CPACK_PACKAGE_FILE_NAME "${_project_name_lower}_${LY_VERSION_STRING}_installer") +string(TOLOWER "${CPACK_PACKAGE_NAME}_${CPACK_PACKAGE_VERSION}" CPACK_PACKAGE_FILE_NAME) set(DEFAULT_LICENSE_NAME "Apache-2.0") set(DEFAULT_LICENSE_FILE "${CMAKE_CURRENT_SOURCE_DIR}/LICENSE.txt") diff --git a/cmake/Platform/Windows/PackagingPostBuild.cmake b/cmake/Platform/Windows/PackagingPostBuild.cmake index 1dcbedcba7..63f60994c5 100644 --- a/cmake/Platform/Windows/PackagingPostBuild.cmake +++ b/cmake/Platform/Windows/PackagingPostBuild.cmake @@ -17,7 +17,7 @@ string(REPLACE "/" "\\" _fixed_package_install_dir ${CPACK_PACKAGE_INSTALL_DIREC set(_cpack_wix_out_dir ${CPACK_TOPLEVEL_DIRECTORY}) set(_bootstrap_out_dir "${CPACK_TOPLEVEL_DIRECTORY}/bootstrap") -set(_bootstrap_filename "${CPACK_PACKAGE_FILE_NAME}.exe") +set(_bootstrap_filename "${CPACK_PACKAGE_FILE_NAME}_installer.exe") set(_bootstrap_output_file ${_cpack_wix_out_dir}/${_bootstrap_filename}) set(_ext_flags diff --git a/cmake/Platform/Windows/Packaging_windows.cmake b/cmake/Platform/Windows/Packaging_windows.cmake index 6dc806ff56..95c00a3d0f 100644 --- a/cmake/Platform/Windows/Packaging_windows.cmake +++ b/cmake/Platform/Windows/Packaging_windows.cmake @@ -26,6 +26,12 @@ set(_cmake_package_name "cmake-${CPACK_DESIRED_CMAKE_VERSION}-windows-x86_64") set(CPACK_CMAKE_PACKAGE_FILE "${_cmake_package_name}.zip") set(CPACK_CMAKE_PACKAGE_HASH "15a49e2ab81c1822d75b1b1a92f7863f58e31f6d6aac1c4103eef2b071be3112") +# workaround for shortening the path cpack installs to by stripping the platform directory and forcing monolithic +# mode to strip out component folders. this unfortunately is the closest we can get to changing the install location +# as CPACK_PACKAGING_INSTALL_PREFIX/CPACK_SET_DESTDIR isn't supported for the WiX generator +set(CPACK_TOPLEVEL_TAG "") +set(CPACK_MONOLITHIC_INSTALL ON) + # CPack will generate the WiX product/upgrade GUIDs further down the chain if they weren't supplied # however, they are unique for each run. instead, let's do the auto generation here and add it to # the cache for run persistence and have the ability to detect if they are still being used. From 071ed777bd767573961667872858a5da71c71436 Mon Sep 17 00:00:00 2001 From: scottr Date: Fri, 18 Jun 2021 08:33:48 -0700 Subject: [PATCH 05/22] [cpack/stabilization/2106-jenkins] removed enforcement of aws profile for s3 upload installer artifacts --- cmake/Packaging.cmake | 4 ---- 1 file changed, 4 deletions(-) diff --git a/cmake/Packaging.cmake b/cmake/Packaging.cmake index e7782ac284..39d49653fe 100644 --- a/cmake/Packaging.cmake +++ b/cmake/Packaging.cmake @@ -136,10 +136,6 @@ if(LY_INSTALLER_UPLOAD_URL) set(CPACK_AWS_PROFILE ${LY_INSTALLER_AWS_PROFILE}) elseif (DEFINED ENV{LY_INSTALLER_AWS_PROFILE}) set(CPACK_AWS_PROFILE $ENV{LY_INSTALLER_AWS_PROFILE}) - else() - message(FATAL_ERROR - "An AWS profile is required for installer S3 uploading. Please provide " - "one via LY_INSTALLER_AWS_PROFILE CLI argument or environment variable") endif() strip_trailing_slash(${LY_INSTALLER_UPLOAD_URL} LY_INSTALLER_UPLOAD_URL) From 7011bb66f18267ffb1890d61ba5b917125a06bf1 Mon Sep 17 00:00:00 2001 From: scottr Date: Fri, 18 Jun 2021 08:36:28 -0700 Subject: [PATCH 06/22] [cpack/stabilization/2106-jenkins] missed cpack log file path shortening --- scripts/build/Platform/Windows/installer_windows.cmd | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/build/Platform/Windows/installer_windows.cmd b/scripts/build/Platform/Windows/installer_windows.cmd index 576e810fb7..d4f3482aba 100644 --- a/scripts/build/Platform/Windows/installer_windows.cmd +++ b/scripts/build/Platform/Windows/installer_windows.cmd @@ -55,7 +55,7 @@ ECHO [ci_build] "!CPACK_PATH!" -C %CONFIGURATION% IF NOT %ERRORLEVEL%==0 ( rem dump the log file generated by cpack specifically for WIX ECHO **************************************************************** - TYPE "_CPack_Packages\\win64\\WIX\\wix.log" + TYPE "_CPack_Packages\\WIX\\wix.log" ECHO **************************************************************** GOTO :popd_error ) From 1519180cec947ff144ff34ae41c20c83cefc335a Mon Sep 17 00:00:00 2001 From: scottr Date: Fri, 18 Jun 2021 08:39:40 -0700 Subject: [PATCH 07/22] [cpack/stabilization/2106-jenkins] updated installer job params to make the online version --- scripts/build/Platform/Windows/build_config.json | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/scripts/build/Platform/Windows/build_config.json b/scripts/build/Platform/Windows/build_config.json index feab627d96..38a310e7cf 100644 --- a/scripts/build/Platform/Windows/build_config.json +++ b/scripts/build/Platform/Windows/build_config.json @@ -314,7 +314,8 @@ "PARAMETERS": { "CONFIGURATION": "profile", "OUTPUT_DIRECTORY": "build\\windows_vs2019", - "CMAKE_OPTIONS": "-G \"Visual Studio 16 2019\" -DCMAKE_SYSTEM_VERSION=10.0 -DLY_UNITY_BUILD=TRUE -DLY_DISABLE_TEST_MODULES=TRUE -DCPACK_WIX_ROOT=\"!WIX! \"", + "CMAKE_OPTIONS": "-G \"Visual Studio 16 2019\" -DCMAKE_SYSTEM_VERSION=10.0 -DLY_UNITY_BUILD=TRUE -DLY_DISABLE_TEST_MODULES=TRUE", + "EXTRA_CMAKE_OPTIONS": "-DCPACK_WIX_ROOT=\"!WIX! \" -DLY_INSTALLER_DOWNLOAD_URL=https://dkb1uj4hs9ikv.cloudfront.net -DLY_INSTALLER_LICENSE_URL=https://example.com", "CMAKE_LY_PROJECTS": "", "CMAKE_TARGET": "ALL_BUILD", "CMAKE_NATIVE_BUILD_ARGS": "/m /nologo" From 8b68aa9f5a9ceaac65d99bbcdec6ecec25aa5687 Mon Sep 17 00:00:00 2001 From: scottr Date: Fri, 18 Jun 2021 16:55:22 -0700 Subject: [PATCH 08/22] [cpack/jenkins-main] rework build tagging to include git repo info in installer job --- cmake/Packaging.cmake | 6 +- .../Platform/Windows/PackagingPostBuild.cmake | 23 ++++++- .../Platform/Windows/installer_windows.cmd | 17 ++++- scripts/build/tools/generate_build_tag.py | 66 +++++++++++++++++++ 4 files changed, 105 insertions(+), 7 deletions(-) create mode 100644 scripts/build/tools/generate_build_tag.py diff --git a/cmake/Packaging.cmake b/cmake/Packaging.cmake index 39d49653fe..1656b49e88 100644 --- a/cmake/Packaging.cmake +++ b/cmake/Packaging.cmake @@ -120,7 +120,7 @@ function(strip_trailing_slash in_url out_url) endif() endfunction() -set(_versioned_target_url_tag ${LY_VERSION_STRING}/${PAL_HOST_PLATFORM_NAME}) +set(CPACK_VERSIONED_TARGET_TAG "${PAL_HOST_PLATFORM_NAME}-${LY_VERSION_STRING}") if(NOT LY_INSTALLER_UPLOAD_URL AND DEFINED ENV{LY_INSTALLER_UPLOAD_URL}) set(LY_INSTALLER_UPLOAD_URL $ENV{LY_INSTALLER_UPLOAD_URL}) @@ -139,7 +139,7 @@ if(LY_INSTALLER_UPLOAD_URL) endif() strip_trailing_slash(${LY_INSTALLER_UPLOAD_URL} LY_INSTALLER_UPLOAD_URL) - set(CPACK_UPLOAD_URL ${LY_INSTALLER_UPLOAD_URL}/${_versioned_target_url_tag}) + set(CPACK_UPLOAD_URL ${LY_INSTALLER_UPLOAD_URL}) endif() # IMPORTANT: required to be included AFTER setting all property overrides @@ -189,7 +189,7 @@ if(LY_INSTALLER_DOWNLOAD_URL) # this will set the following variables: CPACK_DOWNLOAD_SITE, CPACK_DOWNLOAD_ALL, and CPACK_UPLOAD_DIRECTORY (local) cpack_configure_downloads( - ${LY_INSTALLER_DOWNLOAD_URL}/${_versioned_target_url_tag} + ${LY_INSTALLER_DOWNLOAD_URL} UPLOAD_DIRECTORY ${CMAKE_BINARY_DIR}/_CPack_Uploads # to match the _CPack_Packages directory ALL ) diff --git a/cmake/Platform/Windows/PackagingPostBuild.cmake b/cmake/Platform/Windows/PackagingPostBuild.cmake index 63f60994c5..a28ce51f41 100644 --- a/cmake/Platform/Windows/PackagingPostBuild.cmake +++ b/cmake/Platform/Windows/PackagingPostBuild.cmake @@ -24,10 +24,24 @@ set(_ext_flags -ext WixBalExtension ) +set(_build_id_file ${CPACK_BINARY_DIR}/build_id.txt) +if(EXISTS ${_build_id_file}) + file(READ ${_build_id_file} _build_id) + + set(_full_download_url ${CPACK_DOWNLOAD_SITE}/${_build_id}/${CPACK_VERSIONED_TARGET_TAG}) + set(_full_upload_url ${CPACK_UPLOAD_URL}/${_build_id}/${CPACK_VERSIONED_TARGET_TAG}) +else() + # format: YYYY-MM-DD-HHmm UTC + string(TIMESTAMP _timestamp "%Y%-%m-%d-%H%m" UTC) + + set(_full_download_url ${CPACK_DOWNLOAD_SITE}/notag/${_timestamp}/${CPACK_VERSIONED_TARGET_TAG}) + set(_full_upload_url ${CPACK_UPLOAD_URL}/notag/${_timestamp}/${CPACK_VERSIONED_TARGET_TAG}) +endif() + set(_addtional_defines -dCPACK_BOOTSTRAP_THEME_FILE=${CPACK_BINARY_DIR}/BootstrapperTheme -dCPACK_BOOTSTRAP_UPGRADE_GUID=${CPACK_WIX_BOOTSTRAP_UPGRADE_GUID} - -dCPACK_DOWNLOAD_SITE=${CPACK_DOWNLOAD_SITE} + -dCPACK_DOWNLOAD_SITE=${_full_download_url} -dCPACK_LOCAL_INSTALLER_DIR=${_cpack_wix_out_dir} -dCPACK_PACKAGE_FILE_NAME=${CPACK_PACKAGE_FILE_NAME} -dCPACK_PACKAGE_INSTALL_DIRECTORY=${_fixed_package_install_dir} @@ -109,7 +123,7 @@ file(TO_NATIVE_PATH "${_root_path}/scripts/build/tools/upload_to_s3.py" _upload_ file(TO_NATIVE_PATH "${_cpack_wix_out_dir}" _cpack_wix_out_dir) # strip the scheme and extract the bucket/key prefix from the URL -string(REPLACE "s3://" "" _stripped_url ${CPACK_UPLOAD_URL}) +string(REPLACE "s3://" "" _stripped_url ${_full_upload_url}) string(REPLACE "/" ";" _tokens ${_stripped_url}) list(POP_FRONT _tokens _bucket) @@ -127,12 +141,15 @@ set(_upload_command --profile ${CPACK_AWS_PROFILE} ) +message(STATUS "Uploading artifacts to ${_full_upload_url}") execute_process( COMMAND ${_upload_command} RESULT_VARIABLE _upload_result ERROR_VARIABLE _upload_errors ) -if (NOT ${_upload_result} EQUAL 0) +if (${_upload_result} EQUAL 0) + message(STATUS "Artifact uploading complete!") +else() message(FATAL_ERROR "An error occurred uploading artifacts. ${_upload_errors}") endif() diff --git a/scripts/build/Platform/Windows/installer_windows.cmd b/scripts/build/Platform/Windows/installer_windows.cmd index d4f3482aba..5caf8b7b80 100644 --- a/scripts/build/Platform/Windows/installer_windows.cmd +++ b/scripts/build/Platform/Windows/installer_windows.cmd @@ -50,10 +50,25 @@ IF ERRORLEVEL 1 ( GOTO :popd_error ) +REM generate the build ID for artifact upload tagging +PUSHD "%~dp0/../../../.." +SET "ENGINE_ROOT=%cd%" +POPD + +SET "GEN_BUILD_ID_SCRIPT=%ENGINE_ROOT%/scripts/build/tools/generate_build_tag.py" +SET "BUILD_ID_FILE=_CPack/build_id.txt" + +ECHO [ci_build] "%ENGINE_ROOT%/python/python.cmd" -u "%GEN_BUILD_ID_SCRIPT%" "%BUILD_ID_FILE%" +CALL "%ENGINE_ROOT%/python/python.cmd" -u "%GEN_BUILD_ID_SCRIPT%" "%BUILD_ID_FILE%" +IF ERRORLEVEL 1 ( + ECHO [ci_build] Failed to generate build ID + GOTO :popd_error +) + ECHO [ci_build] "!CPACK_PATH!" -C %CONFIGURATION% "!CPACK_PATH!" -C %CONFIGURATION% IF NOT %ERRORLEVEL%==0 ( - rem dump the log file generated by cpack specifically for WIX + REM dump the log file generated by cpack specifically for WIX ECHO **************************************************************** TYPE "_CPack_Packages\\WIX\\wix.log" ECHO **************************************************************** diff --git a/scripts/build/tools/generate_build_tag.py b/scripts/build/tools/generate_build_tag.py new file mode 100644 index 0000000000..56817c064f --- /dev/null +++ b/scripts/build/tools/generate_build_tag.py @@ -0,0 +1,66 @@ +# +# All or portions of this file Copyright (c) Amazon.com, Inc. or its affiliates or +# its licensors. +# +# For complete copyright and license terms please see the LICENSE at the root of this +# distribution (the "License"). All use of this software is governed by the License, +# or, if provided, by the license below or the license accompanying this file. Do not +# remove or modify any license notices. This file is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# + +import argparse +import os +import pathlib +import shutil +import subprocess +import sys + + +def run_git_command(args, repo_root): + + process = subprocess.run(['git', *args], + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + cwd=repo_root, + env=os.environ.copy(), + universal_newlines=True, + ) + + if process.returncode != 0: + print( + f'An error occurred while running a command\n' + f'Command: git {subprocess.list2cmdline(args)}\n' + f'Return Code: {process.returncode}\n' + f'Error: {process.stderr}' + ) + exit(1) + + output = process.stdout.splitlines() + # something went wrong and we somehow got more information then requested + if len(output) != 1: + print(f'Unexpected output received from command: git {subprocess.list2cmdline(args)}') + exit(1) + + return output[0].strip('"') + + +if __name__ == "__main__": + parser = argparse.ArgumentParser(description='Generates a build ID based on the state of the git repository') + parser.add_argument('output_file', help='Path to the output file where the build ID will be written to') + parsed_args = parser.parse_args() + + repo_root = os.path.realpath(os.path.join(os.path.dirname(__file__), '..', '..')) + + branch = run_git_command(['branch', '--show-current'], repo_root) + branch = branch.replace('/', '-') + + commit_hash = run_git_command(['show', '--format="%h"', '--no-patch'], repo_root) + + # include the commit date to allow some sensible way of sorting + commit_date = run_git_command(['show', '-s', '--format="%cs"', commit_hash], repo_root) + + with open(parsed_args.output_file, 'w') as out_file: + out_file.write(f'{branch}/{commit_date}-{commit_hash}') + + sys.exit(0) From a057cbb03a6c2681eed5388b023f5a0140a86187 Mon Sep 17 00:00:00 2001 From: scottr Date: Fri, 18 Jun 2021 17:16:26 -0700 Subject: [PATCH 09/22] [cpack/jenkins-main] updated installer job params --- scripts/build/Platform/Windows/build_config.json | 5 ++++- scripts/build/Platform/Windows/build_installer_windows.cmd | 2 ++ 2 files changed, 6 insertions(+), 1 deletion(-) diff --git a/scripts/build/Platform/Windows/build_config.json b/scripts/build/Platform/Windows/build_config.json index 38a310e7cf..674d6c11bd 100644 --- a/scripts/build/Platform/Windows/build_config.json +++ b/scripts/build/Platform/Windows/build_config.json @@ -316,9 +316,12 @@ "OUTPUT_DIRECTORY": "build\\windows_vs2019", "CMAKE_OPTIONS": "-G \"Visual Studio 16 2019\" -DCMAKE_SYSTEM_VERSION=10.0 -DLY_UNITY_BUILD=TRUE -DLY_DISABLE_TEST_MODULES=TRUE", "EXTRA_CMAKE_OPTIONS": "-DCPACK_WIX_ROOT=\"!WIX! \" -DLY_INSTALLER_DOWNLOAD_URL=https://dkb1uj4hs9ikv.cloudfront.net -DLY_INSTALLER_LICENSE_URL=https://example.com", + "BUILD_TYPE": "staging", + "BUILD_ID": "spectra-prism", "CMAKE_LY_PROJECTS": "", "CMAKE_TARGET": "ALL_BUILD", - "CMAKE_NATIVE_BUILD_ARGS": "/m /nologo" + "CMAKE_NATIVE_BUILD_ARGS": "/m /nologo", + "REGION": "us-west-2" } }, "project_enginesource_profile_vs2019": { diff --git a/scripts/build/Platform/Windows/build_installer_windows.cmd b/scripts/build/Platform/Windows/build_installer_windows.cmd index 4f31fee085..beada918dc 100644 --- a/scripts/build/Platform/Windows/build_installer_windows.cmd +++ b/scripts/build/Platform/Windows/build_installer_windows.cmd @@ -10,6 +10,8 @@ REM remove or modify any license notices. This file is distributed on an "AS IS" REM WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. REM +set "LY_INSTALLER_UPLOAD_URL=s3://%BUILD_ID%-%BUILD_TYPE%-%REGION%" + CALL "%~dp0build_windows.cmd" IF NOT %ERRORLEVEL%==0 GOTO :error From a0d6e2919857a11e2957abe342c2c5f797b1686b Mon Sep 17 00:00:00 2001 From: scottr Date: Fri, 18 Jun 2021 18:05:26 -0700 Subject: [PATCH 10/22] [cpack/jenkins-main] additional logging to build id generator script --- scripts/build/tools/generate_build_tag.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/scripts/build/tools/generate_build_tag.py b/scripts/build/tools/generate_build_tag.py index 56817c064f..0996f613a7 100644 --- a/scripts/build/tools/generate_build_tag.py +++ b/scripts/build/tools/generate_build_tag.py @@ -39,7 +39,11 @@ def run_git_command(args, repo_root): output = process.stdout.splitlines() # something went wrong and we somehow got more information then requested if len(output) != 1: - print(f'Unexpected output received from command: git {subprocess.list2cmdline(args)}') + print(f'Unexpected output received.\n' + f'Command: git {subprocess.list2cmdline(args)}\n' + f'Output:{process.stdout}\n' + f'Error: {process.stderr}\n' + ) exit(1) return output[0].strip('"') From c507422760997dd035d24a29ce553dc33ff628d1 Mon Sep 17 00:00:00 2001 From: scottr Date: Fri, 18 Jun 2021 20:52:04 -0700 Subject: [PATCH 11/22] [cpack/jenkins-main] update build url tag generation to use existing env vars from Jenkins --- scripts/build/Jenkins/Jenkinsfile | 6 +++++ .../build/Platform/Windows/build_config.json | 6 ++--- .../Windows/build_installer_windows.cmd | 2 +- .../Platform/Windows/installer_windows.cmd | 26 ++++++++++--------- 4 files changed, 23 insertions(+), 17 deletions(-) diff --git a/scripts/build/Jenkins/Jenkinsfile b/scripts/build/Jenkins/Jenkinsfile index 693cf31727..295c92e4ab 100644 --- a/scripts/build/Jenkins/Jenkinsfile +++ b/scripts/build/Jenkins/Jenkinsfile @@ -256,6 +256,12 @@ def CheckoutRepo(boolean disableSubmodules = false) { env.CHANGE_ID = readFile file: 'commitid' env.CHANGE_ID = env.CHANGE_ID.trim() palRm('commitid') + + // CHANGE_DATE is used by the installer to provide some ability to sort tagged builds in addition to BRANCH_NAME and CHANGE_ID + palSh("git show -s --format=\"%cs\" ${env.CHANGE_ID} > commitdate", 'Getting commit date') + env.CHANGE_DATE = readFile file: 'commitdate' + env.CHANGE_DATE = env.CHANGE_DATE.trim() + palRm('commitdate') } def PreBuildCommonSteps(Map pipelineConfig, String repositoryName, String projectName, String pipeline, String branchName, String platform, String buildType, String workspace, boolean mount = true, boolean disableSubmodules = false) { diff --git a/scripts/build/Platform/Windows/build_config.json b/scripts/build/Platform/Windows/build_config.json index 674d6c11bd..c13c6939a7 100644 --- a/scripts/build/Platform/Windows/build_config.json +++ b/scripts/build/Platform/Windows/build_config.json @@ -316,12 +316,10 @@ "OUTPUT_DIRECTORY": "build\\windows_vs2019", "CMAKE_OPTIONS": "-G \"Visual Studio 16 2019\" -DCMAKE_SYSTEM_VERSION=10.0 -DLY_UNITY_BUILD=TRUE -DLY_DISABLE_TEST_MODULES=TRUE", "EXTRA_CMAKE_OPTIONS": "-DCPACK_WIX_ROOT=\"!WIX! \" -DLY_INSTALLER_DOWNLOAD_URL=https://dkb1uj4hs9ikv.cloudfront.net -DLY_INSTALLER_LICENSE_URL=https://example.com", - "BUILD_TYPE": "staging", - "BUILD_ID": "spectra-prism", + "CPACK_BUCKET": "spectra-prism-staging-us-west-2", "CMAKE_LY_PROJECTS": "", "CMAKE_TARGET": "ALL_BUILD", - "CMAKE_NATIVE_BUILD_ARGS": "/m /nologo", - "REGION": "us-west-2" + "CMAKE_NATIVE_BUILD_ARGS": "/m /nologo" } }, "project_enginesource_profile_vs2019": { diff --git a/scripts/build/Platform/Windows/build_installer_windows.cmd b/scripts/build/Platform/Windows/build_installer_windows.cmd index beada918dc..5d1408bd60 100644 --- a/scripts/build/Platform/Windows/build_installer_windows.cmd +++ b/scripts/build/Platform/Windows/build_installer_windows.cmd @@ -10,7 +10,7 @@ REM remove or modify any license notices. This file is distributed on an "AS IS" REM WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. REM -set "LY_INSTALLER_UPLOAD_URL=s3://%BUILD_ID%-%BUILD_TYPE%-%REGION%" +SET "LY_INSTALLER_UPLOAD_URL=s3://%CPACK_BUCKET%" CALL "%~dp0build_windows.cmd" IF NOT %ERRORLEVEL%==0 GOTO :error diff --git a/scripts/build/Platform/Windows/installer_windows.cmd b/scripts/build/Platform/Windows/installer_windows.cmd index 5caf8b7b80..5deb4f931d 100644 --- a/scripts/build/Platform/Windows/installer_windows.cmd +++ b/scripts/build/Platform/Windows/installer_windows.cmd @@ -50,21 +50,23 @@ IF ERRORLEVEL 1 ( GOTO :popd_error ) -REM generate the build ID for artifact upload tagging -PUSHD "%~dp0/../../../.." -SET "ENGINE_ROOT=%cd%" -POPD - -SET "GEN_BUILD_ID_SCRIPT=%ENGINE_ROOT%/scripts/build/tools/generate_build_tag.py" -SET "BUILD_ID_FILE=_CPack/build_id.txt" +REM use the git info to generate an identifier used by the online installer urls +SET "BUILD_ID_FILE=_CPack\\build_id.txt" -ECHO [ci_build] "%ENGINE_ROOT%/python/python.cmd" -u "%GEN_BUILD_ID_SCRIPT%" "%BUILD_ID_FILE%" -CALL "%ENGINE_ROOT%/python/python.cmd" -u "%GEN_BUILD_ID_SCRIPT%" "%BUILD_ID_FILE%" -IF ERRORLEVEL 1 ( - ECHO [ci_build] Failed to generate build ID - GOTO :popd_error +SET BRANCH_ID="" +REM limit branch separators to 6 +FOR /F "tokens=1-6 delims=/" %%a in ("!BRANCH_NAME!") DO ( + SET BRANCH_ID=%%a + if NOT "%%b"=="" SET BRANCH_ID=!BRANCH_ID!-%%b + if NOT "%%c"=="" SET BRANCH_ID=!BRANCH_ID!-%%c + if NOT "%%d"=="" SET BRANCH_ID=!BRANCH_ID!-%%d + if NOT "%%e"=="" SET BRANCH_ID=!BRANCH_ID!-%%e + if NOT "%%f"=="" SET BRANCH_ID=!BRANCH_ID!-%%f ) +REM write out the build ID to disk so cpack can consume it +ECHO %BRANCH_ID%/%CHANGE_DATE%-%CHANGE_ID:~0,7% > "%BUILD_ID_FILE%" + ECHO [ci_build] "!CPACK_PATH!" -C %CONFIGURATION% "!CPACK_PATH!" -C %CONFIGURATION% IF NOT %ERRORLEVEL%==0 ( From 68d90366a86c60aa319f488c3466d581e9eaffa6 Mon Sep 17 00:00:00 2001 From: scottr Date: Sat, 19 Jun 2021 07:15:23 -0700 Subject: [PATCH 12/22] [cpack/jenkins-main] missed escaping a character in git command for jenkinsfile --- scripts/build/Jenkins/Jenkinsfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/build/Jenkins/Jenkinsfile b/scripts/build/Jenkins/Jenkinsfile index 295c92e4ab..053383aa72 100644 --- a/scripts/build/Jenkins/Jenkinsfile +++ b/scripts/build/Jenkins/Jenkinsfile @@ -258,7 +258,7 @@ def CheckoutRepo(boolean disableSubmodules = false) { palRm('commitid') // CHANGE_DATE is used by the installer to provide some ability to sort tagged builds in addition to BRANCH_NAME and CHANGE_ID - palSh("git show -s --format=\"%cs\" ${env.CHANGE_ID} > commitdate", 'Getting commit date') + palSh("git show -s --format=\"%%cs\" ${env.CHANGE_ID} > commitdate", 'Getting commit date') env.CHANGE_DATE = readFile file: 'commitdate' env.CHANGE_DATE = env.CHANGE_DATE.trim() palRm('commitdate') From f03b3623d4676bbc999d5be8b42da482010fb393 Mon Sep 17 00:00:00 2001 From: scottr Date: Sun, 20 Jun 2021 11:45:41 -0700 Subject: [PATCH 13/22] [cpack/jenkins-main] fixed git date format on unix machines in jenkinsfile --- scripts/build/Jenkins/Jenkinsfile | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/scripts/build/Jenkins/Jenkinsfile b/scripts/build/Jenkins/Jenkinsfile index 053383aa72..f8507d94f4 100644 --- a/scripts/build/Jenkins/Jenkinsfile +++ b/scripts/build/Jenkins/Jenkinsfile @@ -258,7 +258,10 @@ def CheckoutRepo(boolean disableSubmodules = false) { palRm('commitid') // CHANGE_DATE is used by the installer to provide some ability to sort tagged builds in addition to BRANCH_NAME and CHANGE_ID - palSh("git show -s --format=\"%%cs\" ${env.CHANGE_ID} > commitdate", 'Getting commit date') + commitDateFmt = '%%cs' + if (env.IS_UNIX) commitDateFmt = '%cs' + + palSh("git show -s --format=${commitDateFmt} ${env.CHANGE_ID} > commitdate", 'Getting commit date') env.CHANGE_DATE = readFile file: 'commitdate' env.CHANGE_DATE = env.CHANGE_DATE.trim() palRm('commitdate') From 646fc30dab17bda704e2faa50be2f4167c60d959 Mon Sep 17 00:00:00 2001 From: scottr Date: Sun, 20 Jun 2021 13:31:59 -0700 Subject: [PATCH 14/22] [cpack/jenkins-main] build tag generator script now uses jenkins env vars and matches commands used --- scripts/build/tools/generate_build_tag.py | 42 ++++++++++++----------- 1 file changed, 22 insertions(+), 20 deletions(-) diff --git a/scripts/build/tools/generate_build_tag.py b/scripts/build/tools/generate_build_tag.py index 0996f613a7..78158e430c 100644 --- a/scripts/build/tools/generate_build_tag.py +++ b/scripts/build/tools/generate_build_tag.py @@ -32,39 +32,41 @@ def run_git_command(args, repo_root): f'An error occurred while running a command\n' f'Command: git {subprocess.list2cmdline(args)}\n' f'Return Code: {process.returncode}\n' - f'Error: {process.stderr}' + f'Error: {process.stderr}', + file=sys.stderr ) exit(1) output = process.stdout.splitlines() - # something went wrong and we somehow got more information then requested - if len(output) != 1: - print(f'Unexpected output received.\n' - f'Command: git {subprocess.list2cmdline(args)}\n' - f'Output:{process.stdout}\n' - f'Error: {process.stderr}\n' - ) - exit(1) + if not output: + print(f'No output received for command: git {subprocess.list2cmdline(args)}.') + output return output[0].strip('"') if __name__ == "__main__": - parser = argparse.ArgumentParser(description='Generates a build ID based on the state of the git repository') - parser.add_argument('output_file', help='Path to the output file where the build ID will be written to') - parsed_args = parser.parse_args() - + ''' + Generates a build ID based on the state of the git repository. Will first attempt to use + existing environment variable (e.g. BRANCH_NAME, CHANGE_ID, CHANGE_DATE) before falling + back to running git commands directly + ''' repo_root = os.path.realpath(os.path.join(os.path.dirname(__file__), '..', '..')) - branch = run_git_command(['branch', '--show-current'], repo_root) + branch = os.environ.get('BRANCH_NAME') + if not branch: + branch = run_git_command(['rev-parse', '--abbrev-ref', 'HEAD'], repo_root) branch = branch.replace('/', '-') - commit_hash = run_git_command(['show', '--format="%h"', '--no-patch'], repo_root) + commit_hash = os.environ.get('CHANGE_ID') + if not commit_hash: + commit_hash = run_git_command(['rev-parse', 'HEAD'], repo_root) + commit_hash = commit_hash[0:9] # include the commit date to allow some sensible way of sorting - commit_date = run_git_command(['show', '-s', '--format="%cs"', commit_hash], repo_root) - - with open(parsed_args.output_file, 'w') as out_file: - out_file.write(f'{branch}/{commit_date}-{commit_hash}') + commit_date = os.environ.get('CHANGE_DATE') + if not commit_date: + commit_date = run_git_command(['show', '-s', '--format=%cs', commit_hash], repo_root) - sys.exit(0) + print(f'{branch}/{commit_date}-{commit_hash}') + exit(0) From 21d92a414e895531e2e2ad4e8b591f50810270ed Mon Sep 17 00:00:00 2001 From: scottr Date: Sun, 20 Jun 2021 13:38:16 -0700 Subject: [PATCH 15/22] [cpack/jenkins-main] update commit date format string to something that works on all machines --- scripts/build/Jenkins/Jenkinsfile | 4 ++-- scripts/build/tools/generate_build_tag.py | 3 ++- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/scripts/build/Jenkins/Jenkinsfile b/scripts/build/Jenkins/Jenkinsfile index f8507d94f4..7bb26fdf8c 100644 --- a/scripts/build/Jenkins/Jenkinsfile +++ b/scripts/build/Jenkins/Jenkinsfile @@ -258,8 +258,8 @@ def CheckoutRepo(boolean disableSubmodules = false) { palRm('commitid') // CHANGE_DATE is used by the installer to provide some ability to sort tagged builds in addition to BRANCH_NAME and CHANGE_ID - commitDateFmt = '%%cs' - if (env.IS_UNIX) commitDateFmt = '%cs' + commitDateFmt = '%%cI' + if (env.IS_UNIX) commitDateFmt = '%cI' palSh("git show -s --format=${commitDateFmt} ${env.CHANGE_ID} > commitdate", 'Getting commit date') env.CHANGE_DATE = readFile file: 'commitdate' diff --git a/scripts/build/tools/generate_build_tag.py b/scripts/build/tools/generate_build_tag.py index 78158e430c..b37e180ea6 100644 --- a/scripts/build/tools/generate_build_tag.py +++ b/scripts/build/tools/generate_build_tag.py @@ -66,7 +66,8 @@ if __name__ == "__main__": # include the commit date to allow some sensible way of sorting commit_date = os.environ.get('CHANGE_DATE') if not commit_date: - commit_date = run_git_command(['show', '-s', '--format=%cs', commit_hash], repo_root) + commit_date = run_git_command(['show', '-s', '--format=%cI', commit_hash], repo_root) + commit_date = commit_date[0:10] print(f'{branch}/{commit_date}-{commit_hash}') exit(0) From e8f250bed80f2904c0057358f20e2b024b9b658b Mon Sep 17 00:00:00 2001 From: scottr Date: Sun, 20 Jun 2021 14:31:10 -0700 Subject: [PATCH 16/22] [cpack/jenkins-main] rework build tag generation to be at cpack time through LY_INSTALLER_AUTO_GEN_TAG --- cmake/Packaging.cmake | 30 +++++++++-- cmake/PackagingConfig.cmake | 52 +++++++++++++++++++ .../Platform/Windows/PackagingPostBuild.cmake | 20 ++----- .../Platform/Windows/installer_windows.cmd | 17 ------ 4 files changed, 80 insertions(+), 39 deletions(-) create mode 100644 cmake/PackagingConfig.cmake diff --git a/cmake/Packaging.cmake b/cmake/Packaging.cmake index 1656b49e88..b22b08d64b 100644 --- a/cmake/Packaging.cmake +++ b/cmake/Packaging.cmake @@ -14,11 +14,29 @@ if(NOT PAL_TRAIT_BUILD_CPACK_SUPPORTED) endif() # public facing options will be used for conversion into cpack specific ones below. -set(LY_INSTALLER_DOWNLOAD_URL "" CACHE STRING "URL embedded into the installer to download additional artifacts") set(LY_INSTALLER_LICENSE_URL "" CACHE STRING "Optionally embed a link to the license instead of raw text") + +set(LY_INSTALLER_AUTO_GEN_TAG OFF CACHE BOOL +"Automatically generate a build tag based on the git repo and append it to the download/upload URLs. \ +Format: /-" +) + +set(LY_INSTALLER_DOWNLOAD_URL "" CACHE STRING +"Base URL embedded into the installer to download additional artifacts, the host target and version \ +number will automatically appended as '/'. If LY_INSTALLER_AUTO_GEN_TAG is set, the \ +full URL format will be: //" +) + set(LY_INSTALLER_UPLOAD_URL "" CACHE STRING - "URL used to automatically upload the artifacts. Can also be set via LY_INSTALLER_UPLOAD_URL environment variable. Currently only accepts S3 URLs e.g. s3:///") -set(LY_INSTALLER_AWS_PROFILE "" CACHE STRING "AWS CLI profile for uploading artifacts. Can also be set via LY_INSTALLER_AWS_PROFILE environment variable.") +"Base URL used to upload the installer artifacts after generation, the host target and version number \ +will automatically appended as '/'. If LY_INSTALLER_AUTO_GEN_TAG is set, the full URL \ +format will be: //. Can also be set via LY_INSTALLER_UPLOAD_URL environment \ +variable. Currently only accepts S3 URLs e.g. s3:///" +) + +set(LY_INSTALLER_AWS_PROFILE "" CACHE STRING +"AWS CLI profile for uploading artifacts. Can also be set via LY_INSTALLER_AWS_PROFILE environment variable." +) set(CPACK_DESIRED_CMAKE_VERSION 3.20.2) @@ -46,6 +64,10 @@ set(CPACK_PACKAGE_INSTALL_DIRECTORY "${CPACK_PACKAGE_NAME}/${CPACK_PACKAGE_VERSI set(CPACK_SOURCE_DIR ${CMAKE_CURRENT_SOURCE_DIR}/cmake) set(CPACK_BINARY_DIR ${CMAKE_BINARY_DIR}/_CPack) # to match other CPack out dirs +# this config file allows the dynamic setting of cpack variables at cpack-time instead of cmake configure +set(CPACK_PROJECT_CONFIG_FILE ${CPACK_SOURCE_DIR}/PackagingConfig.cmake) +set(CPACK_AUTO_GEN_TAG ${LY_INSTALLER_AUTO_GEN_TAG}) + # attempt to apply platform specific settings ly_get_absolute_pal_filename(pal_dir ${CPACK_SOURCE_DIR}/Platform/${PAL_HOST_PLATFORM_NAME}) include(${pal_dir}/Packaging_${PAL_HOST_PLATFORM_NAME_LOWERCASE}.cmake) @@ -120,8 +142,6 @@ function(strip_trailing_slash in_url out_url) endif() endfunction() -set(CPACK_VERSIONED_TARGET_TAG "${PAL_HOST_PLATFORM_NAME}-${LY_VERSION_STRING}") - if(NOT LY_INSTALLER_UPLOAD_URL AND DEFINED ENV{LY_INSTALLER_UPLOAD_URL}) set(LY_INSTALLER_UPLOAD_URL $ENV{LY_INSTALLER_UPLOAD_URL}) endif() diff --git a/cmake/PackagingConfig.cmake b/cmake/PackagingConfig.cmake new file mode 100644 index 0000000000..aaf7554115 --- /dev/null +++ b/cmake/PackagingConfig.cmake @@ -0,0 +1,52 @@ +# +# All or portions of this file Copyright (c) Amazon.com, Inc. or its affiliates or +# its licensors. +# +# For complete copyright and license terms please see the LICENSE at the root of this +# distribution (the "License"). All use of this software is governed by the License, +# or, if provided, by the license below or the license accompanying this file. Do not +# remove or modify any license notices. This file is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# + +set(_target_name ${CMAKE_HOST_SYSTEM_NAME}) +if(${_target_name} STREQUAL Darwin) + set(_target_name Mac) +endif() + +if(CPACK_AUTO_GEN_TAG) + set(_python_script python.sh) + if(${_target_name} STREQUAL WINDOWS) + set(_python_script python.cmd) + endif() + + file(REAL_PATH "${CPACK_SOURCE_DIR}/.." _root_path) + file(TO_NATIVE_PATH "${_root_path}/python/${_python_script}" _python_cmd) + file(TO_NATIVE_PATH "${_root_path}/scripts/build/tools/generate_build_tag.py" _gen_tag_script) + + execute_process( + COMMAND ${_python_cmd} -s -u ${_gen_tag_script} + RESULT_VARIABLE _gen_tag_result + OUTPUT_VARIABLE _gen_tag_output + ERROR_VARIABLE _gen_tag_errors + OUTPUT_STRIP_TRAILING_WHITESPACE + ERROR_STRIP_TRAILING_WHITESPACE + ) + + if (NOT ${_gen_tag_result} EQUAL 0) + message(FATAL_ERROR "Failed to generate build tag!") + endif() + + set(_url_tag ${_gen_tag_output}) +else() + set(_url_tag ${CPACK_PACKAGE_VERSION}) +endif() + +set(_full_tag ${_url_tag}/${_target_name}) + +if(CPACK_DOWNLOAD_SITE) + set(CPACK_DOWNLOAD_SITE ${CPACK_DOWNLOAD_SITE}/${_full_tag}) +endif() +if(CPACK_UPLOAD_URL) + set(CPACK_UPLOAD_URL ${CPACK_UPLOAD_URL}/${_full_tag}) +endif() \ No newline at end of file diff --git a/cmake/Platform/Windows/PackagingPostBuild.cmake b/cmake/Platform/Windows/PackagingPostBuild.cmake index a28ce51f41..fae9e00e84 100644 --- a/cmake/Platform/Windows/PackagingPostBuild.cmake +++ b/cmake/Platform/Windows/PackagingPostBuild.cmake @@ -24,24 +24,10 @@ set(_ext_flags -ext WixBalExtension ) -set(_build_id_file ${CPACK_BINARY_DIR}/build_id.txt) -if(EXISTS ${_build_id_file}) - file(READ ${_build_id_file} _build_id) - - set(_full_download_url ${CPACK_DOWNLOAD_SITE}/${_build_id}/${CPACK_VERSIONED_TARGET_TAG}) - set(_full_upload_url ${CPACK_UPLOAD_URL}/${_build_id}/${CPACK_VERSIONED_TARGET_TAG}) -else() - # format: YYYY-MM-DD-HHmm UTC - string(TIMESTAMP _timestamp "%Y%-%m-%d-%H%m" UTC) - - set(_full_download_url ${CPACK_DOWNLOAD_SITE}/notag/${_timestamp}/${CPACK_VERSIONED_TARGET_TAG}) - set(_full_upload_url ${CPACK_UPLOAD_URL}/notag/${_timestamp}/${CPACK_VERSIONED_TARGET_TAG}) -endif() - set(_addtional_defines -dCPACK_BOOTSTRAP_THEME_FILE=${CPACK_BINARY_DIR}/BootstrapperTheme -dCPACK_BOOTSTRAP_UPGRADE_GUID=${CPACK_WIX_BOOTSTRAP_UPGRADE_GUID} - -dCPACK_DOWNLOAD_SITE=${_full_download_url} + -dCPACK_DOWNLOAD_SITE=${CPACK_DOWNLOAD_SITE} -dCPACK_LOCAL_INSTALLER_DIR=${_cpack_wix_out_dir} -dCPACK_PACKAGE_FILE_NAME=${CPACK_PACKAGE_FILE_NAME} -dCPACK_PACKAGE_INSTALL_DIRECTORY=${_fixed_package_install_dir} @@ -123,7 +109,7 @@ file(TO_NATIVE_PATH "${_root_path}/scripts/build/tools/upload_to_s3.py" _upload_ file(TO_NATIVE_PATH "${_cpack_wix_out_dir}" _cpack_wix_out_dir) # strip the scheme and extract the bucket/key prefix from the URL -string(REPLACE "s3://" "" _stripped_url ${_full_upload_url}) +string(REPLACE "s3://" "" _stripped_url ${CPACK_UPLOAD_URL}) string(REPLACE "/" ";" _tokens ${_stripped_url}) list(POP_FRONT _tokens _bucket) @@ -141,7 +127,7 @@ set(_upload_command --profile ${CPACK_AWS_PROFILE} ) -message(STATUS "Uploading artifacts to ${_full_upload_url}") +message(STATUS "Uploading artifacts to ${CPACK_UPLOAD_URL}") execute_process( COMMAND ${_upload_command} RESULT_VARIABLE _upload_result diff --git a/scripts/build/Platform/Windows/installer_windows.cmd b/scripts/build/Platform/Windows/installer_windows.cmd index 5deb4f931d..368658ea95 100644 --- a/scripts/build/Platform/Windows/installer_windows.cmd +++ b/scripts/build/Platform/Windows/installer_windows.cmd @@ -50,23 +50,6 @@ IF ERRORLEVEL 1 ( GOTO :popd_error ) -REM use the git info to generate an identifier used by the online installer urls -SET "BUILD_ID_FILE=_CPack\\build_id.txt" - -SET BRANCH_ID="" -REM limit branch separators to 6 -FOR /F "tokens=1-6 delims=/" %%a in ("!BRANCH_NAME!") DO ( - SET BRANCH_ID=%%a - if NOT "%%b"=="" SET BRANCH_ID=!BRANCH_ID!-%%b - if NOT "%%c"=="" SET BRANCH_ID=!BRANCH_ID!-%%c - if NOT "%%d"=="" SET BRANCH_ID=!BRANCH_ID!-%%d - if NOT "%%e"=="" SET BRANCH_ID=!BRANCH_ID!-%%e - if NOT "%%f"=="" SET BRANCH_ID=!BRANCH_ID!-%%f -) - -REM write out the build ID to disk so cpack can consume it -ECHO %BRANCH_ID%/%CHANGE_DATE%-%CHANGE_ID:~0,7% > "%BUILD_ID_FILE%" - ECHO [ci_build] "!CPACK_PATH!" -C %CONFIGURATION% "!CPACK_PATH!" -C %CONFIGURATION% IF NOT %ERRORLEVEL%==0 ( From 3ca839a58094cd20372814081e021e6864f30b8c Mon Sep 17 00:00:00 2001 From: scottr Date: Sun, 20 Jun 2021 14:45:03 -0700 Subject: [PATCH 17/22] [cpack/jenkins-main] update installer job params and how they are applied --- scripts/build/Platform/Windows/build_config.json | 4 ++-- scripts/build/Platform/Windows/build_installer_windows.cmd | 2 -- scripts/build/Platform/Windows/installer_windows.cmd | 4 ++-- 3 files changed, 4 insertions(+), 6 deletions(-) diff --git a/scripts/build/Platform/Windows/build_config.json b/scripts/build/Platform/Windows/build_config.json index c13c6939a7..45cea2b5d8 100644 --- a/scripts/build/Platform/Windows/build_config.json +++ b/scripts/build/Platform/Windows/build_config.json @@ -314,8 +314,8 @@ "PARAMETERS": { "CONFIGURATION": "profile", "OUTPUT_DIRECTORY": "build\\windows_vs2019", - "CMAKE_OPTIONS": "-G \"Visual Studio 16 2019\" -DCMAKE_SYSTEM_VERSION=10.0 -DLY_UNITY_BUILD=TRUE -DLY_DISABLE_TEST_MODULES=TRUE", - "EXTRA_CMAKE_OPTIONS": "-DCPACK_WIX_ROOT=\"!WIX! \" -DLY_INSTALLER_DOWNLOAD_URL=https://dkb1uj4hs9ikv.cloudfront.net -DLY_INSTALLER_LICENSE_URL=https://example.com", + "CMAKE_OPTIONS": "-G \"Visual Studio 16 2019\" -DCMAKE_SYSTEM_VERSION=10.0 -DLY_UNITY_BUILD=TRUE -DLY_DISABLE_TEST_MODULES=TRUE -DCPACK_WIX_ROOT=\"!WIX! \"", + "EXTRA_CMAKE_OPTIONS": "-DLY_INSTALLER_AUTO_GEN_TAG=ON -DLY_INSTALLER_DOWNLOAD_URL=https://dkb1uj4hs9ikv.cloudfront.net -DLY_INSTALLER_LICENSE_URL=https://example.com", "CPACK_BUCKET": "spectra-prism-staging-us-west-2", "CMAKE_LY_PROJECTS": "", "CMAKE_TARGET": "ALL_BUILD", diff --git a/scripts/build/Platform/Windows/build_installer_windows.cmd b/scripts/build/Platform/Windows/build_installer_windows.cmd index 5d1408bd60..4f31fee085 100644 --- a/scripts/build/Platform/Windows/build_installer_windows.cmd +++ b/scripts/build/Platform/Windows/build_installer_windows.cmd @@ -10,8 +10,6 @@ REM remove or modify any license notices. This file is distributed on an "AS IS" REM WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. REM -SET "LY_INSTALLER_UPLOAD_URL=s3://%CPACK_BUCKET%" - CALL "%~dp0build_windows.cmd" IF NOT %ERRORLEVEL%==0 GOTO :error diff --git a/scripts/build/Platform/Windows/installer_windows.cmd b/scripts/build/Platform/Windows/installer_windows.cmd index 368658ea95..1254e38d3a 100644 --- a/scripts/build/Platform/Windows/installer_windows.cmd +++ b/scripts/build/Platform/Windows/installer_windows.cmd @@ -50,8 +50,8 @@ IF ERRORLEVEL 1 ( GOTO :popd_error ) -ECHO [ci_build] "!CPACK_PATH!" -C %CONFIGURATION% -"!CPACK_PATH!" -C %CONFIGURATION% +ECHO [ci_build] "!CPACK_PATH!" -C %CONFIGURATION% -D CPACK_UPLOAD_URL=s3://%CPACK_BUCKET% %CPACK_OPTIONS% +"!CPACK_PATH!" -C %CONFIGURATION% -D CPACK_UPLOAD_URL=s3://%CPACK_BUCKET% %CPACK_OPTIONS% IF NOT %ERRORLEVEL%==0 ( REM dump the log file generated by cpack specifically for WIX ECHO **************************************************************** From 25f99bce79bdac0dc0f10c37d400c9727730454b Mon Sep 17 00:00:00 2001 From: scottr Date: Sun, 20 Jun 2021 14:48:48 -0700 Subject: [PATCH 18/22] [cpack/jenkins-main] fixed incorrect host name comparison string for python script name --- cmake/PackagingConfig.cmake | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/cmake/PackagingConfig.cmake b/cmake/PackagingConfig.cmake index aaf7554115..86a9667c36 100644 --- a/cmake/PackagingConfig.cmake +++ b/cmake/PackagingConfig.cmake @@ -16,7 +16,7 @@ endif() if(CPACK_AUTO_GEN_TAG) set(_python_script python.sh) - if(${_target_name} STREQUAL WINDOWS) + if(${_target_name} STREQUAL Windows) set(_python_script python.cmd) endif() @@ -34,7 +34,7 @@ if(CPACK_AUTO_GEN_TAG) ) if (NOT ${_gen_tag_result} EQUAL 0) - message(FATAL_ERROR "Failed to generate build tag!") + message(FATAL_ERROR "Failed to generate build tag! Errors: ${_gen_tag_errors}") endif() set(_url_tag ${_gen_tag_output}) From 587aa58b8793922e0ed3c603de4d06bc4d3b575e Mon Sep 17 00:00:00 2001 From: scottr Date: Sun, 20 Jun 2021 15:16:18 -0700 Subject: [PATCH 19/22] [cpack/jenkins-main] fixed bug with implicit aws upload profile --- cmake/Platform/Windows/PackagingPostBuild.cmake | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/cmake/Platform/Windows/PackagingPostBuild.cmake b/cmake/Platform/Windows/PackagingPostBuild.cmake index fae9e00e84..89c9837cf5 100644 --- a/cmake/Platform/Windows/PackagingPostBuild.cmake +++ b/cmake/Platform/Windows/PackagingPostBuild.cmake @@ -124,9 +124,12 @@ set(_upload_command --file_regex="${_file_regex}" --bucket ${_bucket} --key_prefix ${_prefix} - --profile ${CPACK_AWS_PROFILE} ) +if(CPACK_AWS_PROFILE) + list(APPEND _upload_command --profile ${CPACK_AWS_PROFILE}) +endif() + message(STATUS "Uploading artifacts to ${CPACK_UPLOAD_URL}") execute_process( COMMAND ${_upload_command} From b3a5afe8a326accfbc1e0f3bbc20fc564980c470 Mon Sep 17 00:00:00 2001 From: scottr Date: Sun, 20 Jun 2021 17:20:57 -0700 Subject: [PATCH 20/22] [cpack/jenkins-main] trailing new line in PackagingConfig.cmake --- cmake/PackagingConfig.cmake | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cmake/PackagingConfig.cmake b/cmake/PackagingConfig.cmake index 86a9667c36..b340219ff8 100644 --- a/cmake/PackagingConfig.cmake +++ b/cmake/PackagingConfig.cmake @@ -49,4 +49,4 @@ if(CPACK_DOWNLOAD_SITE) endif() if(CPACK_UPLOAD_URL) set(CPACK_UPLOAD_URL ${CPACK_UPLOAD_URL}/${_full_tag}) -endif() \ No newline at end of file +endif() From b089facc232ec020c7566077c3440a5fc03221f3 Mon Sep 17 00:00:00 2001 From: scottr Date: Sun, 20 Jun 2021 20:00:23 -0700 Subject: [PATCH 21/22] [cpack/jenkins-main] fixed bug where some resources were not embedded in the bootstrapper --- cmake/Platform/Windows/Packaging/Bootstrapper.wxs | 8 ++++++++ cmake/Platform/Windows/Packaging/BootstrapperTheme.xml.in | 2 +- cmake/Platform/Windows/PackagingPostBuild.cmake | 1 + 3 files changed, 10 insertions(+), 1 deletion(-) diff --git a/cmake/Platform/Windows/Packaging/Bootstrapper.wxs b/cmake/Platform/Windows/Packaging/Bootstrapper.wxs index c7146b1b7f..cb9d86ede8 100644 --- a/cmake/Platform/Windows/Packaging/Bootstrapper.wxs +++ b/cmake/Platform/Windows/Packaging/Bootstrapper.wxs @@ -17,6 +17,10 @@ Value="[ProgramFiles64Folder]$(var.CPACK_PACKAGE_INSTALL_DIRECTORY)" bal:Overridable="yes"/> + + + + + + @@ -34,6 +40,8 @@ ThemeFile="$(var.CPACK_BOOTSTRAP_THEME_FILE).xml" LocalizationFile="$(var.CPACK_BOOTSTRAP_THEME_FILE).wxl" ShowVersion="yes" /> + + diff --git a/cmake/Platform/Windows/Packaging/BootstrapperTheme.xml.in b/cmake/Platform/Windows/Packaging/BootstrapperTheme.xml.in index f9a177b646..93984d06b8 100644 --- a/cmake/Platform/Windows/Packaging/BootstrapperTheme.xml.in +++ b/cmake/Platform/Windows/Packaging/BootstrapperTheme.xml.in @@ -76,7 +76,7 @@ - + #(loc.FailureHeader) #(loc.FailureInstallHeader) #(loc.FailureUninstallHeader) diff --git a/cmake/Platform/Windows/PackagingPostBuild.cmake b/cmake/Platform/Windows/PackagingPostBuild.cmake index 89c9837cf5..83dff8354e 100644 --- a/cmake/Platform/Windows/PackagingPostBuild.cmake +++ b/cmake/Platform/Windows/PackagingPostBuild.cmake @@ -32,6 +32,7 @@ set(_addtional_defines -dCPACK_PACKAGE_FILE_NAME=${CPACK_PACKAGE_FILE_NAME} -dCPACK_PACKAGE_INSTALL_DIRECTORY=${_fixed_package_install_dir} -dCPACK_WIX_PRODUCT_LOGO=${CPACK_WIX_PRODUCT_LOGO} + -dCPACK_RESOURCE_PATH=${CPACK_SOURCE_DIR}/Platform/Windows/Packaging ) if(CPACK_LICENSE_URL) From 00bb33587ae40edfeb6260290e06fbbe1915c6a9 Mon Sep 17 00:00:00 2001 From: scottr Date: Sun, 20 Jun 2021 20:17:11 -0700 Subject: [PATCH 22/22] [cpack/jenkins-main] fixed bug applying some cpack build job parameters --- scripts/build/Platform/Windows/installer_windows.cmd | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/scripts/build/Platform/Windows/installer_windows.cmd b/scripts/build/Platform/Windows/installer_windows.cmd index 1254e38d3a..f7d4394dc8 100644 --- a/scripts/build/Platform/Windows/installer_windows.cmd +++ b/scripts/build/Platform/Windows/installer_windows.cmd @@ -50,8 +50,12 @@ IF ERRORLEVEL 1 ( GOTO :popd_error ) -ECHO [ci_build] "!CPACK_PATH!" -C %CONFIGURATION% -D CPACK_UPLOAD_URL=s3://%CPACK_BUCKET% %CPACK_OPTIONS% -"!CPACK_PATH!" -C %CONFIGURATION% -D CPACK_UPLOAD_URL=s3://%CPACK_BUCKET% %CPACK_OPTIONS% +IF NOT "%CPACK_BUCKET%"=="" ( + SET "CPACK_OPTIONS=-D CPACK_UPLOAD_URL=s3://%CPACK_BUCKET% %CPACK_OPTIONS%" +) + +ECHO [ci_build] "!CPACK_PATH!" -C %CONFIGURATION% %CPACK_OPTIONS% +"!CPACK_PATH!" -C %CONFIGURATION% %CPACK_OPTIONS% IF NOT %ERRORLEVEL%==0 ( REM dump the log file generated by cpack specifically for WIX ECHO ****************************************************************