diff --git a/cmake/Packaging.cmake b/cmake/Packaging.cmake index 7e45fa6bba..a341cb5f1a 100644 --- a/cmake/Packaging.cmake +++ b/cmake/Packaging.cmake @@ -14,11 +14,29 @@ if(NOT PAL_TRAIT_BUILD_CPACK_SUPPORTED) endif() # public facing options will be used for conversion into cpack specific ones below. -set(LY_INSTALLER_DOWNLOAD_URL "" CACHE STRING "URL embedded into the installer to download additional artifacts") set(LY_INSTALLER_LICENSE_URL "" CACHE STRING "Optionally embed a link to the license instead of raw text") + +set(LY_INSTALLER_AUTO_GEN_TAG OFF CACHE BOOL +"Automatically generate a build tag based on the git repo and append it to the download/upload URLs. \ +Format: /-" +) + +set(LY_INSTALLER_DOWNLOAD_URL "" CACHE STRING +"Base URL embedded into the installer to download additional artifacts, the host target and version \ +number will automatically appended as '/'. If LY_INSTALLER_AUTO_GEN_TAG is set, the \ +full URL format will be: //" +) + set(LY_INSTALLER_UPLOAD_URL "" CACHE STRING - "URL used to automatically upload the artifacts. Can also be set via LY_INSTALLER_UPLOAD_URL environment variable. Currently only accepts S3 URLs e.g. s3:///") -set(LY_INSTALLER_AWS_PROFILE "" CACHE STRING "AWS CLI profile for uploading artifacts. Can also be set via LY_INSTALLER_AWS_PROFILE environment variable.") +"Base URL used to upload the installer artifacts after generation, the host target and version number \ +will automatically appended as '/'. If LY_INSTALLER_AUTO_GEN_TAG is set, the full URL \ +format will be: //. Can also be set via LY_INSTALLER_UPLOAD_URL environment \ +variable. Currently only accepts S3 URLs e.g. s3:///" +) + +set(LY_INSTALLER_AWS_PROFILE "" CACHE STRING +"AWS CLI profile for uploading artifacts. Can also be set via LY_INSTALLER_AWS_PROFILE environment variable." +) set(CPACK_DESIRED_CMAKE_VERSION 3.20.2) @@ -32,8 +50,7 @@ set(CPACK_PACKAGE_VENDOR "O3DE Binary Project a Series of LF Projects, LLC") set(CPACK_PACKAGE_VERSION "${LY_VERSION_STRING}") set(CPACK_PACKAGE_DESCRIPTION_SUMMARY "Installation Tool") -string(TOLOWER ${PROJECT_NAME} _project_name_lower) -set(CPACK_PACKAGE_FILE_NAME "${_project_name_lower}_${LY_VERSION_STRING}_installer") +string(TOLOWER "${CPACK_PACKAGE_NAME}_${CPACK_PACKAGE_VERSION}" CPACK_PACKAGE_FILE_NAME) set(DEFAULT_LICENSE_NAME "Apache-2.0") set(DEFAULT_LICENSE_FILE "${CMAKE_CURRENT_SOURCE_DIR}/LICENSE.txt") @@ -47,6 +64,10 @@ set(CPACK_PACKAGE_INSTALL_DIRECTORY "${CPACK_PACKAGE_NAME}/${CPACK_PACKAGE_VERSI set(CPACK_SOURCE_DIR ${CMAKE_CURRENT_SOURCE_DIR}/cmake) set(CPACK_BINARY_DIR ${CMAKE_BINARY_DIR}/_CPack) # to match other CPack out dirs +# this config file allows the dynamic setting of cpack variables at cpack-time instead of cmake configure +set(CPACK_PROJECT_CONFIG_FILE ${CPACK_SOURCE_DIR}/PackagingConfig.cmake) +set(CPACK_AUTO_GEN_TAG ${LY_INSTALLER_AUTO_GEN_TAG}) + # attempt to apply platform specific settings ly_get_absolute_pal_filename(pal_dir ${CPACK_SOURCE_DIR}/Platform/${PAL_HOST_PLATFORM_NAME}) include(${pal_dir}/Packaging_${PAL_HOST_PLATFORM_NAME_LOWERCASE}.cmake) @@ -121,8 +142,6 @@ function(strip_trailing_slash in_url out_url) endif() endfunction() -set(_versioned_target_url_tag ${LY_VERSION_STRING}/${PAL_HOST_PLATFORM_NAME}) - if(NOT LY_INSTALLER_UPLOAD_URL AND DEFINED ENV{LY_INSTALLER_UPLOAD_URL}) set(LY_INSTALLER_UPLOAD_URL $ENV{LY_INSTALLER_UPLOAD_URL}) endif() @@ -137,14 +156,10 @@ if(LY_INSTALLER_UPLOAD_URL) set(CPACK_AWS_PROFILE ${LY_INSTALLER_AWS_PROFILE}) elseif (DEFINED ENV{LY_INSTALLER_AWS_PROFILE}) set(CPACK_AWS_PROFILE $ENV{LY_INSTALLER_AWS_PROFILE}) - else() - message(FATAL_ERROR - "An AWS profile is required for installer S3 uploading. Please provide " - "one via LY_INSTALLER_AWS_PROFILE CLI argument or environment variable") endif() strip_trailing_slash(${LY_INSTALLER_UPLOAD_URL} LY_INSTALLER_UPLOAD_URL) - set(CPACK_UPLOAD_URL ${LY_INSTALLER_UPLOAD_URL}/${_versioned_target_url_tag}) + set(CPACK_UPLOAD_URL ${LY_INSTALLER_UPLOAD_URL}) endif() # IMPORTANT: required to be included AFTER setting all property overrides @@ -194,7 +209,7 @@ if(LY_INSTALLER_DOWNLOAD_URL) # this will set the following variables: CPACK_DOWNLOAD_SITE, CPACK_DOWNLOAD_ALL, and CPACK_UPLOAD_DIRECTORY (local) cpack_configure_downloads( - ${LY_INSTALLER_DOWNLOAD_URL}/${_versioned_target_url_tag} + ${LY_INSTALLER_DOWNLOAD_URL} UPLOAD_DIRECTORY ${CMAKE_BINARY_DIR}/_CPack_Uploads # to match the _CPack_Packages directory ALL ) diff --git a/cmake/PackagingConfig.cmake b/cmake/PackagingConfig.cmake new file mode 100644 index 0000000000..b340219ff8 --- /dev/null +++ b/cmake/PackagingConfig.cmake @@ -0,0 +1,52 @@ +# +# All or portions of this file Copyright (c) Amazon.com, Inc. or its affiliates or +# its licensors. +# +# For complete copyright and license terms please see the LICENSE at the root of this +# distribution (the "License"). All use of this software is governed by the License, +# or, if provided, by the license below or the license accompanying this file. Do not +# remove or modify any license notices. This file is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# + +set(_target_name ${CMAKE_HOST_SYSTEM_NAME}) +if(${_target_name} STREQUAL Darwin) + set(_target_name Mac) +endif() + +if(CPACK_AUTO_GEN_TAG) + set(_python_script python.sh) + if(${_target_name} STREQUAL Windows) + set(_python_script python.cmd) + endif() + + file(REAL_PATH "${CPACK_SOURCE_DIR}/.." _root_path) + file(TO_NATIVE_PATH "${_root_path}/python/${_python_script}" _python_cmd) + file(TO_NATIVE_PATH "${_root_path}/scripts/build/tools/generate_build_tag.py" _gen_tag_script) + + execute_process( + COMMAND ${_python_cmd} -s -u ${_gen_tag_script} + RESULT_VARIABLE _gen_tag_result + OUTPUT_VARIABLE _gen_tag_output + ERROR_VARIABLE _gen_tag_errors + OUTPUT_STRIP_TRAILING_WHITESPACE + ERROR_STRIP_TRAILING_WHITESPACE + ) + + if (NOT ${_gen_tag_result} EQUAL 0) + message(FATAL_ERROR "Failed to generate build tag! Errors: ${_gen_tag_errors}") + endif() + + set(_url_tag ${_gen_tag_output}) +else() + set(_url_tag ${CPACK_PACKAGE_VERSION}) +endif() + +set(_full_tag ${_url_tag}/${_target_name}) + +if(CPACK_DOWNLOAD_SITE) + set(CPACK_DOWNLOAD_SITE ${CPACK_DOWNLOAD_SITE}/${_full_tag}) +endif() +if(CPACK_UPLOAD_URL) + set(CPACK_UPLOAD_URL ${CPACK_UPLOAD_URL}/${_full_tag}) +endif() diff --git a/cmake/Platform/Windows/Packaging/Bootstrapper.wxs b/cmake/Platform/Windows/Packaging/Bootstrapper.wxs index c7146b1b7f..cb9d86ede8 100644 --- a/cmake/Platform/Windows/Packaging/Bootstrapper.wxs +++ b/cmake/Platform/Windows/Packaging/Bootstrapper.wxs @@ -17,6 +17,10 @@ Value="[ProgramFiles64Folder]$(var.CPACK_PACKAGE_INSTALL_DIRECTORY)" bal:Overridable="yes"/> + + + + + + @@ -34,6 +40,8 @@ ThemeFile="$(var.CPACK_BOOTSTRAP_THEME_FILE).xml" LocalizationFile="$(var.CPACK_BOOTSTRAP_THEME_FILE).wxl" ShowVersion="yes" /> + + diff --git a/cmake/Platform/Windows/Packaging/BootstrapperTheme.xml.in b/cmake/Platform/Windows/Packaging/BootstrapperTheme.xml.in index f9a177b646..93984d06b8 100644 --- a/cmake/Platform/Windows/Packaging/BootstrapperTheme.xml.in +++ b/cmake/Platform/Windows/Packaging/BootstrapperTheme.xml.in @@ -76,7 +76,7 @@ - + #(loc.FailureHeader) #(loc.FailureInstallHeader) #(loc.FailureUninstallHeader) diff --git a/cmake/Platform/Windows/PackagingPostBuild.cmake b/cmake/Platform/Windows/PackagingPostBuild.cmake index 1dcbedcba7..83dff8354e 100644 --- a/cmake/Platform/Windows/PackagingPostBuild.cmake +++ b/cmake/Platform/Windows/PackagingPostBuild.cmake @@ -17,7 +17,7 @@ string(REPLACE "/" "\\" _fixed_package_install_dir ${CPACK_PACKAGE_INSTALL_DIREC set(_cpack_wix_out_dir ${CPACK_TOPLEVEL_DIRECTORY}) set(_bootstrap_out_dir "${CPACK_TOPLEVEL_DIRECTORY}/bootstrap") -set(_bootstrap_filename "${CPACK_PACKAGE_FILE_NAME}.exe") +set(_bootstrap_filename "${CPACK_PACKAGE_FILE_NAME}_installer.exe") set(_bootstrap_output_file ${_cpack_wix_out_dir}/${_bootstrap_filename}) set(_ext_flags @@ -32,6 +32,7 @@ set(_addtional_defines -dCPACK_PACKAGE_FILE_NAME=${CPACK_PACKAGE_FILE_NAME} -dCPACK_PACKAGE_INSTALL_DIRECTORY=${_fixed_package_install_dir} -dCPACK_WIX_PRODUCT_LOGO=${CPACK_WIX_PRODUCT_LOGO} + -dCPACK_RESOURCE_PATH=${CPACK_SOURCE_DIR}/Platform/Windows/Packaging ) if(CPACK_LICENSE_URL) @@ -124,15 +125,21 @@ set(_upload_command --file_regex="${_file_regex}" --bucket ${_bucket} --key_prefix ${_prefix} - --profile ${CPACK_AWS_PROFILE} ) +if(CPACK_AWS_PROFILE) + list(APPEND _upload_command --profile ${CPACK_AWS_PROFILE}) +endif() + +message(STATUS "Uploading artifacts to ${CPACK_UPLOAD_URL}") execute_process( COMMAND ${_upload_command} RESULT_VARIABLE _upload_result ERROR_VARIABLE _upload_errors ) -if (NOT ${_upload_result} EQUAL 0) +if (${_upload_result} EQUAL 0) + message(STATUS "Artifact uploading complete!") +else() message(FATAL_ERROR "An error occurred uploading artifacts. ${_upload_errors}") endif() diff --git a/cmake/Platform/Windows/Packaging_windows.cmake b/cmake/Platform/Windows/Packaging_windows.cmake index 6dc806ff56..95c00a3d0f 100644 --- a/cmake/Platform/Windows/Packaging_windows.cmake +++ b/cmake/Platform/Windows/Packaging_windows.cmake @@ -26,6 +26,12 @@ set(_cmake_package_name "cmake-${CPACK_DESIRED_CMAKE_VERSION}-windows-x86_64") set(CPACK_CMAKE_PACKAGE_FILE "${_cmake_package_name}.zip") set(CPACK_CMAKE_PACKAGE_HASH "15a49e2ab81c1822d75b1b1a92f7863f58e31f6d6aac1c4103eef2b071be3112") +# workaround for shortening the path cpack installs to by stripping the platform directory and forcing monolithic +# mode to strip out component folders. this unfortunately is the closest we can get to changing the install location +# as CPACK_PACKAGING_INSTALL_PREFIX/CPACK_SET_DESTDIR isn't supported for the WiX generator +set(CPACK_TOPLEVEL_TAG "") +set(CPACK_MONOLITHIC_INSTALL ON) + # CPack will generate the WiX product/upgrade GUIDs further down the chain if they weren't supplied # however, they are unique for each run. instead, let's do the auto generation here and add it to # the cache for run persistence and have the ability to detect if they are still being used. diff --git a/scripts/build/Jenkins/Jenkinsfile b/scripts/build/Jenkins/Jenkinsfile index 693cf31727..7bb26fdf8c 100644 --- a/scripts/build/Jenkins/Jenkinsfile +++ b/scripts/build/Jenkins/Jenkinsfile @@ -256,6 +256,15 @@ def CheckoutRepo(boolean disableSubmodules = false) { env.CHANGE_ID = readFile file: 'commitid' env.CHANGE_ID = env.CHANGE_ID.trim() palRm('commitid') + + // CHANGE_DATE is used by the installer to provide some ability to sort tagged builds in addition to BRANCH_NAME and CHANGE_ID + commitDateFmt = '%%cI' + if (env.IS_UNIX) commitDateFmt = '%cI' + + palSh("git show -s --format=${commitDateFmt} ${env.CHANGE_ID} > commitdate", 'Getting commit date') + env.CHANGE_DATE = readFile file: 'commitdate' + env.CHANGE_DATE = env.CHANGE_DATE.trim() + palRm('commitdate') } def PreBuildCommonSteps(Map pipelineConfig, String repositoryName, String projectName, String pipeline, String branchName, String platform, String buildType, String workspace, boolean mount = true, boolean disableSubmodules = false) { diff --git a/scripts/build/Platform/Windows/build_config.json b/scripts/build/Platform/Windows/build_config.json index 552ef2c6fd..45cea2b5d8 100644 --- a/scripts/build/Platform/Windows/build_config.json +++ b/scripts/build/Platform/Windows/build_config.json @@ -308,13 +308,15 @@ }, "windows_installer": { "TAGS": [ - "package" + "nightly-clean" ], "COMMAND": "build_installer_windows.cmd", "PARAMETERS": { "CONFIGURATION": "profile", "OUTPUT_DIRECTORY": "build\\windows_vs2019", - "CMAKE_OPTIONS": "-G \"Visual Studio 16 2019\" -DCMAKE_SYSTEM_VERSION=10.0 -DLY_UNITY_BUILD=TRUE -DLY_DISABLE_TEST_MODULES=TRUE -DCPACK_WIX_ROOT=\"!WIX!\"", + "CMAKE_OPTIONS": "-G \"Visual Studio 16 2019\" -DCMAKE_SYSTEM_VERSION=10.0 -DLY_UNITY_BUILD=TRUE -DLY_DISABLE_TEST_MODULES=TRUE -DCPACK_WIX_ROOT=\"!WIX! \"", + "EXTRA_CMAKE_OPTIONS": "-DLY_INSTALLER_AUTO_GEN_TAG=ON -DLY_INSTALLER_DOWNLOAD_URL=https://dkb1uj4hs9ikv.cloudfront.net -DLY_INSTALLER_LICENSE_URL=https://example.com", + "CPACK_BUCKET": "spectra-prism-staging-us-west-2", "CMAKE_LY_PROJECTS": "", "CMAKE_TARGET": "ALL_BUILD", "CMAKE_NATIVE_BUILD_ARGS": "/m /nologo" diff --git a/scripts/build/Platform/Windows/installer_windows.cmd b/scripts/build/Platform/Windows/installer_windows.cmd index a6ce15d59e..f7d4394dc8 100644 --- a/scripts/build/Platform/Windows/installer_windows.cmd +++ b/scripts/build/Platform/Windows/installer_windows.cmd @@ -21,7 +21,7 @@ IF NOT EXIST %OUTPUT_DIRECTORY% ( PUSHD %OUTPUT_DIRECTORY% REM Override the temporary directory used by wix to the workspace -SET "WIX_TEMP=!WORKSPACE!/temp/wix" +SET "WIX_TEMP=!WORKSPACE_TMP!/wix" IF NOT EXIST "%WIX_TEMP%" ( MKDIR "%WIX_TEMP%" ) @@ -50,9 +50,19 @@ IF ERRORLEVEL 1 ( GOTO :popd_error ) -ECHO [ci_build] "!CPACK_PATH!" -C %CONFIGURATION% -"!CPACK_PATH!" -C %CONFIGURATION% -IF NOT %ERRORLEVEL%==0 GOTO :popd_error +IF NOT "%CPACK_BUCKET%"=="" ( + SET "CPACK_OPTIONS=-D CPACK_UPLOAD_URL=s3://%CPACK_BUCKET% %CPACK_OPTIONS%" +) + +ECHO [ci_build] "!CPACK_PATH!" -C %CONFIGURATION% %CPACK_OPTIONS% +"!CPACK_PATH!" -C %CONFIGURATION% %CPACK_OPTIONS% +IF NOT %ERRORLEVEL%==0 ( + REM dump the log file generated by cpack specifically for WIX + ECHO **************************************************************** + TYPE "_CPack_Packages\\WIX\\wix.log" + ECHO **************************************************************** + GOTO :popd_error +) POPD EXIT /b 0 diff --git a/scripts/build/tools/generate_build_tag.py b/scripts/build/tools/generate_build_tag.py new file mode 100644 index 0000000000..b37e180ea6 --- /dev/null +++ b/scripts/build/tools/generate_build_tag.py @@ -0,0 +1,73 @@ +# +# All or portions of this file Copyright (c) Amazon.com, Inc. or its affiliates or +# its licensors. +# +# For complete copyright and license terms please see the LICENSE at the root of this +# distribution (the "License"). All use of this software is governed by the License, +# or, if provided, by the license below or the license accompanying this file. Do not +# remove or modify any license notices. This file is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# + +import argparse +import os +import pathlib +import shutil +import subprocess +import sys + + +def run_git_command(args, repo_root): + + process = subprocess.run(['git', *args], + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + cwd=repo_root, + env=os.environ.copy(), + universal_newlines=True, + ) + + if process.returncode != 0: + print( + f'An error occurred while running a command\n' + f'Command: git {subprocess.list2cmdline(args)}\n' + f'Return Code: {process.returncode}\n' + f'Error: {process.stderr}', + file=sys.stderr + ) + exit(1) + + output = process.stdout.splitlines() + if not output: + print(f'No output received for command: git {subprocess.list2cmdline(args)}.') + output + + return output[0].strip('"') + + +if __name__ == "__main__": + ''' + Generates a build ID based on the state of the git repository. Will first attempt to use + existing environment variable (e.g. BRANCH_NAME, CHANGE_ID, CHANGE_DATE) before falling + back to running git commands directly + ''' + repo_root = os.path.realpath(os.path.join(os.path.dirname(__file__), '..', '..')) + + branch = os.environ.get('BRANCH_NAME') + if not branch: + branch = run_git_command(['rev-parse', '--abbrev-ref', 'HEAD'], repo_root) + branch = branch.replace('/', '-') + + commit_hash = os.environ.get('CHANGE_ID') + if not commit_hash: + commit_hash = run_git_command(['rev-parse', 'HEAD'], repo_root) + commit_hash = commit_hash[0:9] + + # include the commit date to allow some sensible way of sorting + commit_date = os.environ.get('CHANGE_DATE') + if not commit_date: + commit_date = run_git_command(['show', '-s', '--format=%cI', commit_hash], repo_root) + commit_date = commit_date[0:10] + + print(f'{branch}/{commit_date}-{commit_hash}') + exit(0)