From 57faa2d37701966d2b64f7bd643e49ecdba26eb7 Mon Sep 17 00:00:00 2001 From: scottr Date: Mon, 7 Jun 2021 15:15:14 -0700 Subject: [PATCH] [cpack_installer] installer upload to s3 --- cmake/Packaging.cmake | 43 ++++++++++++++- .../Platform/Windows/PackagingPostBuild.cmake | 52 ++++++++++++++++++- scripts/build/tools/upload_to_s3.py | 5 ++ 3 files changed, 96 insertions(+), 4 deletions(-) diff --git a/cmake/Packaging.cmake b/cmake/Packaging.cmake index 3e23511fa1..d473ac93d7 100644 --- a/cmake/Packaging.cmake +++ b/cmake/Packaging.cmake @@ -16,6 +16,8 @@ endif() # public facing options will be used for conversion into cpack specific ones below. set(LY_INSTALLER_DOWNLOAD_URL "" CACHE STRING "URL embedded into the installer to download additional artifacts") set(LY_INSTALLER_LICENSE_URL "" CACHE STRING "Optionally embed a link to the license instead of raw text") +set(LY_INSTALLER_UPLOAD_URL "" CACHE STRING "URL used to automatically upload the artifacts. Currently only accepts S3 URLs e.g. s3:///") +set(LY_INSTALLER_AWS_PROFILE "" CACHE STRING "AWS CLI profile for uploading artifacts. You can also use LY_INSTALLER_AWS_PROFILE environment variable.") set(CPACK_DESIRED_CMAKE_VERSION 3.20.2) @@ -103,6 +105,41 @@ install(FILES ${_cmake_package_dest} DESTINATION ./Tools/Redistributables/CMake ) +# checks for and removes trailing slash +function(strip_trailing_slash in_url out_url) + string(LENGTH ${in_url} _url_length) + MATH(EXPR _url_length "${_url_length}-1") + + string(SUBSTRING ${in_url} 0 ${_url_length} _clean_url) + if("${in_url}" STREQUAL "${_clean_url}/") + set(${out_url} ${_clean_url} PARENT_SCOPE) + else() + set(${out_url} ${in_url} PARENT_SCOPE) + endif() +endfunction() + +set(_versioned_target_url_tag ${LY_VERSION_STRING}/${PAL_HOST_PLATFORM_NAME}) + +if(LY_INSTALLER_UPLOAD_URL) + ly_is_s3_url(${LY_INSTALLER_UPLOAD_URL} _is_s3_bucket) + if(NOT _is_s3_bucket) + message(FATAL_ERROR "Only S3 installer uploading is supported at this time") + endif() + + if (LY_INSTALLER_AWS_PROFILE) + set(CPACK_AWS_PROFILE ${LY_INSTALLER_AWS_PROFILE}) + elseif (DEFINED ENV{LY_INSTALLER_AWS_PROFILE}) + set(CPACK_AWS_PROFILE $ENV{LY_INSTALLER_AWS_PROFILE}) + else() + message(FATAL_ERROR + "An AWS profile is required for installer S3 uploading. Please provide " + "one via LY_INSTALLER_AWS_PROFILE CLI argument or environment variable") + endif() + + strip_trailing_slash(${LY_INSTALLER_UPLOAD_URL} LY_INSTALLER_UPLOAD_URL) + set(CPACK_UPLOAD_URL ${LY_INSTALLER_UPLOAD_URL}/${_versioned_target_url_tag}) +endif() + # IMPORTANT: required to be included AFTER setting all property overrides include(CPack REQUIRED) @@ -146,9 +183,11 @@ ly_configure_cpack_component( ) if(LY_INSTALLER_DOWNLOAD_URL) - # this will set the following variables: CPACK_DOWNLOAD_SITE, CPACK_DOWNLOAD_ALL, and CPACK_UPLOAD_DIRECTORY + strip_trailing_slash(${LY_INSTALLER_DOWNLOAD_URL} LY_INSTALLER_DOWNLOAD_URL) + + # this will set the following variables: CPACK_DOWNLOAD_SITE, CPACK_DOWNLOAD_ALL, and CPACK_UPLOAD_DIRECTORY (local) cpack_configure_downloads( - ${LY_INSTALLER_DOWNLOAD_URL} + ${LY_INSTALLER_DOWNLOAD_URL}/${_versioned_target_url_tag} UPLOAD_DIRECTORY ${CMAKE_BINARY_DIR}/_CPack_Uploads # to match the _CPack_Packages directory ALL ) diff --git a/cmake/Platform/Windows/PackagingPostBuild.cmake b/cmake/Platform/Windows/PackagingPostBuild.cmake index d379358bf4..89b3efb44b 100644 --- a/cmake/Platform/Windows/PackagingPostBuild.cmake +++ b/cmake/Platform/Windows/PackagingPostBuild.cmake @@ -59,12 +59,21 @@ set(_light_command message(STATUS "Creating Bootstrap Installer...") execute_process( COMMAND ${_candle_command} - COMMAND_ERROR_IS_FATAL ANY + RESULT_VARIABLE _candle_result + ERROR_VARIABLE _candle_errors ) +if(NOT ${_candle_result} EQUAL 0) + message(FATAL_ERROR "An error occurred invoking candle.exe. ${_candle_errors}") +endif() + execute_process( COMMAND ${_light_command} - COMMAND_ERROR_IS_FATAL ANY + RESULT_VARIABLE _light_result + ERROR_VARIABLE _light_errors ) +if(NOT ${_light_result} EQUAL 0) + message(FATAL_ERROR "An error occurred invoking light.exe. ${_light_errors}") +endif() file(COPY ${_bootstrap_output_file} DESTINATION ${CPACK_PACKAGE_DIRECTORY} @@ -87,3 +96,42 @@ file(COPY ${_artifacts} DESTINATION ${CPACK_UPLOAD_DIRECTORY} ) message(STATUS "Artifacts copied to ${CPACK_UPLOAD_DIRECTORY}") + +if(NOT CPACK_UPLOAD_URL) + return() +endif() + +file(REAL_PATH "${CPACK_SOURCE_DIR}/.." _root_path) + +file(TO_NATIVE_PATH "${_root_path}/python/python.cmd" _python_cmd) +file(TO_NATIVE_PATH "${_root_path}/scripts/build/tools/upload_to_s3.py" _upload_script) +file(TO_NATIVE_PATH "${_cpack_wix_out_dir}" _cpack_wix_out_dir) + +# strip the scheme and extract the bucket/key prefix from the URL +string(REPLACE "s3://" "" _stripped_url ${CPACK_UPLOAD_URL}) +string(REPLACE "/" ";" _tokens ${_stripped_url}) + +list(POP_FRONT _tokens _bucket) +string(JOIN "/" _prefix ${_tokens}) + +set(_file_regex ".*(cab|exe|msi)$") + +set(_upload_command + ${_python_cmd} -s + -u ${_upload_script} + --base_dir ${_cpack_wix_out_dir} + --file_regex="${_file_regex}" + --bucket ${_bucket} + --key_prefix ${_prefix} + --profile ${CPACK_AWS_PROFILE} +) + +execute_process( + COMMAND ${_upload_command} + RESULT_VARIABLE _upload_result + ERROR_VARIABLE _upload_errors +) + +if (NOT ${_upload_result} EQUAL 0) + message(FATAL_ERROR "An error occurred uploading artifacts. ${_upload_errors}") +endif() diff --git a/scripts/build/tools/upload_to_s3.py b/scripts/build/tools/upload_to_s3.py index d6d6d8ddb5..5dfe5eb66e 100755 --- a/scripts/build/tools/upload_to_s3.py +++ b/scripts/build/tools/upload_to_s3.py @@ -65,6 +65,11 @@ def get_client(service_name, profile_name): def get_files_to_upload(base_dir, regex): # Get all file names in base directory files = [x for x in os.listdir(base_dir) if os.path.isfile(os.path.join(base_dir, x))] + # strip the surround quotes, if they exist + try: + regex = json.loads(regex) + except: + pass # Get all file names matching the regular expression, those file will be uploaded to S3 files_to_upload = [x for x in files if re.match(regex, x)] return files_to_upload