From 1a46f548a79ea101ff9408041dd5c96d41933f38 Mon Sep 17 00:00:00 2001 From: Esteban Papp <81431996+amznestebanpapp@users.noreply.github.com> Date: Wed, 1 Dec 2021 09:14:07 -0800 Subject: [PATCH] Remove unused scripts/build/package (#5885) Signed-off-by: Esteban Papp <81431996+amznestebanpapp@users.noreply.github.com> --- .../Jenkins/tools/jenkins_pipeline_metrics.py | 15 +- .../Windows/package_build_config.json | 13 -- scripts/build/package/PackageEnv.py | 191 ------------------ scripts/build/package/Params.py | 80 -------- .../Platform/3rdParty/package_env.json | 19 -- .../3rdParty/package_filelists/3rdParty.json | 18 -- .../package/Platform/Android/package_env.json | 25 --- .../package/Platform/Mac/package_env.json | 40 ---- .../Mac/package_filelists/3rdParty.json | 82 -------- .../package/Platform/Windows/package_env.json | 35 ---- .../Windows/package_filelists/3rdParty.json | 113 ----------- scripts/build/package/glob3.py | 163 --------------- scripts/build/package/package.py | 187 ----------------- scripts/build/package/package_env.json | 11 - .../build/package/package_filelists/all.json | 15 -- .../package/package_filelists/symbols.json | 5 - .../build/package/platform_exclusions.json | 12 -- scripts/{build/package => util}/util.py | 15 -- 18 files changed, 7 insertions(+), 1032 deletions(-) delete mode 100644 scripts/build/Platform/Windows/package_build_config.json delete mode 100755 scripts/build/package/PackageEnv.py delete mode 100755 scripts/build/package/Params.py delete mode 100644 scripts/build/package/Platform/3rdParty/package_env.json delete mode 100644 scripts/build/package/Platform/3rdParty/package_filelists/3rdParty.json delete mode 100644 scripts/build/package/Platform/Android/package_env.json delete mode 100644 scripts/build/package/Platform/Mac/package_env.json delete mode 100644 scripts/build/package/Platform/Mac/package_filelists/3rdParty.json delete mode 100644 scripts/build/package/Platform/Windows/package_env.json delete mode 100644 scripts/build/package/Platform/Windows/package_filelists/3rdParty.json delete mode 100755 scripts/build/package/glob3.py delete mode 100755 scripts/build/package/package.py delete mode 100644 scripts/build/package/package_env.json delete mode 100644 scripts/build/package/package_filelists/all.json delete mode 100644 scripts/build/package/package_filelists/symbols.json delete mode 100644 scripts/build/package/platform_exclusions.json rename scripts/{build/package => util}/util.py (78%) mode change 100755 => 100644 diff --git a/scripts/build/Jenkins/tools/jenkins_pipeline_metrics.py b/scripts/build/Jenkins/tools/jenkins_pipeline_metrics.py index 8b0164e3e8..6ea2e135a7 100644 --- a/scripts/build/Jenkins/tools/jenkins_pipeline_metrics.py +++ b/scripts/build/Jenkins/tools/jenkins_pipeline_metrics.py @@ -16,9 +16,8 @@ from datetime import datetime, timezone from requests.auth import HTTPBasicAuth cur_dir = os.path.dirname(os.path.abspath(__file__)) -sys.path.append(os.path.join(os.path.dirname(os.path.dirname(cur_dir)), 'package')) -from util import * - +sys.path.insert(0, os.path.abspath(f'{cur_dir}/../../../util')) +import util class JenkinsAPIClient: def __init__(self, jenkins_base_url, jenkins_username, jenkins_api_token): @@ -36,7 +35,7 @@ class JenkinsAPIClient: except Exception: traceback.print_exc() print(f'WARN: Get request {url} failed, retying....') - error(f'Get request {url} failed, see exception for more details.') + util.error(f'Get request {url} failed, see exception for more details.') def get_builds(self, pipeline_name, branch_name=''): url = self.jenkins_base_url + self.blueocean_api_path + f'/{pipeline_name}/{branch_name}/runs' @@ -123,9 +122,9 @@ def upload_files_to_s3(env, formatted_date): else: python = os.path.join(engine_root, 'python', 'python.sh') upload_csv_cmd = [python, upload_to_s3_script_path, '--base_dir', cur_dir, '--file_regex', env['CSV_REGEX'], '--bucket', env['BUCKET'], '--key_prefix', csv_s3_prefix] - execute_system_call(upload_csv_cmd) + util.execute_system_call(upload_csv_cmd) upload_manifest_cmd = [python, upload_to_s3_script_path, '--base_dir', cur_dir, '--file_regex', env['MANIFEST_REGEX'], '--bucket', env['BUCKET'], '--key_prefix', manifest_s3_prefix] - execute_system_call(upload_manifest_cmd) + util.execute_system_call(upload_manifest_cmd) def get_required_env(env, keys): @@ -134,7 +133,7 @@ def get_required_env(env, keys): try: env[key] = os.environ[key].strip() except KeyError: - error(f'{key} is not set in environment variable') + util.error(f'{key} is not set in environment variable') success = False return success @@ -143,7 +142,7 @@ def main(): env = {} required_env_list = ['JENKINS_URL', 'PIPELINE_NAME', 'BRANCH_NAME', 'JENKINS_USERNAME', 'JENKINS_API_TOKEN', 'BUCKET', 'CSV_REGEX', 'CSV_PREFIX', 'MANIFEST_REGEX', 'MANIFEST_PREFIX', 'DAYS_TO_COLLECT'] if not get_required_env(env, required_env_list): - error('Required environment variable is not set, see log for more details.') + util.error('Required environment variable is not set, see log for more details.') target_date = datetime.today().date() formatted_date = f'{target_date.year}/{target_date:%m}/{target_date:%d}' diff --git a/scripts/build/Platform/Windows/package_build_config.json b/scripts/build/Platform/Windows/package_build_config.json deleted file mode 100644 index 1a94fb802a..0000000000 --- a/scripts/build/Platform/Windows/package_build_config.json +++ /dev/null @@ -1,13 +0,0 @@ -{ - "profile_atom": { - "COMMAND":"build_windows.cmd", - "PARAMETERS": { - "CONFIGURATION":"profile", - "OUTPUT_DIRECTORY":"windows", - "CMAKE_OPTIONS":"-G \"Visual Studio 16 2019\" -DCMAKE_SYSTEM_VERSION=10.0", - "CMAKE_LY_PROJECTS":"AtomTest;AtomSampleViewer", - "CMAKE_TARGET":"ALL_BUILD", - "CMAKE_NATIVE_BUILD_ARGS": "/m /nologo" - } - } -} diff --git a/scripts/build/package/PackageEnv.py b/scripts/build/package/PackageEnv.py deleted file mode 100755 index 1f2d102214..0000000000 --- a/scripts/build/package/PackageEnv.py +++ /dev/null @@ -1,191 +0,0 @@ -# -# Copyright (c) Contributors to the Open 3D Engine Project. -# For complete copyright and license terms please see the LICENSE at the root of this distribution. -# -# SPDX-License-Identifier: Apache-2.0 OR MIT -# -# - -from Params import Params -from util import * - - -class PackageEnv(Params): - def __init__(self, platform, type, json_file): - super(PackageEnv, self).__init__() - self.__cur_dir = os.path.dirname(os.path.abspath(__file__)) - global_env_file = os.path.join(self.__cur_dir, json_file) - with open(global_env_file, 'r') as source: - data = json.load(source) - self.__global_env = data.get('global_env') - platform_env_file = os.path.join(self.__cur_dir, 'Platform', platform, json_file) - if not os.path.exists(platform_env_file): - print(f'{platform_env_file} is not found.') - # Search restricted platform folders - engine_root = self.get('ENGINE_ROOT') - # Use real path in case engine root is a symlink path - if os.name == 'posix' and os.path.islink(engine_root): - engine_root = os.readlink(engine_root) - rel_path = os.path.relpath(self.__cur_dir, engine_root) - platform_env_file = os.path.join(engine_root, 'restricted', platform, rel_path, json_file) - if not os.path.exists(platform_env_file): - ly_build_error(f'{platform_env_file} is not found.') - with open(platform_env_file, 'r') as source: - data = json.load(source) - types = data.get('types') - if type not in types: - ly_build_error(f'Package type {type} is not supported') - self.__platform = platform - self.__platform_env = data.get('local_env') - self.__platform_env.update(self.__global_env) - self.__type = type - self.__type_env = types.get(type) - - def get_platform(self): - return self.__platform - - def get_type(self): - return self.__type - - def get_platform_env(self): - return self.__platform_env - - def get_type_env(self): - return self.__type_env - - def __get_platform_value(self, key): - key = key.upper() - value = self.__platform_env.get(key) - if value is None: - ly_build_error(f'{key} is not defined in global env nor in local env') - return value - - def __get_type_value(self, key): - key = key.upper() - value = self.__type_env.get(key) - if value is None: - ly_build_error(f'{key} is not defined in package type {self.__type} for platform {self.__platform}') - return value - - def __evaluate_boolean(self, v): - return str(v).lower() in ['1', 'true'] - - def __get_engine_root(self): - def validate_engine_root(engine_root): - if not os.path.isdir(engine_root): - return False - return os.path.exists(os.path.join(engine_root, 'engine.json')) - - workspace = os.getenv('WORKSPACE') - if workspace is not None: - print(f'Environment variable WORKSPACE={workspace} detected') - if validate_engine_root(workspace): - print(f'Setting ENGINE_ROOT to {workspace}') - return workspace - print('Cannot locate ENGINE_ROOT with Environment variable WORKSPACE') - - engine_root = os.getenv('ENGINE_ROOT', '') - if validate_engine_root(engine_root): - return engine_root - - print('Environment variable ENGINE_ROOT is not set or invalid, checking ENGINE_ROOT in env json file') - engine_root = self.__global_env.get('ENGINE_ROOT') - if validate_engine_root(engine_root): - return engine_root - - # Set engine_root based on script location - engine_root = os.path.dirname(os.path.dirname(os.path.dirname(self.__cur_dir))) - print(f'ENGINE_ROOT from env json file is invalid, defaulting to {engine_root}') - if validate_engine_root(engine_root): - return engine_root - else: - error('Cannot Locate ENGINE_ROOT') - - def __get_thirdparty_home(self): - third_party_home = os.getenv('LY_3RDPARTY_PATH', '') - if os.path.exists(third_party_home): - print(f'LY_3RDPARTY_PATH found, using {third_party_home} as 3rdParty path.') - return third_party_home - third_party_home = self.__get_platform_value('THIRDPARTY_HOME') - if os.path.isdir(third_party_home): - return third_party_home - - # Set engine_root based on script location - print('THIRDPARTY_HOME is not valid, looking for THIRD_PARTY_HOME') - - # Finding THIRD_PARTY_HOME - cur_dir = self.__get_engine_root() - last_dir = None - while last_dir != cur_dir: - third_party_home = os.path.join(cur_dir, '3rdParty') - print(f'Cheking THIRDPARTY_HOME {third_party_home}') - if os.path.exists(os.path.join(third_party_home, '3rdParty.txt')): - print(f'Setting THIRDPARTY_HOME to {third_party_home}') - return third_party_home - last_dir = cur_dir - cur_dir = os.path.dirname(cur_dir) - error('Cannot locate THIRDPARTY_HOME') - - def __get_package_name_pattern(self): - package_name_pattern = self.__get_platform_value('PACKAGE_NAME_PATTERN') - if os.getenv('PACKAGE_NAME_PATTERN') is not None: - package_name_pattern = os.getenv('PACKAGE_NAME_PATTERN') - return package_name_pattern - - def __get_branch_name(self): - branch_name = self.__get_platform_value('BRANCH_NAME') - if os.getenv('BRANCH_NAME') is not None: - branch_name = os.getenv('BRANCH_NAME') - branch_name = branch_name.replace('/', '_').replace('\\', '_') - return branch_name - - def __get_build_number(self): - build_number = self.__get_platform_value('BUILD_NUMBER') - if os.getenv('BUILD_NUMBER') is not None: - build_number = os.getenv('BUILD_NUMBER') - return build_number - - def __get_scrub_params(self): - return self.__get_type_value('SCRUB_PARAMS') - - def __get_validator_platforms(self): - return self.__get_type_value('VALIDATOR_PLATFORMS') - - def __get_package_targets(self): - return self.__get_type_value('PACKAGE_TARGETS') - - def __get_build_targets(self): - return self.__get_type_value('BUILD_TARGETS') - - def __get_asset_processor_path(self): - return self.__get_type_value('ASSET_PROCESSOR_PATH') - - def __get_asset_game_folders(self): - return self.__get_type_value('ASSET_GAME_FOLDERS') - - def __get_asset_platform(self): - return self.__get_type_value('ASSET_PLATFORM') - - def __get_bootstrap_cfg_game_folder(self): - return self.__get_type_value('BOOTSTRAP_CFG_GAME_FOLDER') - - def __get_skip_build(self): - skip_build = os.getenv('SKIP_BUILD') - if skip_build is None: - skip_build = self.__get_type_value('SKIP_BUILD') - return self.__evaluate_boolean(skip_build) - - def __get_skip_scrubbing(self): - skip_scrubbing = os.getenv('SKIP_SCRUBBING') - if skip_scrubbing is None: - skip_scrubbing = self.__type_env.get('SKIP_SCRUBBING', 'False') - return self.__evaluate_boolean(skip_scrubbing) - - def __get_internal_s3_bucket(self): - return self.__get_platform_value('INTERNAL_S3_BUCKET') - - def __get_qa_s3_bucket(self): - return self.__get_platform_value('QA_S3_BUCKET') - - def __get_s3_prefix(self): - return self.__get_platform_value('S3_PREFIX') diff --git a/scripts/build/package/Params.py b/scripts/build/package/Params.py deleted file mode 100755 index 994d28486a..0000000000 --- a/scripts/build/package/Params.py +++ /dev/null @@ -1,80 +0,0 @@ -# -# Copyright (c) Contributors to the Open 3D Engine Project. -# For complete copyright and license terms please see the LICENSE at the root of this distribution. -# -# SPDX-License-Identifier: Apache-2.0 OR MIT -# -# -import os -import sys -import re -from util import ly_build_error - - -class Params(object): - def __init__(self): - # Cache params - self.__params = {} - - def get(self, param_name): - param_value = self.__params.get(param_name) - if param_value is not None: - return param_value - # Call __get_${param_name} function - func = getattr(self, '_{}__get_{}'.format(self.__class__.__name__, param_name.lower()), None) - if func is not None: - param_value = func() - # Replace all ${env} in value - if isinstance(param_value, str): - param_value = self.__process_string(param_name, param_value) - elif isinstance(param_value, list): - param_value = self.__process_list(param_name, param_value) - elif isinstance(param_value, dict): - param_value = self.__process_dict(param_name, param_value) - # Cache param - self.__params[param_name] = param_value - return param_value - ly_build_error('method __get_{} is not defined in class {}'.format(param_name.lower(), self.__class__.__name__)) - - def set(self, param_name, param_value): - self.__params[param_name] = param_value - - def exists(self, param_name): - try: - self.get(param_name) - except LyBuildError: - return False - return True - - def __process_string(self, param_name, param_value): - # Find all param with format ${param} - params = re.findall('\${(\w+)}', param_value) - # Avoid using the same param name in value, like 'WORKSPACE': '${WORKSPACE} some string' - if param_name in params: - ly_build_error('The use of same parameter name({}) in value is not allowed'.format(param_name)) - # Replace ${param} with actual value - for param in params: - param_value = param_value.replace('${' + param + '}', self.get(param)) - return param_value - - def __process_list(self, param_name, param_value): - processed_list = [] - for entry in param_value: - if isinstance(entry, str): - entry = self.__process_string(param_name, entry) - elif isinstance(entry, list): - entry = self.__process_list(param_name, entry) - elif isinstance(entry, dict): - entry = self.__process_dict(param_name, entry) - processed_list.append(entry) - return processed_list - - def __process_dict(self, param_name, param_value): - for key in param_value: - if isinstance(param_value[key], str): - param_value[key] = self.__process_string(param_name, param_value[key]) - elif isinstance(param_value[key], list): - param_value[key] = self.__process_list(param_name, param_value[key]) - elif isinstance(param_value[key], dict): - param_value[key] = self.__process_dict(param_name, param_value[key]) - return param_value \ No newline at end of file diff --git a/scripts/build/package/Platform/3rdParty/package_env.json b/scripts/build/package/Platform/3rdParty/package_env.json deleted file mode 100644 index 5059132664..0000000000 --- a/scripts/build/package/Platform/3rdParty/package_env.json +++ /dev/null @@ -1,19 +0,0 @@ -{ - "local_env": { - "S3_PREFIX": "${BRANCH_NAME}/3rdParty" - }, - "types": { - "3rdParty_all": { - "PACKAGE_TARGETS":[ - { - "FILE_LIST": "3rdParty.json", - "FILE_LIST_TYPE": "3rdParty", - "PACKAGE_NAME": "${PACKAGE_NAME_PATTERN}-3rdParty-all-${BUILD_NUMBER}.zip" - } - ], - "BOOTSTRAP_CFG_GAME_FOLDER":"CMakeTestbed", - "SKIP_BUILD": 1, - "SKIP_SCRUBBING": 1 - } - } -} diff --git a/scripts/build/package/Platform/3rdParty/package_filelists/3rdParty.json b/scripts/build/package/Platform/3rdParty/package_filelists/3rdParty.json deleted file mode 100644 index 915368dd0b..0000000000 --- a/scripts/build/package/Platform/3rdParty/package_filelists/3rdParty.json +++ /dev/null @@ -1,18 +0,0 @@ -{ - "@3rdParty": { - "3rdParty.txt": "#include", - "AWS/AWSNativeSDK/1.7.167-az.2/**": "#include", - "CMake/3.19.1/**": "#include", - "DirectXShaderCompiler/1.0.1-az.1/**": "#include", - "DirectXShaderCompiler/2020.08.07/**": "#include", - "DirectXShaderCompiler/5.0.0-az/**": "#include", - "dyad/0.2.0-17-amazon/**": "#include", - "etc2comp/2017_04_24-az.2/**": "#include", - "expat/2.1.0-pkg.3/**": "#include", - "FbxSdk/2016.1.2-az.1/**": "#include", - "OpenSSL/1.1.1b-noasm-az/**": "#include", - "Qt/5.15.1.2-az/**": "#include", - "tiff/3.9.5-az.3/**": "#include", - "Wwise/2019.2.8.7432/**": "#include" - } -} \ No newline at end of file diff --git a/scripts/build/package/Platform/Android/package_env.json b/scripts/build/package/Platform/Android/package_env.json deleted file mode 100644 index 017937413a..0000000000 --- a/scripts/build/package/Platform/Android/package_env.json +++ /dev/null @@ -1,25 +0,0 @@ -{ - "local_env": { - "S3_PREFIX": "${BRANCH_NAME}/Android" - }, - "types":{ - "all":{ - "PACKAGE_TARGETS":[ - { - "FILE_LIST": "all.json", - "FILE_LIST_TYPE": "All", - "PACKAGE_NAME": "${PACKAGE_NAME_PATTERN}-android-all-${BUILD_NUMBER}.zip" - } - ], - "BOOTSTRAP_CFG_GAME_FOLDER":"AutomatedTesting", - "SKIP_BUILD": 0, - "BUILD_TARGETS":[ - { - "BUILD_CONFIG_FILENAME": "build_config.json", - "PLATFORM": "Android", - "TYPE": "profile" - } - ] - } - } -} diff --git a/scripts/build/package/Platform/Mac/package_env.json b/scripts/build/package/Platform/Mac/package_env.json deleted file mode 100644 index f21c9fdaab..0000000000 --- a/scripts/build/package/Platform/Mac/package_env.json +++ /dev/null @@ -1,40 +0,0 @@ -{ - "local_env": { - "S3_PREFIX": "${BRANCH_NAME}/Mac" - }, - "types":{ - "all":{ - "PACKAGE_TARGETS":[ - { - "FILE_LIST": "all.json", - "FILE_LIST_TYPE": "All", - "PACKAGE_NAME": "${PACKAGE_NAME_PATTERN}-mac-all-${BUILD_NUMBER}.zip" - }, - { - "FILE_LIST": "3rdParty.json", - "FILE_LIST_TYPE": "Mac", - "PACKAGE_NAME": "${PACKAGE_NAME_PATTERN}-mac-3rdParty-${BUILD_NUMBER}.zip" - }, - { - "FILE_LIST": "3rdParty.json", - "FILE_LIST_TYPE": "3rdParty", - "PACKAGE_NAME": "${PACKAGE_NAME_PATTERN}-mac-3rdParty-Warsaw-${BUILD_NUMBER}.zip" - } - ], - "BOOTSTRAP_CFG_GAME_FOLDER":"CMakeTestbed", - "SKIP_BUILD": 0, - "BUILD_TARGETS":[ - { - "BUILD_CONFIG_FILENAME": "build_config.json", - "PLATFORM": "Mac", - "TYPE": "profile" - }, - { - "BUILD_CONFIG_FILENAME": "build_config.json", - "PLATFORM": "iOS", - "TYPE": "profile" - } - ] - } - } -} diff --git a/scripts/build/package/Platform/Mac/package_filelists/3rdParty.json b/scripts/build/package/Platform/Mac/package_filelists/3rdParty.json deleted file mode 100644 index ff019f701f..0000000000 --- a/scripts/build/package/Platform/Mac/package_filelists/3rdParty.json +++ /dev/null @@ -1,82 +0,0 @@ -{ - "@3rdParty":{ - "3rdParty.txt":"#include", - "AWS/AWSNativeSDK/1.7.167-az.2":{ - "*":"#include", - "include/**":"#include", - "LICENSE*":"#include", - "lib/mac/**":"#include", - "bin/mac/**":"#include", - "lib/ios/**":"#include", - "bin/ios/**":"#include" - }, - "DirectXShaderCompiler/1.0.1-az.1":{ - "*":"#include", - "src/**":"#include", - "bin/darwin_x64/**":"#include" - }, - "DirectXShaderCompiler/2020.08.07":{ - "*":"#include", - "bin/darwin_x64/**":"#include" - }, - "DirectXShaderCompiler/5.0.0-az":{ - "*":"#include", - "bin/darwin_x64/**":"#include" - }, - "etc2comp/2017_04_24-az.2":{ - "*":"#include", - "EtcLib/Etc/**":"#include", - "EtcLib/EtcCodec/**":"#include", - "EtcLib/*":"#include", - "EtcLib/OSX_x86/**":"#include" - }, - "expat/2.1.0-pkg.3":{ - "*":"#include", - "amiga/**":"#include", - "bcb5/**":"#include", - "conftools/**":"#include", - "doc/**":"#include", - "examples/**":"#include", - "lib/**":"#include", - "m4/**":"#include", - "tests/**":"#include", - "vms/**":"#include", - "win32/**":"#include", - "xmlwf/**":"#include", - "build/osx/**":"#include" - }, - "FreeType2/2.5.0.1-pkg.3":{ - "freetype-2.5.0.1/**":"#include", - "dist/**":"#include", - "mac/**":"#include", - "ios*/**":"#include", - "build/osx/**":"#include" - }, - "Redistributables/FbxSdk/2016.1.2":{ - "*mac*":"#include" - }, - "OpenSSL/1.1.1b-noasm-az":{ - "include/**":"#include", - "ssl/**":"#include", - "LICENSE":"#include", - "bin/**":"#include", - "lib/darwin*/**":"#include", - "lib/ios*/**":"#include" - }, - "Qt/5.15.1.2-az":{ - "LICENSE":"#include", - "LGPL_EXCEPTION.TXT":"#include", - "LICENSE.GPLV3":"#include", - "LICENSE.LGPLV3":"#include", - "QT-NOTICE.TXT":"#include", - "clang_64/**":"#include" - }, - "tiff/3.9.5-az.3":{ - "COPYRIGHT":"#include", - "README":"#include", - "RELEASE-DATE":"#include", - "VERSION":"#include", - "libtiff/macosx_clang/**":"#include" - } - } -} \ No newline at end of file diff --git a/scripts/build/package/Platform/Windows/package_env.json b/scripts/build/package/Platform/Windows/package_env.json deleted file mode 100644 index bfe526bc16..0000000000 --- a/scripts/build/package/Platform/Windows/package_env.json +++ /dev/null @@ -1,35 +0,0 @@ -{ - "local_env": { - "S3_PREFIX": "${BRANCH_NAME}/Windows" - }, - "types":{ - "all":{ - "PACKAGE_TARGETS":[ - { - "FILE_LIST": "all.json", - "FILE_LIST_TYPE": "All", - "PACKAGE_NAME": "${PACKAGE_NAME_PATTERN}-windows-all-${BUILD_NUMBER}.zip" - }, - { - "FILE_LIST": "symbols.json", - "FILE_LIST_TYPE": "All", - "PACKAGE_NAME": "${PACKAGE_NAME_PATTERN}-windows-all-symbols-${BUILD_NUMBER}.zip" - }, - { - "FILE_LIST": "3rdParty.json", - "FILE_LIST_TYPE": "Windows", - "PACKAGE_NAME": "${PACKAGE_NAME_PATTERN}-windows-3rdparty-${BUILD_NUMBER}.zip" - } - ], - "BOOTSTRAP_CFG_GAME_FOLDER":"CMakeTestbed", - "SKIP_BUILD": 0, - "BUILD_TARGETS":[ - { - "BUILD_CONFIG_FILENAME": "build_config.json", - "PLATFORM": "Windows", - "TYPE": "profile" - } - ] - } - } -} diff --git a/scripts/build/package/Platform/Windows/package_filelists/3rdParty.json b/scripts/build/package/Platform/Windows/package_filelists/3rdParty.json deleted file mode 100644 index d9e5d85090..0000000000 --- a/scripts/build/package/Platform/Windows/package_filelists/3rdParty.json +++ /dev/null @@ -1,113 +0,0 @@ -{ - "@3rdParty":{ - "3rdParty.txt":"#include", - "AMD/AGS Lib/2.2":{ - "*":"#include", - "inc/**":"#include", - "lib/x64/**":"#include" - }, - "AWS/AWSNativeSDK/1.7.167-az.2":{ - "*":"#include", - "include/**":"#include", - "LICENSE*":"#include", - "lib/windows/**":"#include", - "bin/windows/**":"#include", - "lib/android/arm64-v8a/**":"#include", - "bin/android/arm64-v8a/**":"#include", - "bin/linux/**":"#include", - "lib/linux/**":"#include" - }, - "DirectXShaderCompiler/1.0.1-az.1":{ - "*":"#include", - "src/**":"#include", - "bin/win_x64/**":"#include" - }, - "DirectXShaderCompiler/2020.08.07":{ - "*":"#include", - "bin/win_x64/**":"#include" - }, - "DirectXShaderCompiler/5.0.0-az":{ - "*":"#include", - "bin/win_x64/**":"#include" - }, - "dyad/0.2.0-17-amazon":{ - "*":"#include", - "doc/**":"#include", - "example/**":"#include", - "projects/**":"#include", - "src/**":"#include", - "lib/x64_v140_Debug/**":"#include", - "lib/x64_v140_Release/**":"#include", - "lib/linux_debug/**":"#include", - "lib/linux_release/**":"#include" - }, - "etc2comp/2017_04_24-az.2":{ - "EtcLib/Etc/**":"#include", - "EtcLib/EtcCodec/**":"#include", - "LICENSE":"#include", - "EtcLib/Windows_x86_64/**":"#include", - "EtcLib/Linux_x64_linux/**":"#include" - }, - "expat/2.1.0-pkg.3":{ - "*":"#include", - "amiga/**":"#include", - "bcb5/**":"#include", - "conftools/**":"#include", - "doc/**":"#include", - "examples/**":"#include", - "lib/**":"#include", - "m4/**":"#include", - "tests/**":"#include", - "vms/**":"#include", - "win32/**":"#include", - "xmlwf/**":"#include", - "build/win_x64/vc140/**":"#include", - "build/win_x64/android_ndk_r12/android-*/**":"#include", - "build/linux/**":"#include" - }, - "FreeType2/2.5.0.1-pkg.3":{ - "freetype-2.5.0.1/**":"#include", - "dist/**":"#include", - "vc140_x64/**":"#include", - "build/win_x64/vc140/**":"#include", - "android*/**":"#include", - "build/win_x64/android_ndk_r12/android-*/**":"#include", - "build/linux/clang-3.4/**":"#include" - }, - "Redistributables/FbxSdk/2016.1.2":{ - "*win*":"#include", - "*vs2013*":"#exclude" - }, - "OpenSSL/1.1.1b-noasm-az":{ - "include/**":"#include", - "ssl/**":"#include", - "LICENSE":"#include", - "bin/**":"#include", - "lib/vc140_x64_debug/**":"#include", - "lib/vc140_x64_release/**":"#include", - "lib/android_ndk_r15c/android-*/**":"#include", - "lib/linux-x86_64-clang-debug/**":"#include", - "lib/linux-x86_64-clang-release/**":"#include" - }, - "Qt/5.15.1.2-az":{ - "LICENSE":"#include", - "LGPL_EXCEPTION.TXT":"#include", - "LICENSE.GPLV3":"#include", - "LICENSE.LGPLV3":"#include", - "QT-NOTICE.TXT":"#include", - "msvc*/**":"#include", - "gcc_64/**":"#include" - }, - "tiff/3.9.5-az.3":{ - "COPYRIGHT":"#include", - "README":"#include", - "RELEASE-DATE":"#include", - "VERSION":"#include", - "libtiff/buildLib32Lib64.bat":"#include", - "libtiff/readme.txt":"#include", - "include/libtiff/*.h":"#include", - "libtiff/*vc140.lib":"#include", - "libtiff/linux_gcc/**":"#include" - } - } -} \ No newline at end of file diff --git a/scripts/build/package/glob3.py b/scripts/build/package/glob3.py deleted file mode 100755 index b46683a4be..0000000000 --- a/scripts/build/package/glob3.py +++ /dev/null @@ -1,163 +0,0 @@ -# -# Copyright (c) Contributors to the Open 3D Engine Project. -# For complete copyright and license terms please see the LICENSE at the root of this distribution. -# -# SPDX-License-Identifier: Apache-2.0 OR MIT -# -# - -import os -import re -import fnmatch - -__all__ = ["glob", "iglob", "escape"] - -def glob(pathname, recursive=False): - """Return a list of paths matching a pathname pattern. - The pattern may contain simple shell-style wildcards a la - fnmatch. However, unlike fnmatch, filenames starting with a - dot are special cases that are not matched by '*' and '?' - patterns. - If recursive is true, the pattern '**' will match any files and - zero or more directories and subdirectories. - """ - return list(iglob(pathname, recursive=recursive)) - -def iglob(pathname, recursive=False): - """Return an iterator which yields the paths matching a pathname pattern. - The pattern may contain simple shell-style wildcards a la - fnmatch. However, unlike fnmatch, filenames starting with a - dot are special cases that are not matched by '*' and '?' - patterns. - If recursive is true, the pattern '**' will match any files and - zero or more directories and subdirectories. - """ - it = _iglob(pathname, recursive, False) - if recursive and _isrecursive(pathname): - s = next(it) # skip empty string - assert not s - return it - -def _iglob(pathname, recursive, dironly): - dirname, basename = os.path.split(pathname) - if not has_magic(pathname): - assert not dironly - if basename: - if os.path.lexists(pathname): - yield pathname - else: - # Patterns ending with a slash should match only directories - if os.path.isdir(dirname): - yield pathname - return - if not dirname: - if recursive and _isrecursive(basename): - yield _glob2(dirname, basename, dironly) - else: - yield _glob1(dirname, basename, dironly) - return - # `os.path.split()` returns the argument itself as a dirname if it is a - # drive or UNC path. Prevent an infinite recursion if a drive or UNC path - # contains magic characters (i.e. r'\\?\C:'). - if dirname != pathname and has_magic(dirname): - dirs = _iglob(dirname, recursive, True) - else: - dirs = [dirname] - if has_magic(basename): - if recursive and _isrecursive(basename): - glob_in_dir = _glob2 - else: - glob_in_dir = _glob1 - else: - glob_in_dir = _glob0 - for dirname in dirs: - for name in glob_in_dir(dirname, basename, dironly): - yield os.path.join(dirname, name) - -# These 2 helper functions non-recursively glob inside a literal directory. -# They return a list of basenames. _glob1 accepts a pattern while _glob0 -# takes a literal basename (so it only has to check for its existence). - -def _glob1(dirname, pattern, dironly): - names = list(_iterdir(dirname, dironly)) - return fnmatch.filter(names, pattern) - -def _glob0(dirname, basename, dironly): - if not basename: - # `os.path.split()` returns an empty basename for paths ending with a - # directory separator. 'q*x/' should match only directories. - if os.path.isdir(dirname): - return [basename] - else: - if os.path.lexists(os.path.join(dirname, basename)): - return [basename] - return [] - -# Following functions are not public but can be used by third-party code. - -def glob0(dirname, pattern): - return _glob0(dirname, pattern, False) - -def glob1(dirname, pattern): - return _glob1(dirname, pattern, False) - -# This helper function recursively yields relative pathnames inside a literal -# directory. - -def _glob2(dirname, pattern, dironly): - assert _isrecursive(pattern) - return [pattern[:0]] + list(_rlistdir(dirname, dironly)) - -# If dironly is false, yields all file names inside a directory. -# If dironly is true, yields only directory names. -def _iterdir(dirname, dironly): - if not dirname: - if isinstance(dirname, bytes): - dirname = bytes(os.curdir, 'ASCII') - else: - dirname = os.curdir - try: - for entry in os.listdir(dirname): - yield entry - except OSError: - return - -# Recursively yields relative pathnames inside a literal directory. -def _rlistdir(dirname, dironly): - if not os.path.islink(dirname): - names = list(_iterdir(dirname, dironly)) - for x in names: - yield x - path = os.path.join(dirname, x) if dirname else x - for y in _rlistdir(path, dironly): - yield os.path.join(x, y) -magic_check = re.compile('([*?[])') -magic_check_bytes = re.compile(b'([*?[])') - -def has_magic(s): - if isinstance(s, bytes): - match = magic_check_bytes.search(s) - else: - match = magic_check.search(s) - return match is not None - -def _ishidden(path): - return path[0] in ('.', b'.'[0]) - -def _isrecursive(pattern): - if isinstance(pattern, bytes): - return pattern == b'**' - else: - return pattern == '**' - -def escape(pathname): - """Escape all special characters. - """ - # Escaping is done by wrapping any of "*?[" between square brackets. - # Metacharacters do not work in the drive part and shouldn't be escaped. - drive, pathname = os.path.splitdrive(pathname) - if isinstance(pathname, bytes): - pathname = magic_check_bytes.sub(br'[\1]', pathname) - else: - pathname = magic_check.sub(r'[\1]', pathname) - return drive + pathname \ No newline at end of file diff --git a/scripts/build/package/package.py b/scripts/build/package/package.py deleted file mode 100755 index d33601ba45..0000000000 --- a/scripts/build/package/package.py +++ /dev/null @@ -1,187 +0,0 @@ -# -# Copyright (c) Contributors to the Open 3D Engine Project. -# For complete copyright and license terms please see the LICENSE at the root of this distribution. -# -# SPDX-License-Identifier: Apache-2.0 OR MIT -# -# -import os -import sys -import zipfile -import timeit -import progressbar -from optparse import OptionParser -from PackageEnv import PackageEnv -cur_dir = cur_dir = os.path.dirname(os.path.abspath(__file__)) -sys.path.insert(0, f'{cur_dir}/..') -from ci_build import build -from util import * -from glob3 import glob - - -def package(options): - package_env = PackageEnv(options.platform, options.type, options.package_env) - - if not package_env.get('SKIP_BUILD'): - print(package_env.get('SKIP_BUILD')) - print('SKIP_BUILD is False, running CMake build...') - cmake_build(package_env) - - # TODO Compile Assets - #if package_env.exists('ASSET_PROCESSOR_PATH'): - # compile_assets(package_env) - - #create packages - package_targets = package_env.get('PACKAGE_TARGETS') - for package_target in package_targets: - create_package(package_env, package_target) - upload_package(package_env, package_target) - - -def get_python_path(package_env): - if sys.platform == 'win32': - return os.path.join(package_env.get('ENGINE_ROOT'), 'python', 'python.cmd') - else: - return os.path.join(package_env.get('ENGINE_ROOT'), 'python', 'python.sh') - - -def cmake_build(package_env): - build_targets = package_env.get('BUILD_TARGETS') - for build_target in build_targets: - build(build_target['BUILD_CONFIG_FILENAME'], build_target['PLATFORM'], build_target['TYPE']) - - -def create_package(package_env, package_target): - print('Creating zipfile for package target {}'.format(package_target)) - cur_dir = os.path.dirname(os.path.abspath(__file__)) - file_list_type = package_target['FILE_LIST_TYPE'] - if file_list_type == 'All': - filelist = os.path.join(cur_dir, 'package_filelists', package_target['FILE_LIST']) - else: - filelist = os.path.join(cur_dir, 'Platform', file_list_type, 'package_filelists', package_target['FILE_LIST']) - with open(filelist, 'r') as source: - data = json.load(source) - lyengine = package_env.get('ENGINE_ROOT') - print('Calculating filelists...') - files = {} - - if '@lyengine' in data: - files.update(filter_files(data['@lyengine'], lyengine)) - if '@3rdParty' in data: - files.update(filter_files(data['@3rdParty'], package_env.get('THIRDPARTY_HOME'))) - package_path = os.path.join(lyengine, package_target['PACKAGE_NAME']) - print('Creating zipfile at {}'.format(package_path)) - start = timeit.default_timer() - with progressbar.ProgressBar(max_value=len(files), redirect_stderr=True) as bar: - with zipfile.ZipFile(package_path, 'w', compression=zipfile.ZIP_DEFLATED, allowZip64=True) as myzip: - i = 0 - bar.update(i) - last_bar_update = timeit.default_timer() - for f in files: - if os.path.islink(f): - zipInfo = zipfile.ZipInfo(files[f]) - zipInfo.create_system = 3 - # long type of hex val of '0xA1ED0000L', - # say, symlink attr magic... - zipInfo.external_attr |= 0xA0000000 - myzip.writestr(zipInfo, os.readlink(f)) - else: - myzip.write(f, files[f]) - i += 1 - # Update progress bar every 2 minutes - if int(timeit.default_timer() - last_bar_update) > 120: - last_bar_update = timeit.default_timer() - bar.update(i) - bar.update(i) - - stop = timeit.default_timer() - total_time = int(stop - start) - print('{} is created. Total time: {} seconds.'.format(package_path, total_time)) - - def get_MD5(file_path): - from hashlib import md5 - chunk_size = 200 * 1024 - h = md5() - with open(file_path, 'rb') as f: - while True: - chunk = f.read(chunk_size) - if len(chunk): - h.update(chunk) - else: - break - return h.hexdigest() - - md5_file = '{}.MD5'.format(package_path) - print('Creating MD5 file at {}'.format(md5_file)) - start = timeit.default_timer() - with open(md5_file, 'w') as output: - output.write(get_MD5(package_path)) - stop = timeit.default_timer() - total_time = int(stop - start) - print('{} is created. Total time: {} seconds.'.format(md5_file, total_time)) - - -def upload_package(package_env, package_target): - package_name = package_target['PACKAGE_NAME'] - engine_root = package_env.get('ENGINE_ROOT') - internal_s3_bucket = package_env.get('INTERNAL_S3_BUCKET') - qa_s3_bucket = package_env.get('QA_S3_BUCKET') - s3_prefix = package_env.get('S3_PREFIX') - print(f'Uploading {package_name} to S3://{internal_s3_bucket}/{s3_prefix}/{package_name}') - cmd = ['aws', 's3', 'cp', os.path.join(engine_root, package_name), f's3://{internal_s3_bucket}/{s3_prefix}/{package_name}'] - execute_system_call(cmd, stdout=subprocess.DEVNULL) - print(f'Uploading {package_name} to S3://{qa_s3_bucket}/{s3_prefix}/{package_name}') - cmd = ['aws', 's3', 'cp', os.path.join(engine_root, package_name), f's3://{qa_s3_bucket}/{s3_prefix}/{package_name}', '--acl', 'bucket-owner-full-control'] - execute_system_call(cmd, stdout=subprocess.DEVNULL) - - -def filter_files(data, base, prefix='', support_symlinks=True): - includes = {} - excludes = set() - for key, value in data.items(): - pattern = os.path.join(base, prefix, key) - if not isinstance(value, dict): - pattern = os.path.normpath(pattern) - result = glob(pattern, recursive=True) - files = [x for x in result if os.path.isfile(x) or (support_symlinks and os.path.islink(x))] - if value == "#exclude": - excludes.update(files) - elif value == "#include": - for file in files: - includes[file] = os.path.relpath(file, base) - else: - if value.startswith('#move:'): - for file in files: - file_name = os.path.relpath(file, os.path.join(base, prefix)) - dst_dir = value.replace('#move:', '').strip(' ') - includes[file] = os.path.join(dst_dir, file_name) - elif value.startswith('#rename:'): - for file in files: - dst_file = value.replace('#rename:', '').strip(' ') - includes[file] = dst_file - else: - warn('Unknown directive {} for pattern {}'.format(value, pattern)) - else: - includes.update(filter_files(value, base, os.path.join(prefix, key), support_symlinks)) - - for exclude in excludes: - try: - includes.pop(exclude) - except KeyError: - pass - return includes - - -def parse_args(): - parser = OptionParser() - parser.add_option("--platform", dest="platform", default='consoles', help="Target platform to package") - parser.add_option("--type", dest="type", default='consoles', help="Package type") - parser.add_option("--package_env", dest="package_env", default="package_env.json", - help="JSON file that defines package environment variables") - (options, args) = parser.parse_args() - return options, args - - -if __name__ == "__main__": - (options, args) = parse_args() - package(options) diff --git a/scripts/build/package/package_env.json b/scripts/build/package/package_env.json deleted file mode 100644 index 732ff12286..0000000000 --- a/scripts/build/package/package_env.json +++ /dev/null @@ -1,11 +0,0 @@ -{ - "global_env":{ - "ENGINE_ROOT":"", - "THIRDPARTY_HOME":"", - "BRANCH_NAME":"", - "PACKAGE_NAME_PATTERN":"${BRANCH_NAME}-spectra", - "BUILD_NUMBER":"0", - "INTERNAL_S3_BUCKET": "ly-spectra-packages", - "QA_S3_BUCKET": "amazon.ly.lionbridgeshare/ly-spectra-packages" - } -} diff --git a/scripts/build/package/package_filelists/all.json b/scripts/build/package/package_filelists/all.json deleted file mode 100644 index b1f3f270ca..0000000000 --- a/scripts/build/package/package_filelists/all.json +++ /dev/null @@ -1,15 +0,0 @@ -{ - "@lyengine": { - "**": "#include", - ".git/**": "#exclude", - ".gitattributes": "#exclude", - ".gitignore": "#exclude", - ".gitmodules": "#exclude", - ".lfsconfig": "#exclude", - ".p4ignore": "#exclude", - ".submodules": "#exclude", - "**/*.pyc": "#exclude", - "**/*.pdb": "#exclude", - "build/*/packages/*/*.stamp": "#exclude" - } -} \ No newline at end of file diff --git a/scripts/build/package/package_filelists/symbols.json b/scripts/build/package/package_filelists/symbols.json deleted file mode 100644 index 3f30f6ec1f..0000000000 --- a/scripts/build/package/package_filelists/symbols.json +++ /dev/null @@ -1,5 +0,0 @@ -{ - "@lyengine": { - "**/*.pdb": "#include" - } -} \ No newline at end of file diff --git a/scripts/build/package/platform_exclusions.json b/scripts/build/package/platform_exclusions.json deleted file mode 100644 index f54aeb07c9..0000000000 --- a/scripts/build/package/platform_exclusions.json +++ /dev/null @@ -1,12 +0,0 @@ -{ - "all": { - "@lyengine": { - "**/Gems/Atom/RHI/DX12/External/pix/**": "#exclude", - "**/.idea/**": "#exclude", - "**/*.csproj*": "#exclude", - "**/.owner": "#exclude", - "**/WinPixEventRuntime.dll": "#exclude", - "**/XenonConsole.exe": "#exclude" - } - } -} diff --git a/scripts/build/package/util.py b/scripts/util/util.py old mode 100755 new mode 100644 similarity index 78% rename from scripts/build/package/util.py rename to scripts/util/util.py index bed2b79833..18970e55b9 --- a/scripts/build/package/util.py +++ b/scripts/util/util.py @@ -13,26 +13,11 @@ import sys import subprocess -class LyBuildError(Exception): - def __init__(self, message): - super(LyBuildError, self).__init__(message) - - -def ly_build_error(message): - raise LyBuildError(message) - - def error(message): print(('Error: {}'.format(message))) exit(1) -# Exit with status code 0 means it won't fail the whole build process -def safe_exit_with_error(message): - print(('Error: {}'.format(message))) - exit(0) - - def warn(message): print(('Warning: {}'.format(message)))