birkeh/o3de
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Remove unused scripts/build/package (#5885)

Browse Source 
Signed-off-by: Esteban Papp <81431996+amznestebanpapp@users.noreply.github.com>
This commit is contained in:
Esteban Papp
2021-12-01 09:14:07 -08:00
committed by GitHub
parent e64df3c141
commit 1a46f548a7
18 changed files with 7 additions and 1032 deletions
Download Patch File Download Diff File Expand all files Collapse all files
scripts/build/Jenkins/tools/jenkins_pipeline_metrics.py
+7 -8
View File
@@ -16,9 +16,8 @@ from datetime import datetime, timezone
from requests.auth import HTTPBasicAuth
cur_dir = os.path.dirname(os.path.abspath(__file__))
sys.path.append(os.path.join(os.path.dirname(os.path.dirname(cur_dir)), 'package'))
from util import *
sys.path.insert(0, os.path.abspath(f'{cur_dir}/../../../util'))
import util
class JenkinsAPIClient:
def __init__(self, jenkins_base_url, jenkins_username, jenkins_api_token):
@@ -36,7 +35,7 @@ class JenkinsAPIClient:
except Exception:
traceback.print_exc()
print(f'WARN: Get request {url} failed, retying....')
error(f'Get request {url} failed, see exception for more details.')
util.error(f'Get request {url} failed, see exception for more details.')
def get_builds(self, pipeline_name, branch_name=''):
url = self.jenkins_base_url + self.blueocean_api_path + f'/{pipeline_name}/{branch_name}/runs'
@@ -123,9 +122,9 @@ def upload_files_to_s3(env, formatted_date):
else:
python = os.path.join(engine_root, 'python', 'python.sh')
upload_csv_cmd = [python, upload_to_s3_script_path, '--base_dir', cur_dir, '--file_regex', env['CSV_REGEX'], '--bucket', env['BUCKET'], '--key_prefix', csv_s3_prefix]
execute_system_call(upload_csv_cmd)
util.execute_system_call(upload_csv_cmd)
upload_manifest_cmd = [python, upload_to_s3_script_path, '--base_dir', cur_dir, '--file_regex', env['MANIFEST_REGEX'], '--bucket', env['BUCKET'], '--key_prefix', manifest_s3_prefix]
execute_system_call(upload_manifest_cmd)
util.execute_system_call(upload_manifest_cmd)
def get_required_env(env, keys):
@@ -134,7 +133,7 @@ def get_required_env(env, keys):
try:
env[key] = os.environ[key].strip()
except KeyError:
error(f'{key} is not set in environment variable')
util.error(f'{key} is not set in environment variable')
success = False
return success
@@ -143,7 +142,7 @@ def main():
env = {}
required_env_list = ['JENKINS_URL', 'PIPELINE_NAME', 'BRANCH_NAME', 'JENKINS_USERNAME', 'JENKINS_API_TOKEN', 'BUCKET', 'CSV_REGEX', 'CSV_PREFIX', 'MANIFEST_REGEX', 'MANIFEST_PREFIX', 'DAYS_TO_COLLECT']
if not get_required_env(env, required_env_list):
error('Required environment variable is not set, see log for more details.')
util.error('Required environment variable is not set, see log for more details.')
target_date = datetime.today().date()
formatted_date = f'{target_date.year}/{target_date:%m}/{target_date:%d}'
scripts/build/Platform/Windows/package_build_config.json
-13
View File
@@ -1,13 +0,0 @@
{
"profile_atom": {
"COMMAND":"build_windows.cmd",
"PARAMETERS": {
"CONFIGURATION":"profile",
"OUTPUT_DIRECTORY":"windows",
"CMAKE_OPTIONS":"-G \"Visual Studio 16 2019\" -DCMAKE_SYSTEM_VERSION=10.0",
"CMAKE_LY_PROJECTS":"AtomTest;AtomSampleViewer",
"CMAKE_TARGET":"ALL_BUILD",
"CMAKE_NATIVE_BUILD_ARGS": "/m /nologo"
}
}
}
scripts/build/package/PackageEnv.py
-191
View File
@@ -1,191 +0,0 @@
#
# Copyright (c) Contributors to the Open 3D Engine Project.
# For complete copyright and license terms please see the LICENSE at the root of this distribution.
#
# SPDX-License-Identifier: Apache-2.0 OR MIT
#
#
from Params import Params
from util import *
class PackageEnv(Params):
def __init__(self, platform, type, json_file):
super(PackageEnv, self).__init__()
self.__cur_dir = os.path.dirname(os.path.abspath(__file__))
global_env_file = os.path.join(self.__cur_dir, json_file)
with open(global_env_file, 'r') as source:
data = json.load(source)
self.__global_env = data.get('global_env')
platform_env_file = os.path.join(self.__cur_dir, 'Platform', platform, json_file)
if not os.path.exists(platform_env_file):
print(f'{platform_env_file} is not found.')
# Search restricted platform folders
engine_root = self.get('ENGINE_ROOT')
# Use real path in case engine root is a symlink path
if os.name == 'posix' and os.path.islink(engine_root):
engine_root = os.readlink(engine_root)
rel_path = os.path.relpath(self.__cur_dir, engine_root)
platform_env_file = os.path.join(engine_root, 'restricted', platform, rel_path, json_file)
if not os.path.exists(platform_env_file):
ly_build_error(f'{platform_env_file} is not found.')
with open(platform_env_file, 'r') as source:
data = json.load(source)
types = data.get('types')
if type not in types:
ly_build_error(f'Package type {type} is not supported')
self.__platform = platform
self.__platform_env = data.get('local_env')
self.__platform_env.update(self.__global_env)
self.__type = type
self.__type_env = types.get(type)
def get_platform(self):
return self.__platform
def get_type(self):
return self.__type
def get_platform_env(self):
return self.__platform_env
def get_type_env(self):
return self.__type_env
def __get_platform_value(self, key):
key = key.upper()
value = self.__platform_env.get(key)
if value is None:
ly_build_error(f'{key} is not defined in global env nor in local env')
return value
def __get_type_value(self, key):
key = key.upper()
value = self.__type_env.get(key)
if value is None:
ly_build_error(f'{key} is not defined in package type {self.__type} for platform {self.__platform}')
return value
def __evaluate_boolean(self, v):
return str(v).lower() in ['1', 'true']
def __get_engine_root(self):
def validate_engine_root(engine_root):
if not os.path.isdir(engine_root):
return False
return os.path.exists(os.path.join(engine_root, 'engine.json'))
workspace = os.getenv('WORKSPACE')
if workspace is not None:
print(f'Environment variable WORKSPACE={workspace} detected')
if validate_engine_root(workspace):
print(f'Setting ENGINE_ROOT to {workspace}')
return workspace
print('Cannot locate ENGINE_ROOT with Environment variable WORKSPACE')
engine_root = os.getenv('ENGINE_ROOT', '')
if validate_engine_root(engine_root):
return engine_root
print('Environment variable ENGINE_ROOT is not set or invalid, checking ENGINE_ROOT in env json file')
engine_root = self.__global_env.get('ENGINE_ROOT')
if validate_engine_root(engine_root):
return engine_root
# Set engine_root based on script location
engine_root = os.path.dirname(os.path.dirname(os.path.dirname(self.__cur_dir)))
print(f'ENGINE_ROOT from env json file is invalid, defaulting to {engine_root}')
if validate_engine_root(engine_root):
return engine_root
else:
error('Cannot Locate ENGINE_ROOT')
def __get_thirdparty_home(self):
third_party_home = os.getenv('LY_3RDPARTY_PATH', '')
if os.path.exists(third_party_home):
print(f'LY_3RDPARTY_PATH found, using {third_party_home} as 3rdParty path.')
return third_party_home
third_party_home = self.__get_platform_value('THIRDPARTY_HOME')
if os.path.isdir(third_party_home):
return third_party_home
# Set engine_root based on script location
print('THIRDPARTY_HOME is not valid, looking for THIRD_PARTY_HOME')
# Finding THIRD_PARTY_HOME
cur_dir = self.__get_engine_root()
last_dir = None
while last_dir != cur_dir:
third_party_home = os.path.join(cur_dir, '3rdParty')
print(f'Cheking THIRDPARTY_HOME {third_party_home}')
if os.path.exists(os.path.join(third_party_home, '3rdParty.txt')):
print(f'Setting THIRDPARTY_HOME to {third_party_home}')
return third_party_home
last_dir = cur_dir
cur_dir = os.path.dirname(cur_dir)
error('Cannot locate THIRDPARTY_HOME')
def __get_package_name_pattern(self):
package_name_pattern = self.__get_platform_value('PACKAGE_NAME_PATTERN')
if os.getenv('PACKAGE_NAME_PATTERN') is not None:
package_name_pattern = os.getenv('PACKAGE_NAME_PATTERN')
return package_name_pattern
def __get_branch_name(self):
branch_name = self.__get_platform_value('BRANCH_NAME')
if os.getenv('BRANCH_NAME') is not None:
branch_name = os.getenv('BRANCH_NAME')
branch_name = branch_name.replace('/', '_').replace('\\', '_')
return branch_name
def __get_build_number(self):
build_number = self.__get_platform_value('BUILD_NUMBER')
if os.getenv('BUILD_NUMBER') is not None:
build_number = os.getenv('BUILD_NUMBER')
return build_number
def __get_scrub_params(self):
return self.__get_type_value('SCRUB_PARAMS')
def __get_validator_platforms(self):
return self.__get_type_value('VALIDATOR_PLATFORMS')
def __get_package_targets(self):
return self.__get_type_value('PACKAGE_TARGETS')
def __get_build_targets(self):
return self.__get_type_value('BUILD_TARGETS')
def __get_asset_processor_path(self):
return self.__get_type_value('ASSET_PROCESSOR_PATH')
def __get_asset_game_folders(self):
return self.__get_type_value('ASSET_GAME_FOLDERS')
def __get_asset_platform(self):
return self.__get_type_value('ASSET_PLATFORM')
def __get_bootstrap_cfg_game_folder(self):
return self.__get_type_value('BOOTSTRAP_CFG_GAME_FOLDER')
def __get_skip_build(self):
skip_build = os.getenv('SKIP_BUILD')
if skip_build is None:
skip_build = self.__get_type_value('SKIP_BUILD')
return self.__evaluate_boolean(skip_build)
def __get_skip_scrubbing(self):
skip_scrubbing = os.getenv('SKIP_SCRUBBING')
if skip_scrubbing is None:
skip_scrubbing = self.__type_env.get('SKIP_SCRUBBING', 'False')
return self.__evaluate_boolean(skip_scrubbing)
def __get_internal_s3_bucket(self):
return self.__get_platform_value('INTERNAL_S3_BUCKET')
def __get_qa_s3_bucket(self):
return self.__get_platform_value('QA_S3_BUCKET')
def __get_s3_prefix(self):
return self.__get_platform_value('S3_PREFIX')
scripts/build/package/Params.py
-80
View File
@@ -1,80 +0,0 @@
#
# Copyright (c) Contributors to the Open 3D Engine Project.
# For complete copyright and license terms please see the LICENSE at the root of this distribution.
#
# SPDX-License-Identifier: Apache-2.0 OR MIT
#
#
import os
import sys
import re
from util import ly_build_error
class Params(object):
def __init__(self):
# Cache params
self.__params = {}
def get(self, param_name):
param_value = self.__params.get(param_name)
if param_value is not None:
return param_value
# Call __get_${param_name} function
func = getattr(self, '_{}__get_{}'.format(self.__class__.__name__, param_name.lower()), None)
if func is not None:
param_value = func()
# Replace all ${env} in value
if isinstance(param_value, str):
param_value = self.__process_string(param_name, param_value)
elif isinstance(param_value, list):
param_value = self.__process_list(param_name, param_value)
elif isinstance(param_value, dict):
param_value = self.__process_dict(param_name, param_value)
# Cache param
self.__params[param_name] = param_value
return param_value
ly_build_error('method __get_{} is not defined in class {}'.format(param_name.lower(), self.__class__.__name__))
def set(self, param_name, param_value):
self.__params[param_name] = param_value
def exists(self, param_name):
try:
self.get(param_name)
except LyBuildError:
return False
return True
def __process_string(self, param_name, param_value):
# Find all param with format ${param}
params = re.findall('\${(\w+)}', param_value)
# Avoid using the same param name in value, like 'WORKSPACE': '${WORKSPACE} some string'
if param_name in params:
ly_build_error('The use of same parameter name({}) in value is not allowed'.format(param_name))
# Replace ${param} with actual value
for param in params:
param_value = param_value.replace('${' + param + '}', self.get(param))
return param_value
def __process_list(self, param_name, param_value):
processed_list = []
for entry in param_value:
if isinstance(entry, str):
entry = self.__process_string(param_name, entry)
elif isinstance(entry, list):
entry = self.__process_list(param_name, entry)
elif isinstance(entry, dict):
entry = self.__process_dict(param_name, entry)
processed_list.append(entry)
return processed_list
def __process_dict(self, param_name, param_value):
for key in param_value:
if isinstance(param_value[key], str):
param_value[key] = self.__process_string(param_name, param_value[key])
elif isinstance(param_value[key], list):
param_value[key] = self.__process_list(param_name, param_value[key])
elif isinstance(param_value[key], dict):
param_value[key] = self.__process_dict(param_name, param_value[key])
return param_value
scripts/build/package/Platform/3rdParty/package_env.json
Vendored
-19
View File
@@ -1,19 +0,0 @@
{
"local_env": {
"S3_PREFIX": "${BRANCH_NAME}/3rdParty"
},
"types": {
"3rdParty_all": {
"PACKAGE_TARGETS":[
{
"FILE_LIST": "3rdParty.json",
"FILE_LIST_TYPE": "3rdParty",
"PACKAGE_NAME": "${PACKAGE_NAME_PATTERN}-3rdParty-all-${BUILD_NUMBER}.zip"
}
],
"BOOTSTRAP_CFG_GAME_FOLDER":"CMakeTestbed",
"SKIP_BUILD": 1,
"SKIP_SCRUBBING": 1
}
}
}
scripts/build/package/Platform/3rdParty/package_filelists/3rdParty.json
Vendored
-18
View File
@@ -1,18 +0,0 @@
{
"@3rdParty": {
"3rdParty.txt": "#include",
"AWS/AWSNativeSDK/1.7.167-az.2/**": "#include",
"CMake/3.19.1/**": "#include",
"DirectXShaderCompiler/1.0.1-az.1/**": "#include",
"DirectXShaderCompiler/2020.08.07/**": "#include",
"DirectXShaderCompiler/5.0.0-az/**": "#include",
"dyad/0.2.0-17-amazon/**": "#include",
"etc2comp/2017_04_24-az.2/**": "#include",
"expat/2.1.0-pkg.3/**": "#include",
"FbxSdk/2016.1.2-az.1/**": "#include",
"OpenSSL/1.1.1b-noasm-az/**": "#include",
"Qt/5.15.1.2-az/**": "#include",
"tiff/3.9.5-az.3/**": "#include",
"Wwise/2019.2.8.7432/**": "#include"
}
}
scripts/build/package/Platform/Android/package_env.json
-25
View File
@@ -1,25 +0,0 @@
{
"local_env": {
"S3_PREFIX": "${BRANCH_NAME}/Android"
},
"types":{
"all":{
"PACKAGE_TARGETS":[
{
"FILE_LIST": "all.json",
"FILE_LIST_TYPE": "All",
"PACKAGE_NAME": "${PACKAGE_NAME_PATTERN}-android-all-${BUILD_NUMBER}.zip"
}
],
"BOOTSTRAP_CFG_GAME_FOLDER":"AutomatedTesting",
"SKIP_BUILD": 0,
"BUILD_TARGETS":[
{
"BUILD_CONFIG_FILENAME": "build_config.json",
"PLATFORM": "Android",
"TYPE": "profile"
}
]
}
}
}
scripts/build/package/Platform/Mac/package_env.json
-40
View File
@@ -1,40 +0,0 @@
{
"local_env": {
"S3_PREFIX": "${BRANCH_NAME}/Mac"
},
"types":{
"all":{
"PACKAGE_TARGETS":[
{
"FILE_LIST": "all.json",
"FILE_LIST_TYPE": "All",
"PACKAGE_NAME": "${PACKAGE_NAME_PATTERN}-mac-all-${BUILD_NUMBER}.zip"
},
{
"FILE_LIST": "3rdParty.json",
"FILE_LIST_TYPE": "Mac",
"PACKAGE_NAME": "${PACKAGE_NAME_PATTERN}-mac-3rdParty-${BUILD_NUMBER}.zip"
},
{
"FILE_LIST": "3rdParty.json",
"FILE_LIST_TYPE": "3rdParty",
"PACKAGE_NAME": "${PACKAGE_NAME_PATTERN}-mac-3rdParty-Warsaw-${BUILD_NUMBER}.zip"
}
],
"BOOTSTRAP_CFG_GAME_FOLDER":"CMakeTestbed",
"SKIP_BUILD": 0,
"BUILD_TARGETS":[
{
"BUILD_CONFIG_FILENAME": "build_config.json",
"PLATFORM": "Mac",
"TYPE": "profile"
},
{
"BUILD_CONFIG_FILENAME": "build_config.json",
"PLATFORM": "iOS",
"TYPE": "profile"
}
]
}
}
}
scripts/build/package/Platform/Mac/package_filelists/3rdParty.json
-82
View File
@@ -1,82 +0,0 @@
{
"@3rdParty":{
"3rdParty.txt":"#include",
"AWS/AWSNativeSDK/1.7.167-az.2":{
"*":"#include",
"include/**":"#include",
"LICENSE*":"#include",
"lib/mac/**":"#include",
"bin/mac/**":"#include",
"lib/ios/**":"#include",
"bin/ios/**":"#include"
},
"DirectXShaderCompiler/1.0.1-az.1":{
"*":"#include",
"src/**":"#include",
"bin/darwin_x64/**":"#include"
},
"DirectXShaderCompiler/2020.08.07":{
"*":"#include",
"bin/darwin_x64/**":"#include"
},
"DirectXShaderCompiler/5.0.0-az":{
"*":"#include",
"bin/darwin_x64/**":"#include"
},
"etc2comp/2017_04_24-az.2":{
"*":"#include",
"EtcLib/Etc/**":"#include",
"EtcLib/EtcCodec/**":"#include",
"EtcLib/*":"#include",
"EtcLib/OSX_x86/**":"#include"
},
"expat/2.1.0-pkg.3":{
"*":"#include",
"amiga/**":"#include",
"bcb5/**":"#include",
"conftools/**":"#include",
"doc/**":"#include",
"examples/**":"#include",
"lib/**":"#include",
"m4/**":"#include",
"tests/**":"#include",
"vms/**":"#include",
"win32/**":"#include",
"xmlwf/**":"#include",
"build/osx/**":"#include"
},
"FreeType2/2.5.0.1-pkg.3":{
"freetype-2.5.0.1/**":"#include",
"dist/**":"#include",
"mac/**":"#include",
"ios*/**":"#include",
"build/osx/**":"#include"
},
"Redistributables/FbxSdk/2016.1.2":{
"*mac*":"#include"
},
"OpenSSL/1.1.1b-noasm-az":{
"include/**":"#include",
"ssl/**":"#include",
"LICENSE":"#include",
"bin/**":"#include",
"lib/darwin*/**":"#include",
"lib/ios*/**":"#include"
},
"Qt/5.15.1.2-az":{
"LICENSE":"#include",
"LGPL_EXCEPTION.TXT":"#include",
"LICENSE.GPLV3":"#include",
"LICENSE.LGPLV3":"#include",
"QT-NOTICE.TXT":"#include",
"clang_64/**":"#include"
},
"tiff/3.9.5-az.3":{
"COPYRIGHT":"#include",
"README":"#include",
"RELEASE-DATE":"#include",
"VERSION":"#include",
"libtiff/macosx_clang/**":"#include"
}
}
}
scripts/build/package/Platform/Windows/package_env.json
-35
View File
@@ -1,35 +0,0 @@
{
"local_env": {
"S3_PREFIX": "${BRANCH_NAME}/Windows"
},
"types":{
"all":{
"PACKAGE_TARGETS":[
{
"FILE_LIST": "all.json",
"FILE_LIST_TYPE": "All",
"PACKAGE_NAME": "${PACKAGE_NAME_PATTERN}-windows-all-${BUILD_NUMBER}.zip"
},
{
"FILE_LIST": "symbols.json",
"FILE_LIST_TYPE": "All",
"PACKAGE_NAME": "${PACKAGE_NAME_PATTERN}-windows-all-symbols-${BUILD_NUMBER}.zip"
},
{
"FILE_LIST": "3rdParty.json",
"FILE_LIST_TYPE": "Windows",
"PACKAGE_NAME": "${PACKAGE_NAME_PATTERN}-windows-3rdparty-${BUILD_NUMBER}.zip"
}
],
"BOOTSTRAP_CFG_GAME_FOLDER":"CMakeTestbed",
"SKIP_BUILD": 0,
"BUILD_TARGETS":[
{
"BUILD_CONFIG_FILENAME": "build_config.json",
"PLATFORM": "Windows",
"TYPE": "profile"
}
]
}
}
}
scripts/build/package/Platform/Windows/package_filelists/3rdParty.json
-113
View File
@@ -1,113 +0,0 @@
{
"@3rdParty":{
"3rdParty.txt":"#include",
"AMD/AGS Lib/2.2":{
"*":"#include",
"inc/**":"#include",
"lib/x64/**":"#include"
},
"AWS/AWSNativeSDK/1.7.167-az.2":{
"*":"#include",
"include/**":"#include",
"LICENSE*":"#include",
"lib/windows/**":"#include",
"bin/windows/**":"#include",
"lib/android/arm64-v8a/**":"#include",
"bin/android/arm64-v8a/**":"#include",
"bin/linux/**":"#include",
"lib/linux/**":"#include"
},
"DirectXShaderCompiler/1.0.1-az.1":{
"*":"#include",
"src/**":"#include",
"bin/win_x64/**":"#include"
},
"DirectXShaderCompiler/2020.08.07":{
"*":"#include",
"bin/win_x64/**":"#include"
},
"DirectXShaderCompiler/5.0.0-az":{
"*":"#include",
"bin/win_x64/**":"#include"
},
"dyad/0.2.0-17-amazon":{
"*":"#include",
"doc/**":"#include",
"example/**":"#include",
"projects/**":"#include",
"src/**":"#include",
"lib/x64_v140_Debug/**":"#include",
"lib/x64_v140_Release/**":"#include",
"lib/linux_debug/**":"#include",
"lib/linux_release/**":"#include"
},
"etc2comp/2017_04_24-az.2":{
"EtcLib/Etc/**":"#include",
"EtcLib/EtcCodec/**":"#include",
"LICENSE":"#include",
"EtcLib/Windows_x86_64/**":"#include",
"EtcLib/Linux_x64_linux/**":"#include"
},
"expat/2.1.0-pkg.3":{
"*":"#include",
"amiga/**":"#include",
"bcb5/**":"#include",
"conftools/**":"#include",
"doc/**":"#include",
"examples/**":"#include",
"lib/**":"#include",
"m4/**":"#include",
"tests/**":"#include",
"vms/**":"#include",
"win32/**":"#include",
"xmlwf/**":"#include",
"build/win_x64/vc140/**":"#include",
"build/win_x64/android_ndk_r12/android-*/**":"#include",
"build/linux/**":"#include"
},
"FreeType2/2.5.0.1-pkg.3":{
"freetype-2.5.0.1/**":"#include",
"dist/**":"#include",
"vc140_x64/**":"#include",
"build/win_x64/vc140/**":"#include",
"android*/**":"#include",
"build/win_x64/android_ndk_r12/android-*/**":"#include",
"build/linux/clang-3.4/**":"#include"
},
"Redistributables/FbxSdk/2016.1.2":{
"*win*":"#include",
"*vs2013*":"#exclude"
},
"OpenSSL/1.1.1b-noasm-az":{
"include/**":"#include",
"ssl/**":"#include",
"LICENSE":"#include",
"bin/**":"#include",
"lib/vc140_x64_debug/**":"#include",
"lib/vc140_x64_release/**":"#include",
"lib/android_ndk_r15c/android-*/**":"#include",
"lib/linux-x86_64-clang-debug/**":"#include",
"lib/linux-x86_64-clang-release/**":"#include"
},
"Qt/5.15.1.2-az":{
"LICENSE":"#include",
"LGPL_EXCEPTION.TXT":"#include",
"LICENSE.GPLV3":"#include",
"LICENSE.LGPLV3":"#include",
"QT-NOTICE.TXT":"#include",
"msvc*/**":"#include",
"gcc_64/**":"#include"
},
"tiff/3.9.5-az.3":{
"COPYRIGHT":"#include",
"README":"#include",
"RELEASE-DATE":"#include",
"VERSION":"#include",
"libtiff/buildLib32Lib64.bat":"#include",
"libtiff/readme.txt":"#include",
"include/libtiff/*.h":"#include",
"libtiff/*vc140.lib":"#include",
"libtiff/linux_gcc/**":"#include"
}
}
}
scripts/build/package/glob3.py
-163
View File
@@ -1,163 +0,0 @@
#
# Copyright (c) Contributors to the Open 3D Engine Project.
# For complete copyright and license terms please see the LICENSE at the root of this distribution.
#
# SPDX-License-Identifier: Apache-2.0 OR MIT
#
#
import os
import re
import fnmatch
__all__ = ["glob", "iglob", "escape"]
def glob(pathname, recursive=False):
"""Return a list of paths matching a pathname pattern.
The pattern may contain simple shell-style wildcards a la
fnmatch. However, unlike fnmatch, filenames starting with a
dot are special cases that are not matched by '*' and '?'
patterns.
If recursive is true, the pattern '**' will match any files and
zero or more directories and subdirectories.
"""
return list(iglob(pathname, recursive=recursive))
def iglob(pathname, recursive=False):
"""Return an iterator which yields the paths matching a pathname pattern.
The pattern may contain simple shell-style wildcards a la
fnmatch. However, unlike fnmatch, filenames starting with a
dot are special cases that are not matched by '*' and '?'
patterns.
If recursive is true, the pattern '**' will match any files and
zero or more directories and subdirectories.
"""
it = _iglob(pathname, recursive, False)
if recursive and _isrecursive(pathname):
s = next(it) # skip empty string
assert not s
return it
def _iglob(pathname, recursive, dironly):
dirname, basename = os.path.split(pathname)
if not has_magic(pathname):
assert not dironly
if basename:
if os.path.lexists(pathname):
yield pathname
else:
# Patterns ending with a slash should match only directories
if os.path.isdir(dirname):
yield pathname
return
if not dirname:
if recursive and _isrecursive(basename):
yield _glob2(dirname, basename, dironly)
else:
yield _glob1(dirname, basename, dironly)
return
# `os.path.split()` returns the argument itself as a dirname if it is a
# drive or UNC path. Prevent an infinite recursion if a drive or UNC path
# contains magic characters (i.e. r'\\?\C:').
if dirname != pathname and has_magic(dirname):
dirs = _iglob(dirname, recursive, True)
else:
dirs = [dirname]
if has_magic(basename):
if recursive and _isrecursive(basename):
glob_in_dir = _glob2
else:
glob_in_dir = _glob1
else:
glob_in_dir = _glob0
for dirname in dirs:
for name in glob_in_dir(dirname, basename, dironly):
yield os.path.join(dirname, name)
# These 2 helper functions non-recursively glob inside a literal directory.
# They return a list of basenames. _glob1 accepts a pattern while _glob0
# takes a literal basename (so it only has to check for its existence).
def _glob1(dirname, pattern, dironly):
names = list(_iterdir(dirname, dironly))
return fnmatch.filter(names, pattern)
def _glob0(dirname, basename, dironly):
if not basename:
# `os.path.split()` returns an empty basename for paths ending with a
# directory separator. 'q*x/' should match only directories.
if os.path.isdir(dirname):
return [basename]
else:
if os.path.lexists(os.path.join(dirname, basename)):
return [basename]
return []
# Following functions are not public but can be used by third-party code.
def glob0(dirname, pattern):
return _glob0(dirname, pattern, False)
def glob1(dirname, pattern):
return _glob1(dirname, pattern, False)
# This helper function recursively yields relative pathnames inside a literal
# directory.
def _glob2(dirname, pattern, dironly):
assert _isrecursive(pattern)
return [pattern[:0]] + list(_rlistdir(dirname, dironly))
# If dironly is false, yields all file names inside a directory.
# If dironly is true, yields only directory names.
def _iterdir(dirname, dironly):
if not dirname:
if isinstance(dirname, bytes):
dirname = bytes(os.curdir, 'ASCII')
else:
dirname = os.curdir
try:
for entry in os.listdir(dirname):
yield entry
except OSError:
return
# Recursively yields relative pathnames inside a literal directory.
def _rlistdir(dirname, dironly):
if not os.path.islink(dirname):
names = list(_iterdir(dirname, dironly))
for x in names:
yield x
path = os.path.join(dirname, x) if dirname else x
for y in _rlistdir(path, dironly):
yield os.path.join(x, y)
magic_check = re.compile('([*?[])')
magic_check_bytes = re.compile(b'([*?[])')
def has_magic(s):
if isinstance(s, bytes):
match = magic_check_bytes.search(s)
else:
match = magic_check.search(s)
return match is not None
def _ishidden(path):
return path[0] in ('.', b'.'[0])
def _isrecursive(pattern):
if isinstance(pattern, bytes):
return pattern == b'**'
else:
return pattern == '**'
def escape(pathname):
"""Escape all special characters.
"""
# Escaping is done by wrapping any of "*?[" between square brackets.
# Metacharacters do not work in the drive part and shouldn't be escaped.
drive, pathname = os.path.splitdrive(pathname)
if isinstance(pathname, bytes):
pathname = magic_check_bytes.sub(br'[\1]', pathname)
else:
pathname = magic_check.sub(r'[\1]', pathname)
return drive + pathname
scripts/build/package/package.py
-187
View File
@@ -1,187 +0,0 @@
#
# Copyright (c) Contributors to the Open 3D Engine Project.
# For complete copyright and license terms please see the LICENSE at the root of this distribution.
#
# SPDX-License-Identifier: Apache-2.0 OR MIT
#
#
import os
import sys
import zipfile
import timeit
import progressbar
from optparse import OptionParser
from PackageEnv import PackageEnv
cur_dir = cur_dir = os.path.dirname(os.path.abspath(__file__))
sys.path.insert(0, f'{cur_dir}/..')
from ci_build import build
from util import *
from glob3 import glob
def package(options):
package_env = PackageEnv(options.platform, options.type, options.package_env)
if not package_env.get('SKIP_BUILD'):
print(package_env.get('SKIP_BUILD'))
print('SKIP_BUILD is False, running CMake build...')
cmake_build(package_env)
# TODO Compile Assets
#if package_env.exists('ASSET_PROCESSOR_PATH'):
# compile_assets(package_env)
#create packages
package_targets = package_env.get('PACKAGE_TARGETS')
for package_target in package_targets:
create_package(package_env, package_target)
upload_package(package_env, package_target)
def get_python_path(package_env):
if sys.platform == 'win32':
return os.path.join(package_env.get('ENGINE_ROOT'), 'python', 'python.cmd')
else:
return os.path.join(package_env.get('ENGINE_ROOT'), 'python', 'python.sh')
def cmake_build(package_env):
build_targets = package_env.get('BUILD_TARGETS')
for build_target in build_targets:
build(build_target['BUILD_CONFIG_FILENAME'], build_target['PLATFORM'], build_target['TYPE'])
def create_package(package_env, package_target):
print('Creating zipfile for package target {}'.format(package_target))
cur_dir = os.path.dirname(os.path.abspath(__file__))
file_list_type = package_target['FILE_LIST_TYPE']
if file_list_type == 'All':
filelist = os.path.join(cur_dir, 'package_filelists', package_target['FILE_LIST'])
else:
filelist = os.path.join(cur_dir, 'Platform', file_list_type, 'package_filelists', package_target['FILE_LIST'])
with open(filelist, 'r') as source:
data = json.load(source)
lyengine = package_env.get('ENGINE_ROOT')
print('Calculating filelists...')
files = {}
if '@lyengine' in data:
files.update(filter_files(data['@lyengine'], lyengine))
if '@3rdParty' in data:
files.update(filter_files(data['@3rdParty'], package_env.get('THIRDPARTY_HOME')))
package_path = os.path.join(lyengine, package_target['PACKAGE_NAME'])
print('Creating zipfile at {}'.format(package_path))
start = timeit.default_timer()
with progressbar.ProgressBar(max_value=len(files), redirect_stderr=True) as bar:
with zipfile.ZipFile(package_path, 'w', compression=zipfile.ZIP_DEFLATED, allowZip64=True) as myzip:
i = 0
bar.update(i)
last_bar_update = timeit.default_timer()
for f in files:
if os.path.islink(f):
zipInfo = zipfile.ZipInfo(files[f])
zipInfo.create_system = 3
# long type of hex val of '0xA1ED0000L',
# say, symlink attr magic...
zipInfo.external_attr |= 0xA0000000
myzip.writestr(zipInfo, os.readlink(f))
else:
myzip.write(f, files[f])
i += 1
# Update progress bar every 2 minutes
if int(timeit.default_timer() - last_bar_update) > 120:
last_bar_update = timeit.default_timer()
bar.update(i)
bar.update(i)
stop = timeit.default_timer()
total_time = int(stop - start)
print('{} is created. Total time: {} seconds.'.format(package_path, total_time))
def get_MD5(file_path):
from hashlib import md5
chunk_size = 200 * 1024
h = md5()
with open(file_path, 'rb') as f:
while True:
chunk = f.read(chunk_size)
if len(chunk):
h.update(chunk)
else:
break
return h.hexdigest()
md5_file = '{}.MD5'.format(package_path)
print('Creating MD5 file at {}'.format(md5_file))
start = timeit.default_timer()
with open(md5_file, 'w') as output:
output.write(get_MD5(package_path))
stop = timeit.default_timer()
total_time = int(stop - start)
print('{} is created. Total time: {} seconds.'.format(md5_file, total_time))
def upload_package(package_env, package_target):
package_name = package_target['PACKAGE_NAME']
engine_root = package_env.get('ENGINE_ROOT')
internal_s3_bucket = package_env.get('INTERNAL_S3_BUCKET')
qa_s3_bucket = package_env.get('QA_S3_BUCKET')
s3_prefix = package_env.get('S3_PREFIX')
print(f'Uploading {package_name} to S3://{internal_s3_bucket}/{s3_prefix}/{package_name}')
cmd = ['aws', 's3', 'cp', os.path.join(engine_root, package_name), f's3://{internal_s3_bucket}/{s3_prefix}/{package_name}']
execute_system_call(cmd, stdout=subprocess.DEVNULL)
print(f'Uploading {package_name} to S3://{qa_s3_bucket}/{s3_prefix}/{package_name}')
cmd = ['aws', 's3', 'cp', os.path.join(engine_root, package_name), f's3://{qa_s3_bucket}/{s3_prefix}/{package_name}', '--acl', 'bucket-owner-full-control']
execute_system_call(cmd, stdout=subprocess.DEVNULL)
def filter_files(data, base, prefix='', support_symlinks=True):
includes = {}
excludes = set()
for key, value in data.items():
pattern = os.path.join(base, prefix, key)
if not isinstance(value, dict):
pattern = os.path.normpath(pattern)
result = glob(pattern, recursive=True)
files = [x for x in result if os.path.isfile(x) or (support_symlinks and os.path.islink(x))]
if value == "#exclude":
excludes.update(files)
elif value == "#include":
for file in files:
includes[file] = os.path.relpath(file, base)
else:
if value.startswith('#move:'):
for file in files:
file_name = os.path.relpath(file, os.path.join(base, prefix))
dst_dir = value.replace('#move:', '').strip(' ')
includes[file] = os.path.join(dst_dir, file_name)
elif value.startswith('#rename:'):
for file in files:
dst_file = value.replace('#rename:', '').strip(' ')
includes[file] = dst_file
else:
warn('Unknown directive {} for pattern {}'.format(value, pattern))
else:
includes.update(filter_files(value, base, os.path.join(prefix, key), support_symlinks))
for exclude in excludes:
try:
includes.pop(exclude)
except KeyError:
pass
return includes
def parse_args():
parser = OptionParser()
parser.add_option("--platform", dest="platform", default='consoles', help="Target platform to package")
parser.add_option("--type", dest="type", default='consoles', help="Package type")
parser.add_option("--package_env", dest="package_env", default="package_env.json",
help="JSON file that defines package environment variables")
(options, args) = parser.parse_args()
return options, args
if __name__ == "__main__":
(options, args) = parse_args()
package(options)
scripts/build/package/package_env.json
-11
View File
@@ -1,11 +0,0 @@
{
"global_env":{
"ENGINE_ROOT":"",
"THIRDPARTY_HOME":"",
"BRANCH_NAME":"",
"PACKAGE_NAME_PATTERN":"${BRANCH_NAME}-spectra",
"BUILD_NUMBER":"0",
"INTERNAL_S3_BUCKET": "ly-spectra-packages",
"QA_S3_BUCKET": "amazon.ly.lionbridgeshare/ly-spectra-packages"
}
}
scripts/build/package/package_filelists/all.json
-15
View File
@@ -1,15 +0,0 @@
{
"@lyengine": {
"**": "#include",
".git/**": "#exclude",
".gitattributes": "#exclude",
".gitignore": "#exclude",
".gitmodules": "#exclude",
".lfsconfig": "#exclude",
".p4ignore": "#exclude",
".submodules": "#exclude",
"**/*.pyc": "#exclude",
"**/*.pdb": "#exclude",
"build/*/packages/*/*.stamp": "#exclude"
}
}
scripts/build/package/package_filelists/symbols.json
-5
View File
@@ -1,5 +0,0 @@
{
"@lyengine": {
"**/*.pdb": "#include"
}
}
scripts/build/package/platform_exclusions.json
-12
View File
@@ -1,12 +0,0 @@
{
"all": {
"@lyengine": {
"**/Gems/Atom/RHI/DX12/External/pix/**": "#exclude",
"**/.idea/**": "#exclude",
"**/*.csproj*": "#exclude",
"**/.owner": "#exclude",
"**/WinPixEventRuntime.dll": "#exclude",
"**/XenonConsole.exe": "#exclude"
}
}
}
scripts/build/package/util.py → scripts/util/util.py
Executable → Regular
-15
View File
@@ -13,26 +13,11 @@ import sys
import subprocess
class LyBuildError(Exception):
def __init__(self, message):
super(LyBuildError, self).__init__(message)
def ly_build_error(message):
raise LyBuildError(message)
def error(message):
print(('Error: {}'.format(message)))
exit(1)
# Exit with status code 0 means it won't fail the whole build process
def safe_exit_with_error(message):
print(('Error: {}'.format(message)))
exit(0)
def warn(message):
print(('Warning: {}'.format(message)))