Merge branch 'development' into o3de_sdk/installer_configs
Signed-off-by: Esteban Papp <81431996+amznestebanpapp@users.noreply.github.com> # Conflicts: # cmake/Packaging.cmake # cmake/Projects.cmake # scripts/build/Platform/Windows/build_config.json
This commit is contained in:
Vendored
+20
-15
@@ -793,27 +793,32 @@ catch(Exception e) {
|
||||
}
|
||||
finally {
|
||||
try {
|
||||
if(env.SNS_TOPIC) {
|
||||
snsPublish(
|
||||
topicArn: env.SNS_TOPIC,
|
||||
subject:'Build Result',
|
||||
message:"${currentBuild.currentResult}:${BUILD_URL}:${env.RECREATE_VOLUME}:${env.CLEAN_OUTPUT_DIRECTORY}:${env.CLEAN_ASSETS}"
|
||||
)
|
||||
}
|
||||
node('controller') {
|
||||
if("${currentBuild.currentResult}" == "SUCCESS") {
|
||||
buildFailure = ""
|
||||
emailBody = "${BUILD_URL}\nSuccess!"
|
||||
} else {
|
||||
buildFailure = tm('${BUILD_FAILURE_ANALYZER}')
|
||||
emailBody = "${BUILD_URL}\n${buildFailure}!"
|
||||
if(env.SNS_TOPIC_BUILD_FAILURE) {
|
||||
message_json = ["build_url":env.BUILD_URL, "repository_name":env.REPOSITORY_NAME, "branch_name":env.BRANCH_NAME, "build_failure":buildFailure]
|
||||
snsPublish(
|
||||
topicArn: env.SNS_TOPIC_BUILD_FAILURE,
|
||||
subject:'Build Failure',
|
||||
message:JsonOutput.toJson(message_json)
|
||||
)
|
||||
}
|
||||
|
||||
}
|
||||
if(env.POST_AR_BUILD_SNS_TOPIC) {
|
||||
message_json = [
|
||||
"build_url": env.BUILD_URL,
|
||||
"build_number": env.BUILD_NUMBER,
|
||||
"repository_name": env.REPOSITORY_NAME,
|
||||
"branch_name": env.BRANCH_NAME,
|
||||
"build_result": "${currentBuild.currentResult}",
|
||||
"build_failure": buildFailure,
|
||||
"recreate_volume": env.RECREATE_VOLUME,
|
||||
"clean_output_directory": env.CLEAN_OUTPUT_DIRECTORY,
|
||||
"clean_assets": env.CLEAN_ASSETS
|
||||
]
|
||||
snsPublish(
|
||||
topicArn: env.POST_AR_BUILD_SNS_TOPIC,
|
||||
subject:'Build Result',
|
||||
message:JsonOutput.toJson(message_json)
|
||||
)
|
||||
}
|
||||
emailext (
|
||||
body: "${emailBody}",
|
||||
|
||||
@@ -31,7 +31,6 @@
|
||||
],
|
||||
"steps": [
|
||||
"profile_vs2019",
|
||||
"test_impact_analysis_profile_vs2019",
|
||||
"asset_profile_vs2019",
|
||||
"test_cpu_profile_vs2019"
|
||||
]
|
||||
@@ -306,6 +305,7 @@
|
||||
},
|
||||
"release_vs2019": {
|
||||
"TAGS": [
|
||||
"default",
|
||||
"nightly-incremental",
|
||||
"nightly-clean",
|
||||
"weekly-build-metrics"
|
||||
@@ -351,17 +351,21 @@
|
||||
"CMAKE_NATIVE_BUILD_ARGS": "/m /nologo"
|
||||
}
|
||||
},
|
||||
"windows_installer": {
|
||||
"installer_vs2019": {
|
||||
"TAGS": [
|
||||
"nightly-clean"
|
||||
"nightly-clean",
|
||||
"nightly-installer"
|
||||
],
|
||||
"PIPELINE_ENV":{
|
||||
"NODE_LABEL":"windows-packaging"
|
||||
},
|
||||
"COMMAND": "build_installer_windows.cmd",
|
||||
"PARAMETERS": {
|
||||
"CONFIGURATION": "profile",
|
||||
"OUTPUT_DIRECTORY": "build\\windows_vs2019",
|
||||
"CMAKE_OPTIONS": "-G \"Visual Studio 16 2019\" -DCMAKE_SYSTEM_VERSION=10.0 -DLY_UNITY_BUILD=TRUE -DLY_DISABLE_TEST_MODULES=TRUE -DLY_VERSION_ENGINE_NAME=o3de-sdk -DLY_INSTALLER_WIX_ROOT=\"!WIX!\"",
|
||||
"EXTRA_CMAKE_OPTIONS": "-DLY_INSTALLER_AUTO_GEN_TAG=ON -DLY_INSTALLER_DOWNLOAD_URL=https://www.o3debinaries.org -DLY_INSTALLER_LICENSE_URL=https://www.o3debinaries.org/license",
|
||||
"CPACK_BUCKET": "spectra-prism-staging-us-west-2",
|
||||
"EXTRA_CMAKE_OPTIONS": "-DLY_INSTALLER_AUTO_GEN_TAG=ON -DLY_INSTALLER_DOWNLOAD_URL=!INSTALLER_DOWNLOAD_URL! -DLY_INSTALLER_LICENSE_URL=!INSTALLER_DOWNLOAD_URL!/license",
|
||||
"CPACK_BUCKET": "!INSTALLER_BUCKET!",
|
||||
"CMAKE_LY_PROJECTS": "",
|
||||
"CMAKE_TARGET": "ALL_BUILD",
|
||||
"CMAKE_NATIVE_BUILD_ARGS": "/m /nologo"
|
||||
|
||||
@@ -57,7 +57,7 @@ IF ERRORLEVEL 1 (
|
||||
exit /b 1
|
||||
)
|
||||
|
||||
CALL :DeployCDKApplication AWSCore --all
|
||||
CALL :DeployCDKApplication AWSCore "-c disable_access_log=true --all"
|
||||
IF ERRORLEVEL 1 (
|
||||
exit /b 1
|
||||
)
|
||||
|
||||
@@ -18,6 +18,8 @@ from contextlib import contextmanager
|
||||
import threading
|
||||
import _thread
|
||||
|
||||
from botocore.config import Config
|
||||
|
||||
DEFAULT_REGION = 'us-west-2'
|
||||
DEFAULT_DISK_SIZE = 300
|
||||
DEFAULT_DISK_TYPE = 'gp2'
|
||||
@@ -173,10 +175,27 @@ def get_region_name():
|
||||
|
||||
|
||||
def get_ec2_client(region):
|
||||
client = boto3.client('ec2', region_name=region)
|
||||
client_config = Config(
|
||||
region_name=region,
|
||||
retries={
|
||||
'mode': 'standard'
|
||||
}
|
||||
)
|
||||
client = boto3.client('ec2', config=client_config)
|
||||
return client
|
||||
|
||||
|
||||
def get_ec2_resource(region):
|
||||
resource_config = Config(
|
||||
region_name=region,
|
||||
retries={
|
||||
'mode': 'standard'
|
||||
}
|
||||
)
|
||||
resource = boto3.resource('ec2', config=resource_config)
|
||||
return resource
|
||||
|
||||
|
||||
def get_ec2_instance_id():
|
||||
try:
|
||||
instance_id = urllib.request.urlopen('http://169.254.169.254/latest/meta-data/instance-id').read()
|
||||
@@ -233,6 +252,18 @@ def find_snapshot_id(ec2_client, snapshot_hint, repository_name, project, pipeli
|
||||
snapshot_id = snapshot['SnapshotId']
|
||||
return snapshot_id
|
||||
|
||||
|
||||
def offline_drive(disk_number=1):
|
||||
"""Use diskpart to offline a Windows drive"""
|
||||
with tempfile.NamedTemporaryFile(delete=False) as f:
|
||||
f.write(f"""
|
||||
select disk {disk_number}
|
||||
offline disk
|
||||
""".encode('utf-8'))
|
||||
subprocess.run(['diskpart', '/s', f.name])
|
||||
os.unlink(f.name)
|
||||
|
||||
|
||||
def create_volume(ec2_client, availability_zone, snapshot_hint, repository_name, project, pipeline, branch, platform, build_type, disk_size, disk_type):
|
||||
# The actual EBS default calculation for IOps is a floating point number, the closest approxmiation is 4x of the disk size for simplicity
|
||||
mount_name = get_mount_name(repository_name, project, pipeline, branch, platform, build_type)
|
||||
@@ -291,23 +322,26 @@ def create_volume(ec2_client, availability_zone, snapshot_hint, repository_name,
|
||||
def mount_volume_to_device(created):
|
||||
print('Mounting volume...')
|
||||
if os.name == 'nt':
|
||||
f = tempfile.NamedTemporaryFile(delete=False)
|
||||
f.write("""
|
||||
select disk 1
|
||||
online disk
|
||||
attribute disk clear readonly
|
||||
""".encode('utf-8')) # assume disk # for now
|
||||
# Verify drive is in an offline state.
|
||||
# Some Windows configs will automatically set new drives as online causing diskpart setup script to fail.
|
||||
offline_drive()
|
||||
|
||||
if created:
|
||||
print('Creating filesystem on new volume')
|
||||
f.write("""create partition primary
|
||||
select partition 1
|
||||
format quick fs=ntfs
|
||||
assign
|
||||
active
|
||||
""".encode('utf-8'))
|
||||
with tempfile.NamedTemporaryFile(delete=False) as f:
|
||||
f.write("""
|
||||
select disk 1
|
||||
online disk
|
||||
attribute disk clear readonly
|
||||
""".encode('utf-8')) # assume disk # for now
|
||||
|
||||
f.close()
|
||||
if created:
|
||||
print('Creating filesystem on new volume')
|
||||
f.write("""
|
||||
create partition primary
|
||||
select partition 1
|
||||
format quick fs=ntfs
|
||||
assign
|
||||
active
|
||||
""".encode('utf-8'))
|
||||
|
||||
subprocess.call(['diskpart', '/s', f.name])
|
||||
|
||||
@@ -358,14 +392,7 @@ def unmount_volume_from_device():
|
||||
print('Unmounting EBS volume from device...')
|
||||
if os.name == 'nt':
|
||||
kill_processes(MOUNT_PATH + 'workspace')
|
||||
f = tempfile.NamedTemporaryFile(delete=False)
|
||||
f.write("""
|
||||
select disk 1
|
||||
offline disk
|
||||
""".encode('utf-8'))
|
||||
f.close()
|
||||
subprocess.call('diskpart /s %s' % f.name)
|
||||
os.unlink(f.name)
|
||||
offline_drive()
|
||||
else:
|
||||
kill_processes(MOUNT_PATH)
|
||||
subprocess.call(['umount', '-f', MOUNT_PATH])
|
||||
@@ -395,14 +422,11 @@ def detach_volume_from_ec2_instance(volume, ec2_instance_id, force, timeout_dura
|
||||
|
||||
|
||||
def mount_ebs(snapshot_hint, repository_name, project, pipeline, branch, platform, build_type, disk_size, disk_type):
|
||||
session = boto3.session.Session()
|
||||
region = session.region_name
|
||||
if region is None:
|
||||
region = DEFAULT_REGION
|
||||
region = get_region_name()
|
||||
ec2_client = get_ec2_client(region)
|
||||
ec2_instance_id = get_ec2_instance_id()
|
||||
ec2_availability_zone = get_availability_zone()
|
||||
ec2_resource = boto3.resource('ec2', region_name=region)
|
||||
ec2_resource = get_ec2_resource(region)
|
||||
ec2_instance = ec2_resource.Instance(ec2_instance_id)
|
||||
|
||||
for volume in ec2_instance.volumes.all():
|
||||
@@ -469,7 +493,7 @@ def mount_ebs(snapshot_hint, repository_name, project, pipeline, branch, platfor
|
||||
def unmount_ebs():
|
||||
region = get_region_name()
|
||||
ec2_instance_id = get_ec2_instance_id()
|
||||
ec2_resource = boto3.resource('ec2', region_name=region)
|
||||
ec2_resource = get_ec2_resource(region)
|
||||
ec2_instance = ec2_resource.Instance(ec2_instance_id)
|
||||
|
||||
if os.path.isfile('envinject.properties'):
|
||||
|
||||
+6
-2
@@ -29,12 +29,13 @@ def add_args(parser, subparsers) -> None:
|
||||
# add the scripts/o3de directory to the front of the sys.path
|
||||
sys.path.insert(0, str(o3de_package_dir))
|
||||
from o3de import engine_properties, engine_template, gem_properties, global_project, register, print_registration, get_registration, \
|
||||
enable_gem, disable_gem, project_properties, sha256
|
||||
enable_gem, disable_gem, project_properties, sha256, download
|
||||
# Remove the temporarily added path
|
||||
sys.path = sys.path[1:]
|
||||
|
||||
# global_project
|
||||
global_project.add_args(subparsers)
|
||||
|
||||
# engine templaate
|
||||
engine_template.add_args(subparsers)
|
||||
|
||||
@@ -61,10 +62,13 @@ def add_args(parser, subparsers) -> None:
|
||||
|
||||
# modify gem properties
|
||||
gem_properties.add_args(subparsers)
|
||||
|
||||
|
||||
# sha256
|
||||
sha256.add_args(subparsers)
|
||||
|
||||
# download
|
||||
download.add_args(subparsers)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
# parse the command line args
|
||||
|
||||
@@ -6,20 +6,22 @@
|
||||
#
|
||||
#
|
||||
"""
|
||||
Implements functionality for downloading o3de objecs either locally or from a URI
|
||||
Implements functionality for downloading o3de objects either locally or from a URI
|
||||
"""
|
||||
|
||||
import argparse
|
||||
import hashlib
|
||||
import json
|
||||
import logging
|
||||
import os
|
||||
import pathlib
|
||||
import shutil
|
||||
import sys
|
||||
import urllib.parse
|
||||
import urllib.request
|
||||
import zipfile
|
||||
|
||||
from o3de import manifest, repo, utils, validation
|
||||
from o3de import manifest, repo, utils, validation, register
|
||||
|
||||
logger = logging.getLogger()
|
||||
logging.basicConfig()
|
||||
@@ -37,21 +39,21 @@ def unzip_manifest_json_data(download_zip_path: pathlib.Path, zip_file_name: str
|
||||
|
||||
def validate_downloaded_zip_sha256(download_uri_json_data: dict, download_zip_path: pathlib.Path,
|
||||
manifest_json_name) -> int:
|
||||
# if the engine.json has a sha256 check it against a sha256 of the zip
|
||||
# if the json has a sha256 check it against a sha256 of the zip
|
||||
try:
|
||||
sha256A = download_uri_json_data['sha256']
|
||||
except KeyError as e:
|
||||
logger.warn(f'SECURITY WARNING: The advertised o3de object you downloaded has no "sha256"!!! Be VERY careful!!!'
|
||||
f' We cannot verify this is the actually the advertised object!!!')
|
||||
logger.warn('SECURITY WARNING: The advertised o3de object you downloaded has no "sha256"!!! Be VERY careful!!!'
|
||||
' We cannot verify this is the actually the advertised object!!!')
|
||||
return 1
|
||||
else:
|
||||
sha256B = hashlib.sha256(download_zip_path.open('rb').read()).hexdigest()
|
||||
if sha256A != sha256B:
|
||||
logger.error(f'SECURITY VIOLATION: Downloaded zip sha256 {sha256B} does not match'
|
||||
f' the advertised "sha256":{sha256A} in the f{manifest_json_name}. Deleting unzipped files!!!')
|
||||
shutil.rmtree(dest_path)
|
||||
return 1
|
||||
f' the advertised "sha256":{sha256A} in the f{manifest_json_name}.')
|
||||
return 0
|
||||
|
||||
manifest_json_data = unzip_manifest_json_data(download_zip_path, manifest_json_name)
|
||||
unzipped_manifest_json_data = unzip_manifest_json_data(download_zip_path, manifest_json_name)
|
||||
|
||||
# remove the sha256 if present in the advertised downloadable manifest json
|
||||
# then compare it to the json in the zip, they should now be identical
|
||||
@@ -61,21 +63,13 @@ def validate_downloaded_zip_sha256(download_uri_json_data: dict, download_zip_pa
|
||||
pass
|
||||
|
||||
sha256A = hashlib.sha256(json.dumps(download_uri_json_data, indent=4).encode('utf8')).hexdigest()
|
||||
with unzipped_manifest_json.open('r') as s:
|
||||
try:
|
||||
unzipped_manifest_json_data = json.load(s)
|
||||
except json.JSONDecodeError as e:
|
||||
logger.error(f'Failed to read manifest json {unzipped_manifest_json}. Unable to confirm this'
|
||||
f' is the same template that was advertised.')
|
||||
return 1
|
||||
sha256B = hashlib.sha256(json.dumps(unzipped_manifest_json_data, indent=4).encode('utf8')).hexdigest()
|
||||
if sha256A != sha256B:
|
||||
logger.error(f'SECURITY VIOLATION: Downloaded manifest json does not match'
|
||||
f' the advertised manifest json. Deleting unzipped files!!!')
|
||||
shutil.rmtree(dest_path)
|
||||
return 1
|
||||
logger.error('SECURITY VIOLATION: Downloaded manifest json does not match'
|
||||
' the advertised manifest json.')
|
||||
return 0
|
||||
|
||||
return 0
|
||||
return 1
|
||||
|
||||
|
||||
def get_downloadable(engine_name: str = None,
|
||||
@@ -91,23 +85,15 @@ def get_downloadable(engine_name: str = None,
|
||||
return None
|
||||
|
||||
manifest_json = 'repo.json'
|
||||
search_func = lambda: repo.search_repo(manifest_json, engine_name, project_name, gem_name, template_name)
|
||||
search_func = lambda manifest_json_data: repo.search_repo(manifest_json_data, engine_name, project_name, gem_name, template_name)
|
||||
return repo.search_o3de_object(manifest_json, o3de_object_uris, search_func)
|
||||
|
||||
|
||||
def download_o3de_object(object_name: str, default_folder_name: str, dest_path: str or pathlib.Path,
|
||||
object_type: str, downloadable_kwarg_key) -> int:
|
||||
if not dest_path:
|
||||
dest_path = manifest.get_registered(default_folder=default_folder_name)
|
||||
if not dest_path:
|
||||
logger.error(f'Destination path not cannot be empty.')
|
||||
return 1
|
||||
object_type: str, downloadable_kwarg_key, skip_auto_register: bool) -> int:
|
||||
|
||||
dest_path = pathlib.Path(dest_path).resolve()
|
||||
dest_path.mkdir(exist_ok=True)
|
||||
|
||||
download_path = manifest.get_o3de_download_folder() / default_folder_name / object_name
|
||||
download_path.mkdir(exist_ok=True)
|
||||
download_path = manifest.get_o3de_cache_folder() / default_folder_name / object_name
|
||||
download_path.mkdir(parents=True, exist_ok=True)
|
||||
download_zip_path = download_path / f'{object_type}.zip'
|
||||
|
||||
downloadable_object_data = get_downloadable(**{downloadable_kwarg_key : object_name})
|
||||
@@ -115,41 +101,79 @@ def download_o3de_object(object_name: str, default_folder_name: str, dest_path:
|
||||
logger.error(f'Downloadable o3de object {object_name} not found.')
|
||||
return 1
|
||||
|
||||
origin = downloadable_json_data['origin']
|
||||
url = f'{origin}/object_type.zip'
|
||||
parsed_uri = urllib.parse.urlparse(url)
|
||||
origin_uri = downloadable_object_data['originuri']
|
||||
parsed_uri = urllib.parse.urlparse(origin_uri)
|
||||
|
||||
download_zip_result = utils.download_zip_file(parsed_uri, download_zip_path)
|
||||
if download_zip_result != 0:
|
||||
return download_zip_result
|
||||
|
||||
return validate_downloaded_zip_sha256(downloadable_object_data, download_zip_path)
|
||||
if not validate_downloaded_zip_sha256(downloadable_object_data, download_zip_path, f'{object_type}.json'):
|
||||
logger.error(f'Could not validate zip, deleting {download_zip_path}')
|
||||
os.unlink(download_zip_path)
|
||||
return 1
|
||||
|
||||
if not dest_path:
|
||||
dest_path = manifest.get_registered(default_folder=default_folder_name)
|
||||
dest_path = pathlib.Path(dest_path).resolve()
|
||||
dest_path = dest_path / object_name
|
||||
else:
|
||||
dest_path = pathlib.Path(dest_path).resolve()
|
||||
|
||||
if not dest_path:
|
||||
logger.error(f'Destination path cannot be empty.')
|
||||
return 1
|
||||
if dest_path.exists():
|
||||
logger.error(f'Destination path {dest_path} already exists.')
|
||||
return 1
|
||||
|
||||
dest_path.mkdir(exist_ok=True)
|
||||
|
||||
# extract zip
|
||||
with zipfile.ZipFile(download_zip_path, 'r') as zip_file_ref:
|
||||
try:
|
||||
zip_file_ref.extractall(dest_path)
|
||||
except Exception:
|
||||
logger.error(f'Error unzipping {download_zip_path} to {dest_path}. Deleting {dest_path}.')
|
||||
shutil.rmtree(dest_path)
|
||||
return 1
|
||||
|
||||
if not skip_auto_register:
|
||||
if object_type == 'gem':
|
||||
return register.register(gem_path=dest_path)
|
||||
|
||||
return 0
|
||||
|
||||
|
||||
def download_engine(engine_name: str,
|
||||
dest_path: str or pathlib.Path) -> int:
|
||||
return download_o3de_object(engine_name, 'engines', dest_path, 'engine', 'engine_name')
|
||||
dest_path: str or pathlib.Path,
|
||||
skip_auto_register: bool) -> int:
|
||||
return download_o3de_object(engine_name, 'engines', dest_path, 'engine', 'engine_name', skip_auto_register)
|
||||
|
||||
|
||||
def download_project(project_name: str,
|
||||
dest_path: str or pathlib.Path) -> int:
|
||||
return download_o3de_object(project_name, 'projects', dest_path, 'project', 'project_name')
|
||||
dest_path: str or pathlib.Path,
|
||||
skip_auto_register: bool) -> int:
|
||||
return download_o3de_object(project_name, 'projects', dest_path, 'project', 'project_name', skip_auto_register)
|
||||
|
||||
|
||||
def download_gem(gem_name: str,
|
||||
dest_path: str or pathlib.Path) -> int:
|
||||
return download_o3de_object(gem_name, 'gems', dest_path, 'gem', 'gem_name')
|
||||
dest_path: str or pathlib.Path,
|
||||
skip_auto_register: bool) -> int:
|
||||
return download_o3de_object(gem_name, 'gems', dest_path, 'gem', 'gem_name', skip_auto_register)
|
||||
|
||||
|
||||
def download_template(template_name: str,
|
||||
dest_path: str or pathlib.Path) -> int:
|
||||
return download_o3de_object(template_name, 'templates', dest_path, 'template', 'template_name')
|
||||
dest_path: str or pathlib.Path,
|
||||
skip_auto_register: bool) -> int:
|
||||
return download_o3de_object(template_name, 'templates', dest_path, 'template', 'template_name', skip_auto_register)
|
||||
|
||||
|
||||
|
||||
def download_restricted(restricted_name: str,
|
||||
dest_path: str or pathlib.Path) -> int:
|
||||
return download_o3de_object(restricted_name, 'restricted', dest_path, 'restricted', 'restricted_name')
|
||||
dest_path: str or pathlib.Path,
|
||||
skip_auto_register: bool) -> int:
|
||||
return download_o3de_object(restricted_name, 'restricted', dest_path, 'restricted', 'restricted_name', skip_auto_register)
|
||||
|
||||
|
||||
def _run_download(args: argparse) -> int:
|
||||
@@ -158,16 +182,20 @@ def _run_download(args: argparse) -> int:
|
||||
|
||||
if args.engine_name:
|
||||
return download_engine(args.engine_name,
|
||||
args.dest_path)
|
||||
args.dest_path,
|
||||
args.skip_auto_register)
|
||||
elif args.project_name:
|
||||
return download_project(args.project_name,
|
||||
args.dest_path)
|
||||
elif args.gem_nanme:
|
||||
args.dest_path,
|
||||
args.skip_auto_register)
|
||||
elif args.gem_name:
|
||||
return download_gem(args.gem_name,
|
||||
args.dest_path)
|
||||
args.dest_path,
|
||||
args.skip_auto_register)
|
||||
elif args.template_name:
|
||||
return download_template(args.template_name,
|
||||
args.dest_path)
|
||||
args.dest_path,
|
||||
args.skip_auto_register)
|
||||
|
||||
return 1
|
||||
|
||||
@@ -188,14 +216,16 @@ def add_parser_args(parser):
|
||||
group.add_argument('-t', '--template-name', type=str, required=False,
|
||||
help='Downloadable template name.')
|
||||
parser.add_argument('-dp', '--dest-path', type=str, required=False,
|
||||
default=None,
|
||||
help='Optional destination folder to download into.'
|
||||
' i.e. download --project-name "AstomSamplerViewer" --dest-path "C:/projects"'
|
||||
' will result in C:/projects/AtomSampleViewer'
|
||||
' If blank will download to default object type folder')
|
||||
|
||||
default=None,
|
||||
help='Optional destination folder to download into.'
|
||||
' i.e. download --project-name "CustomProject" --dest-path "C:/projects"'
|
||||
' will result in C:/projects/CustomProject'
|
||||
' If blank will download to default object type folder')
|
||||
parser.add_argument('-sar', '--skip-auto-register', action='store_true', required=False,
|
||||
default=False,
|
||||
help = 'Skip the automatic registration of new object download')
|
||||
parser.add_argument('-ohf', '--override-home-folder', type=str, required=False,
|
||||
help='By default the home folder is the user folder, override it to this folder.')
|
||||
help='By default the home folder is the user folder, override it to this folder.')
|
||||
|
||||
parser.set_defaults(func=_run_download)
|
||||
|
||||
|
||||
@@ -59,6 +59,14 @@ binary_file_ext = {
|
||||
'.motionset'
|
||||
}
|
||||
|
||||
cpp_file_ext = {
|
||||
'.cpp',
|
||||
'.h',
|
||||
'.hpp',
|
||||
'.hxx',
|
||||
'.inl'
|
||||
}
|
||||
|
||||
expect_license_info_ext = {
|
||||
'.cpp',
|
||||
'.h',
|
||||
@@ -402,7 +410,7 @@ def create_template(source_path: pathlib.Path,
|
||||
# if no template path, error
|
||||
if not template_path:
|
||||
logger.info(f'Template path empty. Using source name {source_name}')
|
||||
template_path = source_name
|
||||
template_path = pathlib.Path(source_name)
|
||||
if not template_path.is_absolute():
|
||||
default_templates_folder = manifest.get_registered(default_folder='templates')
|
||||
template_path = default_templates_folder / template_path
|
||||
@@ -518,21 +526,52 @@ def create_template(source_path: pathlib.Path,
|
||||
replacements.append((source_name.upper(), '${NameUpper}'))
|
||||
replacements.append((source_name, '${Name}'))
|
||||
replacements.append((sanitized_source_name, '${SanitizedCppName}'))
|
||||
sanitized_name_index = len(replacements) - 1
|
||||
|
||||
def _transform_into_template(s_data: object) -> (bool, str):
|
||||
def _is_cpp_file(file_path: pathlib.Path) -> bool:
|
||||
"""
|
||||
Internal helper method to check if a file is a C++ file based
|
||||
on its extension, so we can determine if we need to prefer
|
||||
the ${SanitizedCppName}
|
||||
:param file_path: The input file path
|
||||
:return: bool: Whether or not the input file path has a C++ extension
|
||||
"""
|
||||
name, ext = os.path.splitext(file_path)
|
||||
|
||||
return ext.lower() in cpp_file_ext
|
||||
|
||||
def _transform_into_template(s_data: object,
|
||||
prefer_sanitized_name: bool = False) -> (bool, str):
|
||||
"""
|
||||
Internal function to transform any data into templated data
|
||||
:param s_data: the input data, this could be file data or file name data
|
||||
:param prefer_sanitized_name: Optionally swap the sanitized name with the normal name
|
||||
This can be necessary when creating the template, the source
|
||||
name and sanitized source name might be the same, but C++
|
||||
files will need to prefer the sanitized version, or else
|
||||
there might be compile errors (e.g. '-' characters in the name)
|
||||
:return: bool: whether or not the returned data MAY need to be transformed to instantiate it
|
||||
t_data: potentially transformed data 0 for success or non 0 failure code
|
||||
"""
|
||||
def swap_sanitized_name_and_normal():
|
||||
replacements[sanitized_name_index-1], replacements[sanitized_name_index] = \
|
||||
replacements[sanitized_name_index], replacements[sanitized_name_index-1]
|
||||
|
||||
# copy the src data to the transformed data, then operate only on transformed data
|
||||
t_data = str(s_data)
|
||||
|
||||
# If we need to prefer the sanitized name, then swap it for the normal
|
||||
if prefer_sanitized_name:
|
||||
swap_sanitized_name_and_normal()
|
||||
|
||||
# run all the replacements
|
||||
for replacement in replacements:
|
||||
t_data = t_data.replace(replacement[0], replacement[1])
|
||||
|
||||
# Once we are done running the replacements, reset the list if we had modified it
|
||||
if prefer_sanitized_name:
|
||||
swap_sanitized_name_and_normal()
|
||||
|
||||
if not keep_license_text:
|
||||
t_data = _replace_license_text(t_data)
|
||||
|
||||
@@ -704,7 +743,7 @@ def create_template(source_path: pathlib.Path,
|
||||
# open the file and attempt to transform it
|
||||
with open(entry_abs, 'r') as s:
|
||||
source_data = s.read()
|
||||
templated, source_data = _transform_into_template(source_data)
|
||||
templated, source_data = _transform_into_template(source_data, _is_cpp_file(entry_abs))
|
||||
|
||||
# if the file type is a file that we expect to fins license header and we don't find any
|
||||
# warn that the we didn't find the license info, this makes it easy to make sure we didn't
|
||||
@@ -840,7 +879,7 @@ def create_template(source_path: pathlib.Path,
|
||||
# open the file and attempt to transform it
|
||||
with open(entry_abs, 'r') as s:
|
||||
source_data = s.read()
|
||||
templated, source_data = _transform_into_template(source_data)
|
||||
templated, source_data = _transform_into_template(source_data, _is_cpp_file(entry_abs))
|
||||
|
||||
# if the file type is a file that we expect to fins license header and we don't find any
|
||||
# warn that the we didn't find the license info, this makes it easy to make sure we didn't
|
||||
|
||||
+68
-117
@@ -13,8 +13,10 @@ import json
|
||||
import logging
|
||||
import os
|
||||
import pathlib
|
||||
import shutil
|
||||
import hashlib
|
||||
|
||||
from o3de import validation
|
||||
from o3de import validation, utils
|
||||
|
||||
logger = logging.getLogger()
|
||||
logging.basicConfig()
|
||||
@@ -135,12 +137,12 @@ def get_o3de_manifest() -> pathlib.Path:
|
||||
json_data.update({'default_restricted_folder': default_restricted_folder.as_posix()})
|
||||
json_data.update({'default_third_party_folder': default_third_party_folder.as_posix()})
|
||||
|
||||
json_data.update({'engines': []})
|
||||
json_data.update({'projects': []})
|
||||
json_data.update({'external_subdirectories': []})
|
||||
json_data.update({'templates': []})
|
||||
json_data.update({'restricted': []})
|
||||
json_data.update({'repos': []})
|
||||
json_data.update({'engines': []})
|
||||
|
||||
default_restricted_folder_json = default_restricted_folder / 'restricted.json'
|
||||
if not default_restricted_folder_json.is_file():
|
||||
@@ -197,11 +199,11 @@ def load_o3de_manifest(manifest_path: pathlib.Path = None) -> dict:
|
||||
|
||||
def save_o3de_manifest(json_data: dict, manifest_path: pathlib.Path = None) -> bool:
|
||||
"""
|
||||
Save the json dictionary to the supplied manifest file or ~/.o3de/o3de_manifest.json if manifest_path is None
|
||||
Save the json dictionary to the supplied manifest file or ~/.o3de/o3de_manifest.json if None
|
||||
|
||||
:param json_data: dictionary to save in json format at the file path
|
||||
:param manifest_path: optional path to manifest file to save
|
||||
"""
|
||||
:param json_data: dictionary to save in json format at the file path
|
||||
:param manifest_path: optional path to manifest file to save
|
||||
"""
|
||||
if not manifest_path:
|
||||
manifest_path = get_o3de_manifest()
|
||||
with manifest_path.open('w') as s:
|
||||
@@ -213,7 +215,6 @@ def save_o3de_manifest(json_data: dict, manifest_path: pathlib.Path = None) -> b
|
||||
return False
|
||||
|
||||
|
||||
|
||||
def get_gems_from_subdirectories(external_subdirs: list) -> list:
|
||||
'''
|
||||
Helper Method for scanning a set of external subdirectories for gem.json files
|
||||
@@ -235,7 +236,6 @@ def get_gems_from_subdirectories(external_subdirs: list) -> list:
|
||||
return gem_directories
|
||||
|
||||
|
||||
# Data query methods
|
||||
def get_engines() -> list:
|
||||
json_data = load_o3de_manifest()
|
||||
engine_list = json_data['engines'] if 'engines' in json_data else []
|
||||
@@ -421,6 +421,48 @@ def get_templates_for_generic_creation(): # temporary until we have a better wa
|
||||
|
||||
return list(filter(filter_project_and_gem_templates_out, get_all_templates()))
|
||||
|
||||
def get_json_file_path(object_typename: str,
|
||||
object_path: str or pathlib.Path) -> pathlib.Path:
|
||||
if not object_typename or not object_path:
|
||||
logger.error('Must specify an object typename and object path.')
|
||||
return None
|
||||
|
||||
object_path = pathlib.Path(object_path).resolve()
|
||||
return object_path / f'{object_typename}.json'
|
||||
|
||||
|
||||
def get_json_data_file(object_json: pathlib.Path,
|
||||
object_typename: str,
|
||||
object_validator: callable) -> dict or None:
|
||||
if not object_typename:
|
||||
logger.error('Missing object typename.')
|
||||
return None
|
||||
|
||||
if not object_json or not object_json.is_file():
|
||||
logger.error(f'Invalid {object_typename} json {object_json} supplied or file missing.')
|
||||
return None
|
||||
|
||||
if not object_validator or not object_validator(object_json):
|
||||
logger.error(f'{object_typename} json {object_json} is not valid or could not be validated.')
|
||||
return None
|
||||
|
||||
with object_json.open('r') as f:
|
||||
try:
|
||||
object_json_data = json.load(f)
|
||||
except json.JSONDecodeError as e:
|
||||
logger.warn(f'{object_json} failed to load: {e}')
|
||||
else:
|
||||
return object_json_data
|
||||
|
||||
return None
|
||||
|
||||
def get_json_data(object_typename: str,
|
||||
object_path: str or pathlib.Path,
|
||||
object_validator: callable) -> dict or None:
|
||||
object_json = get_json_file_path(object_typename, object_path)
|
||||
|
||||
return get_json_data_file(object_json, object_typename, object_validator)
|
||||
|
||||
|
||||
def get_engine_json_data(engine_name: str = None,
|
||||
engine_path: str or pathlib.Path = None) -> dict or None:
|
||||
@@ -431,28 +473,7 @@ def get_engine_json_data(engine_name: str = None,
|
||||
if engine_name and not engine_path:
|
||||
engine_path = get_registered(engine_name=engine_name)
|
||||
|
||||
if not engine_path:
|
||||
logger.error(f'Engine Path {engine_path} has not been registered.')
|
||||
return None
|
||||
|
||||
engine_path = pathlib.Path(engine_path).resolve()
|
||||
engine_json = engine_path / 'engine.json'
|
||||
if not engine_json.is_file():
|
||||
logger.error(f'Engine json {engine_json} is not present.')
|
||||
return None
|
||||
if not validation.valid_o3de_engine_json(engine_json):
|
||||
logger.error(f'Engine json {engine_json} is not valid.')
|
||||
return None
|
||||
|
||||
with engine_json.open('r') as f:
|
||||
try:
|
||||
engine_json_data = json.load(f)
|
||||
except json.JSONDecodeError as e:
|
||||
logger.warn(f'{engine_json} failed to load: {str(e)}')
|
||||
else:
|
||||
return engine_json_data
|
||||
|
||||
return None
|
||||
return get_json_data('engine', engine_path, validation.valid_o3de_engine_json)
|
||||
|
||||
|
||||
def get_project_json_data(project_name: str = None,
|
||||
@@ -464,28 +485,7 @@ def get_project_json_data(project_name: str = None,
|
||||
if project_name and not project_path:
|
||||
project_path = get_registered(project_name=project_name)
|
||||
|
||||
if not project_path:
|
||||
logger.error(f'Project Path {project_path} has not been registered.')
|
||||
return None
|
||||
|
||||
project_path = pathlib.Path(project_path).resolve()
|
||||
project_json = project_path / 'project.json'
|
||||
if not project_json.is_file():
|
||||
logger.error(f'Project json {project_json} is not present.')
|
||||
return None
|
||||
if not validation.valid_o3de_project_json(project_json):
|
||||
logger.error(f'Project json {project_json} is not valid.')
|
||||
return None
|
||||
|
||||
with project_json.open('r') as f:
|
||||
try:
|
||||
project_json_data = json.load(f)
|
||||
except json.JSONDecodeError as e:
|
||||
logger.warn(f'{project_json} failed to load: {str(e)}')
|
||||
else:
|
||||
return project_json_data
|
||||
|
||||
return None
|
||||
return get_json_data('project', project_path, validation.valid_o3de_project_json)
|
||||
|
||||
|
||||
def get_gem_json_data(gem_name: str = None, gem_path: str or pathlib.Path = None,
|
||||
@@ -497,28 +497,7 @@ def get_gem_json_data(gem_name: str = None, gem_path: str or pathlib.Path = None
|
||||
if gem_name and not gem_path:
|
||||
gem_path = get_registered(gem_name=gem_name, project_path=project_path)
|
||||
|
||||
if not gem_path:
|
||||
logger.error(f'Gem Path {gem_path} has not been registered.')
|
||||
return None
|
||||
|
||||
gem_path = pathlib.Path(gem_path).resolve()
|
||||
gem_json = gem_path / 'gem.json'
|
||||
if not gem_json.is_file():
|
||||
logger.error(f'Gem json {gem_json} is not present.')
|
||||
return None
|
||||
if not validation.valid_o3de_gem_json(gem_json):
|
||||
logger.error(f'Gem json {gem_json} is not valid.')
|
||||
return None
|
||||
|
||||
with gem_json.open('r') as f:
|
||||
try:
|
||||
gem_json_data = json.load(f)
|
||||
except json.JSONDecodeError as e:
|
||||
logger.warn(f'{gem_json} failed to load: {str(e)}')
|
||||
else:
|
||||
return gem_json_data
|
||||
|
||||
return None
|
||||
return get_json_data('gem', gem_path, validation.valid_o3de_gem_json)
|
||||
|
||||
|
||||
def get_template_json_data(template_name: str = None, template_path: str or pathlib.Path = None,
|
||||
@@ -530,28 +509,7 @@ def get_template_json_data(template_name: str = None, template_path: str or path
|
||||
if template_name and not template_path:
|
||||
template_path = get_registered(template_name=template_name, project_path=project_path)
|
||||
|
||||
if not template_path:
|
||||
logger.error(f'Template Path {template_path} has not been registered.')
|
||||
return None
|
||||
|
||||
template_path = pathlib.Path(template_path).resolve()
|
||||
template_json = template_path / 'template.json'
|
||||
if not template_json.is_file():
|
||||
logger.error(f'Template json {template_json} is not present.')
|
||||
return None
|
||||
if not validation.valid_o3de_template_json(template_json):
|
||||
logger.error(f'Template json {template_json} is not valid.')
|
||||
return None
|
||||
|
||||
with template_json.open('r') as f:
|
||||
try:
|
||||
template_json_data = json.load(f)
|
||||
except json.JSONDecodeError as e:
|
||||
logger.warn(f'{template_json} failed to load: {str(e)}')
|
||||
else:
|
||||
return template_json_data
|
||||
|
||||
return None
|
||||
return get_json_data('template', template_path, validation.valid_o3de_template_json)
|
||||
|
||||
|
||||
def get_restricted_json_data(restricted_name: str = None, restricted_path: str or pathlib.Path = None,
|
||||
@@ -563,29 +521,24 @@ def get_restricted_json_data(restricted_name: str = None, restricted_path: str o
|
||||
if restricted_name and not restricted_path:
|
||||
restricted_path = get_registered(restricted_name=restricted_name, project_path=project_path)
|
||||
|
||||
if not restricted_path:
|
||||
logger.error(f'Restricted Path {restricted_path} has not been registered.')
|
||||
return get_json_data('restricted', restricted_path, validation.valid_o3de_restricted_json)
|
||||
|
||||
def get_repo_json_data(repo_uri: str) -> dict or None:
|
||||
if not repo_uri:
|
||||
logger.error('Must specify a Repo Uri.')
|
||||
return None
|
||||
|
||||
restricted_path = pathlib.Path(restricted_path).resolve()
|
||||
restricted_json = restricted_path / 'restricted.json'
|
||||
if not restricted_json.is_file():
|
||||
logger.error(f'Restricted json {restricted_json} is not present.')
|
||||
return None
|
||||
if not validation.valid_o3de_restricted_json(restricted_json):
|
||||
logger.error(f'Restricted json {restricted_json} is not valid.')
|
||||
return None
|
||||
repo_json = get_repo_path(repo_uri=repo_uri)
|
||||
|
||||
with restricted_json.open('r') as f:
|
||||
try:
|
||||
restricted_json_data = json.load(f)
|
||||
except json.JSONDecodeError as e:
|
||||
logger.warn(f'{restricted_json} failed to load: {str(e)}')
|
||||
else:
|
||||
return restricted_json_data
|
||||
return get_json_data_file(repo_json, "Repo", validation.valid_o3de_repo_json)
|
||||
|
||||
return None
|
||||
def get_repo_path(repo_uri: str, cache_folder: str = None) -> pathlib.Path:
|
||||
if not cache_folder:
|
||||
cache_folder = get_o3de_cache_folder()
|
||||
|
||||
repo_manifest = f'{repo_uri}/repo.json'
|
||||
repo_sha256 = hashlib.sha256(repo_manifest.encode())
|
||||
return cache_folder / str(repo_sha256.hexdigest() + '.json')
|
||||
|
||||
def get_registered(engine_name: str = None,
|
||||
project_name: str = None,
|
||||
@@ -721,9 +674,7 @@ def get_registered(engine_name: str = None,
|
||||
elif isinstance(repo_name, str):
|
||||
cache_folder = get_o3de_cache_folder()
|
||||
for repo_uri in json_data['repos']:
|
||||
repo_uri = pathlib.Path(repo_uri).resolve()
|
||||
repo_sha256 = hashlib.sha256(repo_uri.encode())
|
||||
cache_file = cache_folder / str(repo_sha256.hexdigest() + '.json')
|
||||
cache_file = get_repo_path(repo_uri=repo_uri, cache_folder=cache_folder)
|
||||
if cache_file.is_file():
|
||||
repo = pathlib.Path(cache_file).resolve()
|
||||
with repo.open('r') as f:
|
||||
|
||||
@@ -487,14 +487,14 @@ def register_repo(json_data: dict,
|
||||
if remove:
|
||||
logger.warn(f'Removing repo uri {repo_uri}.')
|
||||
return 0
|
||||
|
||||
repo_sha256 = hashlib.sha256(url.encode())
|
||||
cache_file = manifest.get_o3de_cache_folder() / str(repo_sha256.hexdigest() + '.json')
|
||||
|
||||
result = utils.download_file(url, cache_file)
|
||||
result = utils.download_file(parsed_uri, cache_file)
|
||||
if result == 0:
|
||||
json_data['repos'].insert(0, repo_uri.as_posix())
|
||||
json_data['repos'].insert(0, repo_uri)
|
||||
|
||||
repo_set = set()
|
||||
result = repo.process_add_o3de_repo(cache_file, repo_set)
|
||||
|
||||
return result
|
||||
@@ -621,6 +621,7 @@ def register(engine_path: pathlib.Path = None,
|
||||
return 1
|
||||
result = result or register_gem_path(json_data, gem_path, remove,
|
||||
external_subdir_engine_path, external_subdir_project_path)
|
||||
|
||||
if isinstance(external_subdir_path, pathlib.PurePath):
|
||||
if not external_subdir_path:
|
||||
logger.error(f'External Subdirectory path is None.')
|
||||
|
||||
+71
-29
@@ -12,6 +12,7 @@ import pathlib
|
||||
import shutil
|
||||
import urllib.parse
|
||||
import urllib.request
|
||||
import hashlib
|
||||
|
||||
from o3de import manifest, utils, validation
|
||||
|
||||
@@ -24,7 +25,6 @@ def process_add_o3de_repo(file_name: str or pathlib.Path,
|
||||
file_name = pathlib.Path(file_name).resolve()
|
||||
if not validation.valid_o3de_repo_json(file_name):
|
||||
return 1
|
||||
|
||||
cache_folder = manifest.get_o3de_cache_folder()
|
||||
|
||||
with file_name.open('r') as f:
|
||||
@@ -34,11 +34,30 @@ def process_add_o3de_repo(file_name: str or pathlib.Path,
|
||||
logger.error(f'{file_name} failed to load: {str(e)}')
|
||||
return 1
|
||||
|
||||
for o3de_object_uris, manifest_json in [(repo_data['engines'], 'engine.json'),
|
||||
(repo_data['projects'], 'project.json'),
|
||||
(repo_data['gems'], 'gem.json'),
|
||||
(repo_data['template'], 'template.json'),
|
||||
(repo_data['restricted'], 'restricted.json')]:
|
||||
# A repo may not contain all types of object.
|
||||
manifest_download_list = []
|
||||
try:
|
||||
manifest_download_list.append((repo_data['engines'], 'engine.json'))
|
||||
except KeyError:
|
||||
pass
|
||||
try:
|
||||
manifest_download_list.append((repo_data['projects'], 'project.json'))
|
||||
except KeyError:
|
||||
pass
|
||||
try:
|
||||
manifest_download_list.append((repo_data['gems'], 'gem.json'))
|
||||
except KeyError:
|
||||
pass
|
||||
try:
|
||||
manifest_download_list.append((repo_data['templates'], 'template.json'))
|
||||
except KeyError:
|
||||
pass
|
||||
try:
|
||||
manifest_download_list.append((repo_data['restricted'], 'restricted.json'))
|
||||
except KeyError:
|
||||
pass
|
||||
|
||||
for o3de_object_uris, manifest_json in manifest_download_list:
|
||||
for o3de_object_uri in o3de_object_uris:
|
||||
manifest_json_uri = f'{o3de_object_uri}/{manifest_json}'
|
||||
manifest_json_sha256 = hashlib.sha256(manifest_json_uri.encode())
|
||||
@@ -49,7 +68,27 @@ def process_add_o3de_repo(file_name: str or pathlib.Path,
|
||||
if download_file_result != 0:
|
||||
return download_file_result
|
||||
|
||||
repo_set |= repo_data['repos']
|
||||
# Having a repo is also optional
|
||||
repo_list = []
|
||||
try:
|
||||
repo_list.append(repo_data['repos'])
|
||||
except KeyError:
|
||||
pass
|
||||
|
||||
for repo in repo_list:
|
||||
if repo not in repo_set:
|
||||
repo_set.add(repo)
|
||||
for o3de_object_uri in o3de_object_uris:
|
||||
parsed_uri = urllib.parse.urlparse(f'{repo}/repo.json')
|
||||
manifest_json_sha256 = hashlib.sha256(parsed_uri.geturl().encode())
|
||||
cache_file = cache_folder / str(manifest_json_sha256.hexdigest() + '.json')
|
||||
if cache_file.is_file():
|
||||
cache_file.unlink()
|
||||
download_file_result = utils.download_file(parsed_uri, cache_file)
|
||||
if download_file_result != 0:
|
||||
return download_file_result
|
||||
|
||||
return process_add_o3de_repo(parsed_uri.geturl(), repo_set)
|
||||
return 0
|
||||
|
||||
|
||||
@@ -70,11 +109,10 @@ def refresh_repos() -> int:
|
||||
if repo_uri not in repo_set:
|
||||
repo_set.add(repo_uri)
|
||||
|
||||
repo_uri = f'{repo_uri}/repo.json'
|
||||
repo_sha256 = hashlib.sha256(repo_uri.encode())
|
||||
parsed_uri = urllib.parse.urlparse(f'{repo_uri}/repo.json')
|
||||
repo_sha256 = hashlib.sha256(parsed_uri.geturl().encode())
|
||||
cache_file = cache_folder / str(repo_sha256.hexdigest() + '.json')
|
||||
if not cache_file.is_file():
|
||||
parsed_uri = urllib.parse.urlparse(repo_uri)
|
||||
download_file_result = utils.download_file(parsed_uri, cache_file)
|
||||
if download_file_result != 0:
|
||||
return download_file_result
|
||||
@@ -91,49 +129,53 @@ def refresh_repos() -> int:
|
||||
return result
|
||||
|
||||
|
||||
def search_repo(repo_json_data: dict,
|
||||
def search_repo(manifest_json_data: dict,
|
||||
engine_name: str = None,
|
||||
project_name: str = None,
|
||||
gem_name: str = None,
|
||||
template_name: str = None,
|
||||
restricted_name: str = None) -> dict or None:
|
||||
|
||||
if isinstance(engine_name, str) or isinstance(engine_name, pathlib.PurePath):
|
||||
o3de_object_uris = repo_json_data['engines']
|
||||
o3de_object_uris = manifest_json_data['engines']
|
||||
manifest_json = 'engine.json'
|
||||
json_key = 'engine_name'
|
||||
search_func = lambda: None if manifest_json_data.get(json_key, '') == engine_name else manifest_json_data
|
||||
search_func = lambda manifest_json_data: manifest_json_data if manifest_json_data.get(json_key, '') == engine_name else None
|
||||
elif isinstance(project_name, str) or isinstance(project_name, pathlib.PurePath):
|
||||
o3de_object_uris = repo_json_data['projects']
|
||||
o3de_object_uris = manifest_json_data['projects']
|
||||
manifest_json = 'project.json'
|
||||
json_key = 'project_name'
|
||||
search_func = lambda: None if manifest_json_data.get(json_key, '') == project_name else manifest_json_data
|
||||
search_func = lambda manifest_json_data: manifest_json_data if manifest_json_data.get(json_key, '') == project_name else None
|
||||
elif isinstance(gem_name, str) or isinstance(gem_name, pathlib.PurePath):
|
||||
o3de_object_uris = repo_json_data['gems']
|
||||
o3de_object_uris = manifest_json_data['gems']
|
||||
manifest_json = 'gem.json'
|
||||
json_key = 'gem_name'
|
||||
search_func = lambda: None if manifest_json_data.get(json_key, '') == gem_name else manifest_json_data
|
||||
search_func = lambda manifest_json_data: manifest_json_data if manifest_json_data.get(json_key, '') == gem_name else None
|
||||
elif isinstance(template_name, str) or isinstance(template_name, pathlib.PurePath):
|
||||
o3de_object_uris = repo_json_data['template']
|
||||
o3de_object_uris = manifest_json_data['template']
|
||||
manifest_json = 'template.json'
|
||||
json_key = 'template_name'
|
||||
search_func = lambda: None if manifest_json_data.get(json_key, '') == template_name_name else manifest_json_data
|
||||
search_func = lambda manifest_json_data: manifest_json_data if manifest_json_data.get(json_key, '') == template_name_name else None
|
||||
elif isinstance(restricted_name, str) or isinstance(restricted_name, pathlib.PurePath):
|
||||
o3de_object_uris = repo_json_data['restricted']
|
||||
o3de_object_uris = manifest_json_data['restricted']
|
||||
manifest_json = 'restricted.json'
|
||||
json_key = 'restricted_name'
|
||||
search_func = lambda: None if manifest_json_data.get(json_key, '') == restricted_name else manifest_json_data
|
||||
search_func = lambda manifest_json_data: manifest_json_data if manifest_json_data.get(json_key, '') == restricted_name else None
|
||||
else:
|
||||
return None
|
||||
|
||||
o3de_object = search_o3de_object(manifest_json, o3de_object_uris, search_func)
|
||||
o3de_object = search_o3de_object(manifest_json, o3de_object_uris, search_func)
|
||||
if o3de_object:
|
||||
o3de_object['repo_name'] = manifest_json_data['repo_name']
|
||||
return o3de_object
|
||||
|
||||
# recurse into the repos object to search for the o3de object
|
||||
o3de_object_uris = repo_json_data['repos']
|
||||
o3de_object_uris = []
|
||||
try:
|
||||
o3de_object_uris = manifest_json_data['repos']
|
||||
except KeyError:
|
||||
pass
|
||||
|
||||
manifest_json = 'repo.json'
|
||||
search_func = lambda: search_repo(manifest_json, engine_name, project_name, gem_name, template_name)
|
||||
search_func = lambda manifest_json_data: search_repo(manifest_json_data, engine_name, project_name, gem_name, template_name)
|
||||
return search_o3de_object(manifest_json, o3de_object_uris, search_func)
|
||||
|
||||
|
||||
@@ -141,8 +183,8 @@ def search_o3de_object(manifest_json, o3de_object_uris, search_func):
|
||||
# Search for the o3de object based on the supplied object name in the current repo
|
||||
cache_folder = manifest.get_o3de_cache_folder()
|
||||
for o3de_object_uri in o3de_object_uris:
|
||||
manifest_json_uri = f'{o3de_object_uri}/{manifest_json}'
|
||||
manifest_json_sha256 = hashlib.sha256(manifest_json_uri.encode())
|
||||
parsed_uri = urllib.parse.urlparse(f'{o3de_object_uri}/{manifest_json}')
|
||||
manifest_json_sha256 = hashlib.sha256(parsed_uri.geturl().encode())
|
||||
cache_file = cache_folder / str(manifest_json_sha256.hexdigest() + '.json')
|
||||
if cache_file.is_file():
|
||||
with cache_file.open('r') as f:
|
||||
@@ -151,7 +193,7 @@ def search_o3de_object(manifest_json, o3de_object_uris, search_func):
|
||||
except json.JSONDecodeError as e:
|
||||
logger.warn(f'{cache_file} failed to load: {str(e)}')
|
||||
else:
|
||||
result_json_data = search_func()
|
||||
result_json_data = search_func(manifest_json_data)
|
||||
if result_json_data:
|
||||
return result_json_data
|
||||
return None
|
||||
|
||||
@@ -13,6 +13,11 @@ import uuid
|
||||
import pathlib
|
||||
import shutil
|
||||
import urllib.request
|
||||
import logging
|
||||
import zipfile
|
||||
|
||||
logger = logging.getLogger()
|
||||
logging.basicConfig()
|
||||
|
||||
def validate_identifier(identifier: str) -> bool:
|
||||
"""
|
||||
@@ -97,11 +102,11 @@ def download_file(parsed_uri, download_path: pathlib.Path) -> int:
|
||||
if download_path.is_file():
|
||||
logger.warn(f'File already downloaded to {download_path}.')
|
||||
elif parsed_uri.scheme in ['http', 'https', 'ftp', 'ftps']:
|
||||
with urllib.request.urlopen(url) as s:
|
||||
with urllib.request.urlopen(parsed_uri.geturl()) as s:
|
||||
with download_path.open('wb') as f:
|
||||
shutil.copyfileobj(s, f)
|
||||
else:
|
||||
origin_file = pathlib.Path(url).resolve()
|
||||
origin_file = pathlib.Path(parsed_uri.geturl()).resolve()
|
||||
if not origin_file.is_file():
|
||||
return 1
|
||||
shutil.copy(origin_file, download_path)
|
||||
|
||||
@@ -27,7 +27,6 @@ def valid_o3de_repo_json(file_name: str or pathlib.Path) -> bool:
|
||||
test = json_data['origin']
|
||||
except (json.JSONDecodeError, KeyError) as e:
|
||||
return False
|
||||
|
||||
return True
|
||||
|
||||
|
||||
|
||||
@@ -31,17 +31,17 @@ TEST_TEMPLATED_CONTENT_WITHOUT_LICENSE = """
|
||||
#include <AzCore/EBus/EBus.h>
|
||||
#include <AzCore/Interface/Interface.h>
|
||||
|
||||
namespace ${Name}
|
||||
namespace ${SanitizedCppName}
|
||||
{
|
||||
class ${Name}Requests
|
||||
class ${SanitizedCppName}Requests
|
||||
{
|
||||
public:
|
||||
AZ_RTTI(${Name}Requests, "{${Random_Uuid}}");
|
||||
virtual ~${Name}Requests() = default;
|
||||
AZ_RTTI(${SanitizedCppName}Requests, "{${Random_Uuid}}");
|
||||
virtual ~${SanitizedCppName}Requests() = default;
|
||||
// Put your public methods here
|
||||
};
|
||||
|
||||
class ${Name}BusTraits
|
||||
class ${SanitizedCppName}BusTraits
|
||||
: public AZ::EBusTraits
|
||||
{
|
||||
public:
|
||||
@@ -52,31 +52,31 @@ namespace ${Name}
|
||||
//////////////////////////////////////////////////////////////////////////
|
||||
};
|
||||
|
||||
using ${Name}RequestBus = AZ::EBus<${Name}Requests, ${Name}BusTraits>;
|
||||
using ${Name}Interface = AZ::Interface<${Name}Requests>;
|
||||
using ${SanitizedCppName}RequestBus = AZ::EBus<${SanitizedCppName}Requests, ${SanitizedCppName}BusTraits>;
|
||||
using ${SanitizedCppName}Interface = AZ::Interface<${SanitizedCppName}Requests>;
|
||||
|
||||
} // namespace ${Name}
|
||||
} // namespace ${SanitizedCppName}
|
||||
"""
|
||||
|
||||
TEST_TEMPLATED_CONTENT_WITH_LICENSE = CPP_LICENSE_TEXT + TEST_TEMPLATED_CONTENT_WITHOUT_LICENSE
|
||||
|
||||
TEST_CONCRETE_TESTTEMPLATE_CONTENT_WITHOUT_LICENSE = string.Template(
|
||||
TEST_TEMPLATED_CONTENT_WITHOUT_LICENSE).safe_substitute({'Name': "TestTemplate"})
|
||||
TEST_TEMPLATED_CONTENT_WITHOUT_LICENSE).safe_substitute({'SanitizedCppName': "TestTemplate"})
|
||||
|
||||
TEST_CONCRETE_TESTTEMPLATE_CONTENT_WITH_LICENSE = string.Template(
|
||||
TEST_TEMPLATED_CONTENT_WITH_LICENSE).safe_substitute({'Name': "TestTemplate"})
|
||||
TEST_TEMPLATED_CONTENT_WITH_LICENSE).safe_substitute({'SanitizedCppName': "TestTemplate"})
|
||||
|
||||
TEST_CONCRETE_TESTPROJECT_TEMPLATE_CONTENT_WITHOUT_LICENSE = string.Template(
|
||||
TEST_TEMPLATED_CONTENT_WITHOUT_LICENSE).safe_substitute({'Name': "TestProject"})
|
||||
TEST_TEMPLATED_CONTENT_WITHOUT_LICENSE).safe_substitute({'SanitizedCppName': "TestProject"})
|
||||
|
||||
TEST_CONCRETE_TESTPROJECT_TEMPLATE_CONTENT_WITH_LICENSE = string.Template(
|
||||
TEST_TEMPLATED_CONTENT_WITH_LICENSE).safe_substitute({'Name': "TestProject"})
|
||||
TEST_TEMPLATED_CONTENT_WITH_LICENSE).safe_substitute({'SanitizedCppName': "TestProject"})
|
||||
|
||||
TEST_CONCRETE_TESTGEM_TEMPLATE_CONTENT_WITHOUT_LICENSE = string.Template(
|
||||
TEST_TEMPLATED_CONTENT_WITHOUT_LICENSE).safe_substitute({'Name': "TestGem"})
|
||||
TEST_TEMPLATED_CONTENT_WITHOUT_LICENSE).safe_substitute({'SanitizedCppName': "TestGem"})
|
||||
|
||||
TEST_CONCRETE_TESTGEM_TEMPLATE_CONTENT_WITH_LICENSE = string.Template(
|
||||
TEST_TEMPLATED_CONTENT_WITH_LICENSE).safe_substitute({'Name': "TestGem"})
|
||||
TEST_TEMPLATED_CONTENT_WITH_LICENSE).safe_substitute({'SanitizedCppName': "TestGem"})
|
||||
|
||||
TEST_TEMPLATE_JSON_CONTENTS = """\
|
||||
{
|
||||
|
||||
@@ -0,0 +1,99 @@
|
||||
#
|
||||
# Copyright (c) Contributors to the Open 3D Engine Project.
|
||||
# For complete copyright and license terms please see the LICENSE at the root of this distribution.
|
||||
#
|
||||
# SPDX-License-Identifier: Apache-2.0 OR MIT
|
||||
#
|
||||
#
|
||||
|
||||
param (
|
||||
[String[]] $exePath,
|
||||
[String[]] $packagePath,
|
||||
[String[]] $bootstrapPath,
|
||||
[String[]] $certificate
|
||||
)
|
||||
|
||||
# Get prerequisites, certs, and paths ready
|
||||
$tempPath = [System.IO.Path]::GetTempPath() # Order of operations defined here: https://docs.microsoft.com/en-us/dotnet/api/system.io.path.gettemppath?view=net-5.0&tabs=windows#remarks
|
||||
$certThumbprint = Get-ChildItem -Path Cert:LocalMachine\MY -CodeSigningCert -ErrorAction Stop | Select-Object -ExpandProperty Thumbprint # Grab first certificate from local machine store
|
||||
|
||||
if ($certificate) {
|
||||
Write-Output "Checking certificate thumbprint $certificate"
|
||||
Get-ChildItem -Path Cert:LocalMachine\MY -ErrorAction SilentlyContinue | Where-Object {$_.Thumbprint -eq $certificate} # Prints certificate Thumbprint and Subject if found
|
||||
if($?) {
|
||||
$certThumbprint = $certificate
|
||||
}
|
||||
else {
|
||||
Write-Error "$certificate thumbprint not found, using $certThumbprint thumbprint instead"
|
||||
}
|
||||
}
|
||||
|
||||
Try {
|
||||
$signtoolPath = Resolve-Path "C:\Program Files*\Windows Kits\10\bin\*\x64\signtool.exe" -ErrorAction Stop | Select-Object -Last 1 -ExpandProperty Path
|
||||
$insigniaPath = Resolve-Path "C:\Program Files*\WiX*\bin\insignia.exe" -ErrorAction Stop | Select-Object -Last 1 -ExpandProperty Path
|
||||
}
|
||||
Catch {
|
||||
Write-Error "Signtool or Wix insignia not found! Exiting."
|
||||
}
|
||||
|
||||
function Write-Signature {
|
||||
param (
|
||||
$signtool,
|
||||
$thumbprint,
|
||||
$filename
|
||||
)
|
||||
|
||||
$attempts = 2
|
||||
$sleepSec = 5
|
||||
|
||||
Do {
|
||||
$attempts--
|
||||
Try {
|
||||
& $signtool sign /tr http://timestamp.digicert.com /td sha256 /fd sha256 /sha1 $thumbprint /sm $filename
|
||||
& $signtool verify /pa /v $filename
|
||||
return
|
||||
}
|
||||
Catch {
|
||||
Write-Error $_.Exception.InnerException.Message -ErrorAction Continue
|
||||
Start-Sleep -Seconds $sleepSec
|
||||
}
|
||||
} while ($attempts -lt 0)
|
||||
|
||||
throw "Failed to sign $filename" # Bypassed in try block if the command is successful
|
||||
}
|
||||
|
||||
# Looping through each path insteaad of globbing to prevent hitting maximum command string length limit
|
||||
if ($exePath) {
|
||||
Write-Output "### Signing EXE files ###"
|
||||
$files = @(Get-ChildItem $exePath -Recurse *.exe | % { $_.FullName })
|
||||
foreach ($file in $files) {
|
||||
Write-Signature -signtool $signtoolPath -thumbprint $certThumbprint -filename $file
|
||||
}
|
||||
}
|
||||
|
||||
if ($packagePath) {
|
||||
Write-Output "### Signing CAB files ###"
|
||||
$files = @(Get-ChildItem $packagePath -Recurse *.cab | % { $_.FullName })
|
||||
foreach ($file in $files) {
|
||||
Write-Signature -signtool $signtoolPath -thumbprint $certThumbprint -filename $file
|
||||
}
|
||||
|
||||
Write-Output "### Signing MSI files ###"
|
||||
$files = @(Get-ChildItem $packagePath -Recurse *.msi | % { $_.FullName })
|
||||
foreach ($file in $files) {
|
||||
& $insigniaPath -im $files
|
||||
Write-Signature -signtool $signtoolPath -thumbprint $certThumbprint -filename $file
|
||||
}
|
||||
}
|
||||
|
||||
if ($bootstrapPath) {
|
||||
Write-Output "### Signing bootstrapper EXE ###"
|
||||
$files = @(Get-ChildItem $bootstrapPath -Recurse *.exe | % { $_.FullName })
|
||||
foreach ($file in $files) {
|
||||
& $insigniaPath -ib $file -o $tempPath\engine.exe
|
||||
Write-Signature -signtool $signtoolPath -thumbprint $certThumbprint -filename $tempPath\engine.exe
|
||||
& $insigniaPath -ab $tempPath\engine.exe $file -o $file
|
||||
Write-Signature -signtool $signtoolPath -thumbprint $certThumbprint -filename $file
|
||||
Remove-Item -Force $tempPath\engine.exe
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user