birkeh/o3de
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Merging from development

Browse Source 
Signed-off-by: Esteban Papp <81431996+amznestebanpapp@users.noreply.github.com>
This commit is contained in:
Esteban Papp
2021-12-07 12:35:41 -08:00
parent b1eeebb6b6
commit cd5306febf
334 changed files with 9946 additions and 3757 deletions
Download Patch File Download Diff File Expand all files Collapse all files
scripts/o3de.py
+18 -14
View File
@@ -7,20 +7,25 @@
#
import argparse
import logging
import pathlib
import sys
logger = logging.getLogger('o3de')
def add_args(parser, subparsers) -> None:
def add_args(parser: argparse.ArgumentParser) -> None:
"""
add_args is called to add expected parser arguments and subparsers arguments to each command such that it can be
invoked by o3de.py
Ex o3de.py can invoke the register downloadable commands by importing register,
call add_args and execute: python o3de.py register --gem-path "C:/TestGem"
:param parser: the caller instantiates a parser and passes it in here
:param subparsers: the caller instantiates subparsers and passes it in here
:param parser: the caller instantiates an ArgumentParser and passes it in here
"""
subparsers = parser.add_subparsers(help='To get help on a sub-command:\no3de.py <sub-command> -h',
title='Sub-Commands')
# As o3de.py shares the same name as the o3de package attempting to use a regular
# from o3de import <module> line tries to import from the current o3de.py script and not the package
# So the {current script directory} / 'o3de' is added to the front of the sys.path
@@ -28,24 +33,25 @@ def add_args(parser, subparsers) -> None:
o3de_package_dir = (script_dir / 'o3de').resolve()
# add the scripts/o3de directory to the front of the sys.path
sys.path.insert(0, str(o3de_package_dir))
from o3de import engine_properties, engine_template, gem_properties, global_project, register, print_registration, get_registration, \
from o3de import engine_properties, engine_template, gem_properties, \
global_project, register, print_registration, get_registration, \
enable_gem, disable_gem, project_properties, sha256, download
# Remove the temporarily added path
sys.path = sys.path[1:]
# global_project
# global project
global_project.add_args(subparsers)
# engine templaate
# engine template
engine_template.add_args(subparsers)
# register
# registration
register.add_args(subparsers)
# show
# show registration
print_registration.add_args(subparsers)
# get-registered
# get registration
get_registration.add_args(subparsers)
# add a gem to a project
@@ -74,11 +80,8 @@ if __name__ == "__main__":
# parse the command line args
the_parser = argparse.ArgumentParser()
# add subparsers
the_subparsers = the_parser.add_subparsers(help='sub-command help')
# add args to the parser
add_args(the_parser, the_subparsers)
add_args(the_parser)
# parse args
the_args = the_parser.parse_args()
@@ -89,7 +92,8 @@ if __name__ == "__main__":
sys.exit(1)
# run
ret = the_args.func(the_args)
ret = the_args.func(the_args) if hasattr(the_args, 'func') else 1
logger.info('Success!' if ret == 0 else 'Completed with issues: result {}'.format(ret))
# return
sys.exit(ret)
scripts/o3de/o3de/cmake.py
+3 -3
View File
@@ -13,10 +13,10 @@ import logging
import os
import pathlib
from o3de import manifest
from o3de import manifest, utils
logger = logging.getLogger()
logging.basicConfig()
logger = logging.getLogger('o3de.cmake')
logging.basicConfig(format=utils.LOG_FORMAT)
enable_gem_start_marker = 'set(ENABLED_GEMS'
enable_gem_end_marker = ')'
scripts/o3de/o3de/disable_gem.py
+8 -6
View File
@@ -15,10 +15,10 @@ import os
import pathlib
import sys
from o3de import cmake, manifest
from o3de import cmake, manifest, utils
logger = logging.getLogger()
logging.basicConfig()
logger = logging.getLogger('o3de.disable_gem')
logging.basicConfig(format=utils.LOG_FORMAT)
def disable_gem_in_project(gem_name: str = None,
@@ -73,7 +73,6 @@ def disable_gem_in_project(gem_name: str = None,
logger.error(f'Gem Path {gem_path} does not exist.')
return 1
# Read gem.json from the gem path
gem_json_data = manifest.get_gem_json_data(gem_path=gem_path, project_path=project_path)
if not gem_json_data:
@@ -116,6 +115,10 @@ def add_parser_args(parser):
Ex. Directly run from this file alone with: python disable_gem.py --project-path D:/Test --gem-name Atom
:param parser: the caller passes an argparse parser like instance to this method
"""
# Sub-commands should declare their own verbosity flag, if desired
utils.add_verbosity_arg(parser)
group = parser.add_mutually_exclusive_group(required=True)
group.add_argument('-pp', '--project-path', type=pathlib.Path, required=False,
help='The path to the project.')
@@ -155,8 +158,6 @@ def main():
# parse the command line args
the_parser = argparse.ArgumentParser()
# add subparsers
# add args to the parser
add_parser_args(the_parser)
@@ -165,6 +166,7 @@ def main():
# run
ret = the_args.func(the_args) if hasattr(the_args, 'func') else 1
logger.info('Success!' if ret == 0 else 'Completed with issues: result {}'.format(ret))
# return
sys.exit(ret)
scripts/o3de/o3de/download.py
+4 -2
View File
@@ -24,8 +24,9 @@ from datetime import datetime
from o3de import manifest, repo, utils, validation, register
logger = logging.getLogger()
logging.basicConfig()
logger = logging.getLogger('o3de.download')
logging.basicConfig(format=utils.LOG_FORMAT)
def unzip_manifest_json_data(download_zip_path: pathlib.Path, zip_file_name: str) -> dict:
json_data = {}
@@ -38,6 +39,7 @@ def unzip_manifest_json_data(download_zip_path: pathlib.Path, zip_file_name: str
return json_data
def validate_downloaded_zip_sha256(download_uri_json_data: dict, download_zip_path: pathlib.Path,
manifest_json_name) -> int:
# if the json has a sha256 check it against a sha256 of the zip
scripts/o3de/o3de/enable_gem.py
+8 -5
View File
@@ -16,10 +16,10 @@ import os
import pathlib
import sys
from o3de import cmake, manifest, register, validation
from o3de import cmake, manifest, register, validation, utils
logger = logging.getLogger()
logging.basicConfig()
logger = logging.getLogger('o3de.enable_gem')
logging.basicConfig(format=utils.LOG_FORMAT)
def enable_gem_in_project(gem_name: str = None,
@@ -132,6 +132,10 @@ def add_parser_args(parser):
Ex. Directly run from this file alone with: python enable_gem.py --project-path "D:/TestProject" --gem-path "D:/TestGem"
:param parser: the caller passes an argparse parser like instance to this method
"""
# Sub-commands should declare their own verbosity flag, if desired
utils.add_verbosity_arg(parser)
group = parser.add_mutually_exclusive_group(required=True)
group.add_argument('-pp', '--project-path', type=pathlib.Path, required=False,
help='The path to the project.')
@@ -171,8 +175,6 @@ def main():
# parse the command line args
the_parser = argparse.ArgumentParser()
# add subparsers
# add args to the parser
add_parser_args(the_parser)
@@ -181,6 +183,7 @@ def main():
# run
ret = the_args.func(the_args) if hasattr(the_args, 'func') else 1
logger.info('Success!' if ret == 0 else 'Completed with issues: result {}'.format(ret))
# return
sys.exit(ret)
scripts/o3de/o3de/engine_properties.py
+8 -2
View File
@@ -15,8 +15,9 @@ import logging
from o3de import manifest, utils
logger = logging.getLogger()
logging.basicConfig()
logger = logging.getLogger('o3de.engine_properties')
logging.basicConfig(format=utils.LOG_FORMAT)
def edit_engine_props(engine_path: pathlib.Path = None,
engine_name: str = None,
@@ -25,6 +26,11 @@ def edit_engine_props(engine_path: pathlib.Path = None,
if not engine_path and not engine_name:
logger.error(f'Either a engine path or a engine name must be supplied to lookup engine.json')
return 1
if not new_name and not new_version:
logger.error('A new engine name or new version, or both must be supplied.')
return 1
if not engine_path:
engine_path = manifest.get_registered(engine_name=engine_name)
scripts/o3de/o3de/engine_template.py
+38 -19
View File
@@ -20,8 +20,8 @@ import re
from o3de import manifest, register, validation, utils
logger = logging.getLogger()
logging.basicConfig()
logger = logging.getLogger('o3de.engine_template')
logging.basicConfig(format=utils.LOG_FORMAT)
binary_file_ext = {
'.pak',
@@ -86,6 +86,7 @@ restricted_platforms = {
template_file_name = 'template.json'
this_script_parent = pathlib.Path(os.path.dirname(os.path.realpath(__file__)))
def _replace_license_text(source_data: str):
while '{BEGIN_LICENSE}' in source_data:
start = source_data.find('{BEGIN_LICENSE}')
@@ -181,7 +182,7 @@ def _execute_template_json(json_data: dict,
# regular copy if not templated
for copy_file in json_data['copyFiles']:
# construct the input file name
in_file = template_path / 'Template' /copy_file['file']
in_file = template_path / 'Template' / copy_file['file']
# the file can be marked as optional, if it is and it does not exist skip
if copy_file['isOptional'] and copy_file['isOptional'] == 'true':
@@ -246,7 +247,7 @@ def _execute_restricted_template_json(json_data: dict,
for copy_file in json_data['copyFiles']:
# construct the input file name
in_file = template_restricted_path / restricted_platform / template_restricted_platform_relative_path\
/ template_name / 'Template'/ copy_file['file']
/ template_name / 'Template' / copy_file['file']
# the file can be marked as optional, if it is and it does not exist skip
if copy_file['isOptional'] and copy_file['isOptional'] == 'true':
@@ -364,7 +365,7 @@ def create_template(source_path: pathlib.Path,
keep_restricted_in_template: bool = False,
keep_license_text: bool = False,
replace: list = None,
force: bool = False) -> int:
force: bool = False) -> int:
"""
Create a template from a source directory using replacement
@@ -1205,12 +1206,12 @@ def create_from_template(destination_path: pathlib.Path,
# something is wrong with either the --template-restricted-platform-relative or the template is.
if template_restricted_platform_relative_path != template_json_restricted_platform_relative_path:
logger.error(f'The supplied --template-restricted-platform-relative-path'
f' "{template_restricted_platform_relative_path}" does not match the'
f' templates.json "restricted_platform_relative_path". Either'
f' --template-restricted-platform-relative-path is incorrect or the templates'
f' "restricted_platform_relative_path" is wrong. Note that since this template'
f' specifies "restricted_platform_relative_path" it need not be supplied and'
f' "{template_json_restricted_platform_relative_path}" will be used.')
f' "{template_restricted_platform_relative_path}" does not match the'
f' templates.json "restricted_platform_relative_path". Either'
f' --template-restricted-platform-relative-path is incorrect or the templates'
f' "restricted_platform_relative_path" is wrong. Note that since this template'
f' specifies "restricted_platform_relative_path" it need not be supplied and'
f' "{template_json_restricted_platform_relative_path}" will be used.')
return 1
else:
# The user has not supplied --template-restricted-platform-relative-path, try to read it from
@@ -1306,7 +1307,7 @@ def create_from_template(destination_path: pathlib.Path,
os.makedirs(destination_restricted_path, exist_ok=True)
# read the restricted_name from the destination restricted.json
restricted_json = destination_restricted_path / restricted.json
restricted_json = destination_restricted_path / 'restricted.json'
if not os.path.isfile(restricted_json):
with open(restricted_json, 'w') as s:
restricted_json_data = {}
@@ -1956,7 +1957,6 @@ def create_gem(gem_path: pathlib.Path,
replacements.append(("${NameLower}", gem_name.lower()))
replacements.append(("${SanitizedCppName}", sanitized_cpp_name))
# module id is a uuid with { and -
if module_id:
replacements.append(("${ModuleClassId}", module_id))
@@ -2157,8 +2157,13 @@ def add_args(subparsers) -> None:
call add_args and execute: python o3de.py create-gem --gem-path TestGem
:param subparsers: the caller instantiates subparsers and passes it in here
"""
# turn a directory into a template
create_template_subparser = subparsers.add_parser('create-template')
# Sub-commands should declare their own verbosity flag, if desired
utils.add_verbosity_arg(create_template_subparser)
create_template_subparser.add_argument('-sp', '--source-path', type=pathlib.Path, required=True,
help='The path to the source that you want to make into a template')
create_template_subparser.add_argument('-tp', '--template-path', type=pathlib.Path, required=False,
@@ -2233,6 +2238,10 @@ def add_args(subparsers) -> None:
# create from template
create_from_template_subparser = subparsers.add_parser('create-from-template')
# Sub-commands should declare their own verbosity flag, if desired
utils.add_verbosity_arg(create_from_template_subparser)
create_from_template_subparser.add_argument('-dp', '--destination-path', type=pathlib.Path, required=True,
help='The path to where you want the template instantiated,'
' can be absolute or relative to the current working directory.'
@@ -2316,6 +2325,10 @@ def add_args(subparsers) -> None:
# creation of a project from a template (like create from template but makes project assumptions)
create_project_subparser = subparsers.add_parser('create-project')
# Sub-commands should declare their own verbosity flag, if desired
utils.add_verbosity_arg(create_project_subparser)
create_project_subparser.add_argument('-pp', '--project-path', type=pathlib.Path, required=True,
help='The location of the project you wish to create from the template,'
' can be an absolute path or relative to the current working directory.'
@@ -2330,7 +2343,7 @@ def add_args(subparsers) -> None:
group = create_project_subparser.add_mutually_exclusive_group(required=False)
group.add_argument('-tp', '--template-path', type=pathlib.Path, required=False,
default=None,
help='the path to the template you want to instance, can be absolute or'
help='The path to the template you want to instance, can be absolute or'
' relative to default templates path')
group.add_argument('-tn', '--template-name', type=str, required=False,
default=None,
@@ -2340,7 +2353,7 @@ def add_args(subparsers) -> None:
group = create_project_subparser.add_mutually_exclusive_group(required=False)
group.add_argument('-prp', '--project-restricted-path', type=pathlib.Path, required=False,
default=None,
help='path to the projects restricted folder, can be absolute or relative to'
help='The path to the projects restricted folder, can be absolute or relative to'
' the default restricted projects directory')
group.add_argument('-prn', '--project-restricted-name', type=str, required=False,
default=None,
@@ -2351,7 +2364,7 @@ def add_args(subparsers) -> None:
group.add_argument('-trp', '--template-restricted-path', type=pathlib.Path, required=False,
default=None,
help='The templates restricted path can be absolute or relative to'
'the default restricted templates directory')
' the default restricted templates directory')
group.add_argument('-trn', '--template-restricted-name', type=str, required=False,
default=None,
help='The name of the registered templates restricted path. If supplied this will resolve'
@@ -2413,6 +2426,10 @@ def add_args(subparsers) -> None:
# creation of a gem from a template (like create from template but makes gem assumptions)
create_gem_subparser = subparsers.add_parser('create-gem')
# Sub-commands should declare their own verbosity flag, if desired
utils.add_verbosity_arg(create_gem_subparser)
create_gem_subparser.add_argument('-gp', '--gem-path', type=pathlib.Path, required=True,
help='The gem path, can be absolute or relative to the current working directory')
create_gem_subparser.add_argument('-gn', '--gem-name', type=str,
@@ -2512,16 +2529,18 @@ if __name__ == "__main__":
the_parser = argparse.ArgumentParser()
# add subparsers
the_subparsers = the_parser.add_subparsers(help='sub-command help', dest='command', required=True)
subparsers = the_parser.add_subparsers(help='To get help on a sub-command:\nengine_template.py <sub-command> -h',
title='Sub-Commands', dest='command', required=True)
# add args to the parser
add_args(the_subparsers)
# add args to the parsers
add_args(subparsers)
# parse args
the_args = the_parser.parse_args()
# run
ret = the_args.func(the_args) if hasattr(the_args, 'func') else 1
logger.info('Success!' if ret == 0 else 'Completed with issues: result {}'.format(ret))
# return
sys.exit(ret)
scripts/o3de/o3de/gem_properties.py
+2 -2
View File
@@ -15,8 +15,8 @@ import logging
from o3de import manifest, utils
logger = logging.getLogger()
logging.basicConfig()
logger = logging.getLogger('o3de.gem_properties')
logging.basicConfig(format=utils.LOG_FORMAT)
def update_values_in_key_list(existing_values: list, new_values: list or str, remove_values: list or str,
scripts/o3de/o3de/get_registration.py
+7 -6
View File
@@ -12,17 +12,18 @@ import sys
from o3de import manifest
def _run_get_registered(args: argparse) -> str or pathlib.Path:
if args.override_home_folder:
manifest.override_home_folder = args.override_home_folder
return manifest.get_registered(args.engine_name,
args.project_name,
args.gem_name,
args.template_name,
args.default_folder,
args.repo_name,
args.restricted_name)
args.project_name,
args.gem_name,
args.template_name,
args.default_folder,
args.repo_name,
args.restricted_name)
def add_parser_args(parser):
scripts/o3de/o3de/global_project.py
+4 -3
View File
@@ -14,14 +14,15 @@ import re
import pathlib
import json
from o3de import manifest, validation
from o3de import manifest, validation, utils
logger = logging.getLogger()
logging.basicConfig()
logger = logging.getLogger('o3de.global_project')
logging.basicConfig(format=utils.LOG_FORMAT)
DEFAULT_BOOTSTRAP_SETREG = pathlib.Path('~/.o3de/Registry/bootstrap.setreg').expanduser()
PROJECT_PATH_KEY = ('Amazon', 'AzCore', 'Bootstrap', 'project_path')
def get_json_data(input_path: pathlib.Path):
setreg_json_data = {}
# If the output_path exist validate that it is a valid json file
scripts/o3de/o3de/manifest.py
+78 -55
View File
@@ -18,8 +18,8 @@ import hashlib
from o3de import validation, utils
logger = logging.getLogger()
logging.basicConfig()
logger = logging.getLogger('o3de.manifest')
logging.basicConfig(format=utils.LOG_FORMAT)
# Directory methods
override_home_folder = None
@@ -41,6 +41,7 @@ def get_o3de_folder() -> pathlib.Path:
o3de_folder.mkdir(parents=True, exist_ok=True)
return o3de_folder
def get_o3de_user_folder() -> pathlib.Path:
o3de_user_folder = get_home_folder() / 'O3DE'
o3de_user_folder.mkdir(parents=True, exist_ok=True)
@@ -177,6 +178,7 @@ def get_default_o3de_manifest_json_data() -> dict:
return json_data
def get_o3de_manifest() -> pathlib.Path:
manifest_path = get_o3de_folder() / 'o3de_manifest.json'
if not manifest_path.is_file():
@@ -228,11 +230,11 @@ def save_o3de_manifest(json_data: dict, manifest_path: pathlib.Path = None) -> b
def get_gems_from_subdirectories(external_subdirs: list) -> list:
'''
"""
Helper Method for scanning a set of external subdirectories for gem.json files
'''
"""
def is_gem_subdirectory(subdir_files):
for name in files:
for name in subdir_files:
if name == 'gem.json':
return True
return False
@@ -285,13 +287,14 @@ def get_repos() -> list:
json_data = load_o3de_manifest()
return json_data['repos'] if 'repos' in json_data else []
# engine.json queries
def get_engine_projects() -> list:
engine_path = get_this_engine_path()
engine_object = get_engine_json_data(engine_path=engine_path)
if engine_object:
return list(map(lambda rel_path: (pathlib.Path(engine_path) / rel_path).as_posix(),
engine_object['projects'])) if 'projects' in engine_object else []
engine_object['projects'])) if 'projects' in engine_object else []
return []
@@ -313,7 +316,7 @@ def get_engine_templates() -> list:
engine_object = get_engine_json_data(engine_path=engine_path)
if engine_object:
return list(map(lambda rel_path: (pathlib.Path(engine_path) / rel_path).as_posix(),
engine_object['templates'])) if 'templates' in engine_object else []
engine_object['templates'])) if 'templates' in engine_object else []
return []
@@ -335,7 +338,7 @@ def get_project_external_subdirectories(project_path: pathlib.Path) -> list:
project_object = get_project_json_data(project_path=project_path)
if project_object:
return list(map(lambda rel_path: (pathlib.Path(project_path) / rel_path).as_posix(),
project_object['external_subdirectories'])) if 'external_subdirectories' in project_object else []
project_object['external_subdirectories'])) if 'external_subdirectories' in project_object else []
return []
@@ -343,7 +346,7 @@ def get_project_templates(project_path: pathlib.Path) -> list:
project_object = get_project_json_data(project_path=project_path)
if project_object:
return list(map(lambda rel_path: (pathlib.Path(project_path) / rel_path).as_posix(),
project_object['templates'])) if 'templates' in project_object else []
project_object['templates'])) if 'templates' in project_object else []
return []
@@ -433,6 +436,7 @@ def get_templates_for_generic_creation(): # temporary until we have a better wa
return list(filter(filter_project_and_gem_templates_out, get_all_templates()))
def get_json_file_path(object_typename: str,
object_path: str or pathlib.Path) -> pathlib.Path:
if not object_typename or not object_path:
@@ -468,6 +472,7 @@ def get_json_data_file(object_json: pathlib.Path,
return None
def get_json_data(object_typename: str,
object_path: str or pathlib.Path,
object_validator: callable) -> dict or None:
@@ -538,6 +543,7 @@ def get_restricted_json_data(restricted_name: str = None, restricted_path: str o
return get_json_data('restricted', restricted_path, validation.valid_o3de_restricted_json)
def get_repo_json_data(repo_uri: str) -> dict or None:
if not repo_uri:
logger.error('Must specify a Repo Uri.')
@@ -547,7 +553,8 @@ def get_repo_json_data(repo_uri: str) -> dict or None:
return get_json_data_file(repo_json, "Repo", validation.valid_o3de_repo_json)
def get_repo_path(repo_uri: str, cache_folder: str = None) -> pathlib.Path:
def get_repo_path(repo_uri: str, cache_folder: str or pathlib.Path = None) -> pathlib.Path:
if not cache_folder:
cache_folder = get_o3de_cache_folder()
@@ -555,6 +562,7 @@ def get_repo_path(repo_uri: str, cache_folder: str = None) -> pathlib.Path:
repo_sha256 = hashlib.sha256(repo_manifest.encode())
return cache_folder / str(repo_sha256.hexdigest() + '.json')
def get_registered(engine_name: str = None,
project_name: str = None,
gem_name: str = None,
@@ -599,75 +607,90 @@ def get_registered(engine_name: str = None,
engine_path = pathlib.Path(engine).resolve()
engine_json = engine_path / 'engine.json'
with engine_json.open('r') as f:
try:
engine_json_data = json.load(f)
except json.JSONDecodeError as e:
logger.warning(f'{engine_json} failed to load: {str(e)}')
else:
this_engines_name = engine_json_data['engine_name']
if this_engines_name == engine_name:
return engine_path
if not pathlib.Path(engine_json).is_file():
logger.warning(f'{engine_json} does not exist')
else:
with engine_json.open('r') as f:
try:
engine_json_data = json.load(f)
except json.JSONDecodeError as e:
logger.warning(f'{engine_json} failed to load: {str(e)}')
else:
this_engines_name = engine_json_data['engine_name']
if this_engines_name == engine_name:
return engine_path
elif isinstance(project_name, str):
projects = get_all_projects()
for project_path in projects:
project_path = pathlib.Path(project_path).resolve()
project_json = project_path / 'project.json'
with project_json.open('r') as f:
try:
project_json_data = json.load(f)
except json.JSONDecodeError as e:
logger.warning(f'{project_json} failed to load: {str(e)}')
else:
this_projects_name = project_json_data['project_name']
if this_projects_name == project_name:
return project_path
if not pathlib.Path(project_json).is_file():
logger.warning(f'{project_json} does not exist')
else:
with project_json.open('r') as f:
try:
project_json_data = json.load(f)
except json.JSONDecodeError as e:
logger.warning(f'{project_json} failed to load: {str(e)}')
else:
this_projects_name = project_json_data['project_name']
if this_projects_name == project_name:
return project_path
elif isinstance(gem_name, str):
gems = get_all_gems(project_path)
for gem_path in gems:
gem_path = pathlib.Path(gem_path).resolve()
gem_json = gem_path / 'gem.json'
with gem_json.open('r') as f:
try:
gem_json_data = json.load(f)
except json.JSONDecodeError as e:
logger.warning(f'{gem_json} failed to load: {str(e)}')
else:
this_gems_name = gem_json_data['gem_name']
if this_gems_name == gem_name:
return gem_path
if not pathlib.Path(gem_json).is_file():
logger.warning(f'{gem_json} does not exist')
else:
with gem_json.open('r') as f:
try:
gem_json_data = json.load(f)
except json.JSONDecodeError as e:
logger.warning(f'{gem_json} failed to load: {str(e)}')
else:
this_gems_name = gem_json_data['gem_name']
if this_gems_name == gem_name:
return gem_path
elif isinstance(template_name, str):
templates = get_all_templates(project_path)
for template_path in templates:
template_path = pathlib.Path(template_path).resolve()
template_json = template_path / 'template.json'
with template_json.open('r') as f:
try:
template_json_data = json.load(f)
except json.JSONDecodeError as e:
logger.warning(f'{template_path} failed to load: {str(e)}')
else:
this_templates_name = template_json_data['template_name']
if this_templates_name == template_name:
return template_path
if not pathlib.Path(template_json).is_file():
logger.warning(f'{template_json} does not exist')
else:
with template_json.open('r') as f:
try:
template_json_data = json.load(f)
except json.JSONDecodeError as e:
logger.warning(f'{template_path} failed to load: {str(e)}')
else:
this_templates_name = template_json_data['template_name']
if this_templates_name == template_name:
return template_path
elif isinstance(restricted_name, str):
restricted = get_all_restricted(project_path)
for restricted_path in restricted:
restricted_path = pathlib.Path(restricted_path).resolve()
restricted_json = restricted_path / 'restricted.json'
with restricted_json.open('r') as f:
try:
restricted_json_data = json.load(f)
except json.JSONDecodeError as e:
logger.warning(f'{restricted_json} failed to load: {str(e)}')
else:
this_restricted_name = restricted_json_data['restricted_name']
if this_restricted_name == restricted_name:
return restricted_path
if not pathlib.Path(restricted_json).is_file():
logger.warning(f'{restricted_json} does not exist')
else:
with restricted_json.open('r') as f:
try:
restricted_json_data = json.load(f)
except json.JSONDecodeError as e:
logger.warning(f'{restricted_json} failed to load: {str(e)}')
else:
this_restricted_name = restricted_json_data['restricted_name']
if this_restricted_name == restricted_name:
return restricted_path
elif isinstance(default_folder, str):
if default_folder == 'engines':
scripts/o3de/o3de/print_registration.py
+3 -3
View File
@@ -14,10 +14,10 @@ import pathlib
import sys
import urllib.parse
from o3de import manifest, validation
from o3de import manifest, validation, utils
logger = logging.getLogger()
logging.basicConfig()
logger = logging.getLogger('o3de.print_registration')
logging.basicConfig(format=utils.LOG_FORMAT)
def get_project_path(project_path: pathlib.Path, project_name: str) -> pathlib.Path:
scripts/o3de/o3de/project_properties.py
+10 -4
View File
@@ -15,8 +15,9 @@ import logging
from o3de import manifest, utils
logger = logging.getLogger()
logging.basicConfig()
logger = logging.getLogger('o3de.project_properties')
logging.basicConfig(format=utils.LOG_FORMAT)
def get_project_props(name: str = None, path: pathlib.Path = None) -> dict:
proj_json = manifest.get_project_json_data(project_name=name, project_path=path)
@@ -26,6 +27,7 @@ def get_project_props(name: str = None, path: pathlib.Path = None) -> dict:
return None
return proj_json
def edit_project_props(proj_path: pathlib.Path = None,
proj_name: str = None,
new_name: str = None,
@@ -71,9 +73,9 @@ def edit_project_props(proj_path: pathlib.Path = None,
tag_list = replace_tags.split() if isinstance(replace_tags, str) else replace_tags
proj_json['user_tags'] = tag_list
return 0 if manifest.save_o3de_manifest(proj_json, pathlib.Path(proj_path) / 'project.json') else 1
def _edit_project_props(args: argparse) -> int:
return edit_project_props(args.project_path,
args.project_name,
@@ -86,6 +88,7 @@ def _edit_project_props(args: argparse) -> int:
args.delete_tags,
args.replace_tags)
def add_parser_args(parser):
group = parser.add_mutually_exclusive_group(required=True)
group.add_argument('-pp', '--project-path', type=pathlib.Path, required=False,
@@ -112,10 +115,12 @@ def add_parser_args(parser):
help='Replace entirety of user_tags property with space delimited list of values')
parser.set_defaults(func=_edit_project_props)
def add_args(subparsers) -> None:
enable_project_props_subparser = subparsers.add_parser('edit-project-properties')
add_parser_args(enable_project_props_subparser)
def main():
the_parser = argparse.ArgumentParser()
add_parser_args(the_parser)
@@ -123,5 +128,6 @@ def main():
ret = the_args.func(the_args) if hasattr(the_args, 'func') else 1
sys.exit(ret)
if __name__ == "__main__":
main()
scripts/o3de/o3de/register.py
+19 -16
View File
@@ -23,8 +23,8 @@ import urllib.request
from o3de import get_registration, manifest, repo, utils, validation
logger = logging.getLogger()
logging.basicConfig()
logger = logging.getLogger('o3de.register')
logging.basicConfig(format=utils.LOG_FORMAT)
def register_shipped_engine_o3de_objects(force: bool = False) -> int:
@@ -306,9 +306,9 @@ def register_o3de_object_path(json_data: dict,
try:
paths_to_remove.append(o3de_object_path.relative_to(save_path.parent))
except ValueError:
pass # It is not an error if a relative path cannot be formed
pass # It is not an error if a relative path cannot be formed
manifest_data[o3de_object_key] = list(filter(lambda p: pathlib.Path(p) not in paths_to_remove,
manifest_data.setdefault(o3de_object_key, [])))
manifest_data.setdefault(o3de_object_key, [])))
if remove:
if save_path:
@@ -411,7 +411,7 @@ def register_project_path(json_data: dict,
if not remove:
# registering a project has the additional step of setting the project.json 'engine' field
this_engine_json = manifest.get_engine_json_data(engine_path=manifest.get_this_engine_path())
this_engine_json = manifest.get_engine_json_data(engine_path=engine_path if engine_path else manifest.get_this_engine_path())
if not this_engine_json:
return 1
project_json_data = manifest.get_project_json_data(project_path=project_path)
@@ -732,14 +732,14 @@ def register(engine_path: pathlib.Path = None,
logger.error(f'Gem path cannot be empty.')
return 1
result = result or register_gem_path(json_data, gem_path, remove,
external_subdir_engine_path, external_subdir_project_path)
external_subdir_engine_path, external_subdir_project_path)
if isinstance(external_subdir_path, pathlib.PurePath):
if not external_subdir_path:
logger.error(f'External Subdirectory path is None.')
return 1
result = result or register_external_subdirectory(json_data, external_subdir_path, remove,
external_subdir_engine_path, external_subdir_project_path)
external_subdir_engine_path, external_subdir_project_path)
if isinstance(template_path, pathlib.PurePath):
if not template_path:
@@ -841,6 +841,10 @@ def add_parser_args(parser):
Ex. Directly run from this file alone with: python register.py --engine-path "C:/o3de"
:param parser: the caller passes an argparse parser like instance to this method
"""
# Sub-commands should declare their own verbosity flag, if desired
utils.add_verbosity_arg(parser)
group = parser.add_mutually_exclusive_group(required=True)
group.add_argument('--this-engine', action='store_true', required=False,
default=False,
@@ -888,18 +892,18 @@ def add_parser_args(parser):
help='Refresh the repo cache.')
parser.add_argument('-ohf', '--override-home-folder', type=pathlib.Path, required=False,
help='By default the home folder is the user folder, override it to this folder.')
help='By default the home folder is the user folder, override it to this folder.')
parser.add_argument('-r', '--remove', action='store_true', required=False,
default=False,
help='Remove entry.')
default=False,
help='Remove entry.')
parser.add_argument('-f', '--force', action='store_true', default=False,
help='For the update of the registration field being modified.')
help='For the update of the registration field being modified.')
external_subdir_group = parser.add_argument_group(title='external-subdirectory',
description='path arguments to use with the --external-subdirectory option')
external_subdir_group = parser.add_argument_group(title='external-subdirectory',
description='path arguments to use with the --external-subdirectory option')
external_subdir_path_group = external_subdir_group.add_mutually_exclusive_group()
external_subdir_path_group.add_argument('-esep', '--external-subdirectory-engine-path', type=pathlib.Path,
help='If supplied, registers the external subdirectory with the engine.json at' \
help='If supplied, registers the external subdirectory with the engine.json at' \
' the engine-path location')
external_subdir_path_group.add_argument('-espp', '--external-subdirectory-project-path', type=pathlib.Path)
parser.set_defaults(func=_run_register)
@@ -924,8 +928,6 @@ def main():
# parse the command line args
the_parser = argparse.ArgumentParser()
# add subparsers
# add args to the parser
add_parser_args(the_parser)
@@ -934,6 +936,7 @@ def main():
# run
ret = the_args.func(the_args) if hasattr(the_args, 'func') else 1
logger.info('Success!' if ret == 0 else 'Completed with issues: result {}'.format(ret))
# return
sys.exit(ret)
scripts/o3de/o3de/repo.py
+2 -2
View File
@@ -15,8 +15,8 @@ import hashlib
from datetime import datetime
from o3de import manifest, utils, validation
logger = logging.getLogger()
logging.basicConfig()
logger = logging.getLogger('o3de.repo')
logging.basicConfig(format=utils.LOG_FORMAT)
def process_add_o3de_repo(file_name: str or pathlib.Path,
scripts/o3de/o3de/sha256.py
+7 -7
View File
@@ -15,8 +15,8 @@ import sys
from o3de import utils
logger = logging.getLogger()
logging.basicConfig()
logger = logging.getLogger('o3de.sha256')
logging.basicConfig(format=utils.LOG_FORMAT)
def sha256(file_path: str or pathlib.Path,
@@ -35,7 +35,7 @@ def sha256(file_path: str or pathlib.Path,
logger.error(f'Json path {json_path} does not exist.')
return 1
sha256 = hashlib.sha256(file_path.open('rb').read()).hexdigest()
the_sha256 = hashlib.sha256(file_path.open('rb').read()).hexdigest()
if json_path:
with json_path.open('r') as s:
@@ -44,7 +44,7 @@ def sha256(file_path: str or pathlib.Path,
except json.JSONDecodeError as e:
logger.error(f'Failed to read Json path {json_path}: {str(e)}')
return 1
json_data.update({"sha256": sha256})
json_data.update({"sha256": the_sha256})
utils.backup_file(json_path)
with json_path.open('w') as s:
try:
@@ -53,7 +53,7 @@ def sha256(file_path: str or pathlib.Path,
logger.error(f'Failed to write Json path {json_path}: {str(e)}')
return 1
else:
print(sha256)
print(the_sha256)
return 0
@@ -70,9 +70,9 @@ def add_parser_args(parser):
:param parser: the caller passes an argparse parser like instance to this method
"""
parser.add_argument('-f', '--file-path', type=str, required=True,
help='The path to the file you want to sha256.')
help='The path to the file you want to sha256.')
parser.add_argument('-j', '--json-path', type=str, required=False,
help='optional path to an o3de json file to add the "sha256" element to.')
help='Optional path to an o3de json file to add the "sha256" element to.')
parser.set_defaults(func=_run_sha256)
scripts/o3de/o3de/utils.py
+67 -19
View File
@@ -8,6 +8,7 @@
"""
This file contains utility functions
"""
import argparse
import sys
import uuid
import os
@@ -17,11 +18,54 @@ import urllib.request
import logging
import zipfile
logger = logging.getLogger()
logging.basicConfig()
LOG_FORMAT = '[%(levelname)s] %(name)s: %(message)s'
logger = logging.getLogger('o3de.utils')
logging.basicConfig(format=LOG_FORMAT)
COPY_BUFSIZE = 64 * 1024
class VerbosityAction(argparse.Action):
def __init__(self,
option_strings,
dest,
default=None,
required=False,
help=None):
super().__init__(
option_strings=option_strings,
dest=dest,
nargs=0,
default=default,
required=required,
help=help,
)
def __call__(self, parser, namespace, values, option_string=None):
count = getattr(namespace, self.dest, None)
if count is None:
count = self.default
count += 1
setattr(namespace, self.dest, count)
# get the parent logger instance
log = logging.getLogger('o3de')
if count >= 2:
log.setLevel(logging.DEBUG)
elif count == 1:
log.setLevel(logging.INFO)
def add_verbosity_arg(parser: argparse.ArgumentParser) -> None:
"""
Add a consistent/common verbosity option to an arg parser
:param parser: The ArgumentParser to modify
:return: None
"""
parser.add_argument('-v', dest='verbosity', action=VerbosityAction, default=0,
help='Additional logging verbosity, can be -v or -vv')
def copyfileobj(fsrc, fdst, callback, length=0):
# This is functionally the same as the python shutil copyfileobj but
# allows for a callback to return the download progress in blocks and allows
@@ -43,10 +87,11 @@ def copyfileobj(fsrc, fdst, callback, length=0):
return 1
return 0
def validate_identifier(identifier: str) -> bool:
"""
Determine if the identifier supplied is valid.
:param identifier: the name which needs to to checked
Determine if the identifier supplied is valid
:param identifier: the name which needs to be checked
:return: bool: if the identifier is valid or not
"""
if not identifier:
@@ -65,7 +110,7 @@ def validate_identifier(identifier: str) -> bool:
def sanitize_identifier_for_cpp(identifier: str) -> str:
"""
Convert the provided identifier to a valid C++ identifier
:param identifier: the name which needs to to sanitized
:param identifier: the name which needs to be sanitized
:return: str: sanitized identifier
"""
if not identifier:
@@ -81,8 +126,8 @@ def sanitize_identifier_for_cpp(identifier: str) -> str:
def validate_uuid4(uuid_string: str) -> bool:
"""
Determine if the uuid supplied is valid.
:param uuid_string: the uuid which needs to to checked
Determine if the uuid supplied is valid
:param uuid_string: the uuid which needs to be checked
:return: bool: if the uuid is valid or not
"""
try:
@@ -117,11 +162,13 @@ def backup_folder(folder: str or pathlib.Path) -> None:
if backup_folder_name.is_dir():
renamed = True
def download_file(parsed_uri, download_path: pathlib.Path, force_overwrite: bool = False, download_progress_callback = None) -> int:
"""
Download file
:param parsed_uri: uniform resource identifier to zip file to download
:param download_path: location path on disk to download file
:download_progress_callback: callback called with the download progress as a percentage, returns true to request to cancel the download
:param download_progress_callback: callback called with the download progress as a percentage, returns true to request to cancel the download
"""
if download_path.is_file():
if not force_overwrite:
@@ -142,10 +189,12 @@ def download_file(parsed_uri, download_path: pathlib.Path, force_overwrite: bool
download_file_size = s.headers['content-length']
except KeyError:
pass
def download_progress(downloaded_bytes):
if download_progress_callback:
return download_progress_callback(int(downloaded_bytes), int(download_file_size))
return False
with download_path.open('wb') as f:
download_cancelled = copyfileobj(s, f, download_progress)
if download_cancelled:
@@ -183,13 +232,12 @@ def download_zip_file(parsed_uri, download_zip_path: pathlib.Path, force_overwri
def find_ancestor_file(target_file_name: pathlib.PurePath, start_path: pathlib.Path,
max_scan_up_range: int=0) -> pathlib.Path or None:
"""
Find a file with the given name in the ancestor directories by walking up the starting path until the file is found.
:param target_file_name: Name of the file to find.
:param start_path: path to start looking for the file.
Find a file with the given name in the ancestor directories by walking up the starting path until the file is found
:param target_file_name: Name of the file to find
:param start_path: path to start looking for the file
:param max_scan_up_range: maximum number of directories to scan upwards when searching for target file
if the value is 0, then there is no max
:return: Path to the file or None if not found.
:return: Path to the file or None if not found
"""
current_path = pathlib.Path(start_path)
candidate_path = current_path / target_file_name
@@ -211,17 +259,17 @@ def find_ancestor_file(target_file_name: pathlib.PurePath, start_path: pathlib.P
return candidate_path if candidate_path.exists() else None
def find_ancestor_dir_containing_file(target_file_name: pathlib.PurePath, start_path: pathlib.Path,
max_scan_up_range: int=0) -> pathlib.Path or None:
"""
Find nearest ancestor directory that contains the file with the given name by walking up
from the starting path.
:param target_file_name: Name of the file to find.
:param start_path: path to start looking for the file.
Find the nearest ancestor directory that contains the file with the given name by walking up
from the starting path
:param target_file_name: Name of the file to find
:param start_path: path to start looking for the file
:param max_scan_up_range: maximum number of directories to scan upwards when searching for target file
if the value is 0, then there is no max
:return: Path to the directory containing file or None if not found.
:return: Path to the directory containing file or None if not found
"""
ancestor_file = find_ancestor_file(target_file_name, start_path, max_scan_up_range)
return ancestor_file.parent if ancestor_file else None
scripts/o3de/o3de/validation.py
+16 -13
View File
@@ -11,6 +11,7 @@ This file validating o3de object json files
import json
import pathlib
def valid_o3de_json_dict(json_data: dict, key: str) -> bool:
return key in json_data
@@ -23,9 +24,9 @@ def valid_o3de_repo_json(file_name: str or pathlib.Path) -> bool:
with file_name.open('r') as f:
try:
json_data = json.load(f)
test = json_data['repo_name']
test = json_data['origin']
except (json.JSONDecodeError, KeyError) as e:
_ = json_data['repo_name']
_ = json_data['origin']
except (json.JSONDecodeError, KeyError):
return False
return True
@@ -38,8 +39,8 @@ def valid_o3de_engine_json(file_name: str or pathlib.Path) -> bool:
with file_name.open('r') as f:
try:
json_data = json.load(f)
test = json_data['engine_name']
except (json.JSONDecodeError, KeyError) as e:
_ = json_data['engine_name']
except (json.JSONDecodeError, KeyError):
return False
return True
@@ -52,8 +53,8 @@ def valid_o3de_project_json(file_name: str or pathlib.Path) -> bool:
with file_name.open('r') as f:
try:
json_data = json.load(f)
test = json_data['project_name']
except (json.JSONDecodeError, KeyError) as e:
_ = json_data['project_name']
except (json.JSONDecodeError, KeyError):
return False
return True
@@ -66,8 +67,8 @@ def valid_o3de_gem_json(file_name: str or pathlib.Path) -> bool:
with file_name.open('r') as f:
try:
json_data = json.load(f)
test = json_data['gem_name']
except (json.JSONDecodeError, KeyError) as e:
_ = json_data['gem_name']
except (json.JSONDecodeError, KeyError):
return False
return True
@@ -76,11 +77,12 @@ def valid_o3de_template_json(file_name: str or pathlib.Path) -> bool:
file_name = pathlib.Path(file_name).resolve()
if not file_name.is_file():
return False
with file_name.open('r') as f:
try:
json_data = json.load(f)
test = json_data['template_name']
except (json.JSONDecodeError, KeyError) as e:
_ = json_data['template_name']
except (json.JSONDecodeError, KeyError):
return False
return True
@@ -89,10 +91,11 @@ def valid_o3de_restricted_json(file_name: str or pathlib.Path) -> bool:
file_name = pathlib.Path(file_name).resolve()
if not file_name.is_file():
return False
with file_name.open('r') as f:
try:
json_data = json.load(f)
test = json_data['restricted_name']
except (json.JSONDecodeError, KeyError) as e:
_ = json_data['restricted_name']
except (json.JSONDecodeError, KeyError):
return False
return True