From d3fb2dd68c2907779c8b8832fee8b058e3082873 Mon Sep 17 00:00:00 2001 From: lumberyard-employee-dm <56135373+lumberyard-employee-dm@users.noreply.github.com> Date: Mon, 24 May 2021 22:49:37 -0500 Subject: [PATCH] Removed the add_external_subdirectory and add_gem_cmake python scripts as well as their remove counterparts. Updated teh register.py script to be able to register subdirectories to the o3de_manifest.json Also added the ability to register external subdirectories to the project.json if the --external-subdirectory-project-path is supplied Added the ability to register external subdirectories to the engine.json if the --external-subdirector-engine-path is supplied --- scripts/o3de.py | 15 +- .../o3de/o3de/add_external_subdirectory.py | 168 ---------- scripts/o3de/o3de/add_gem_cmake.py | 138 -------- scripts/o3de/o3de/add_gem_project.py | 5 +- scripts/o3de/o3de/manifest.py | 66 +++- scripts/o3de/o3de/register.py | 315 ++++++++---------- .../o3de/o3de/remove_external_subdirectory.py | 120 ------- scripts/o3de/o3de/remove_gem_cmake.py | 122 ------- scripts/o3de/o3de/remove_gem_project.py | 10 +- 9 files changed, 190 insertions(+), 769 deletions(-) delete mode 100644 scripts/o3de/o3de/add_external_subdirectory.py delete mode 100644 scripts/o3de/o3de/add_gem_cmake.py delete mode 100644 scripts/o3de/o3de/remove_external_subdirectory.py delete mode 100644 scripts/o3de/o3de/remove_gem_cmake.py diff --git a/scripts/o3de.py b/scripts/o3de.py index cc3a14a8c3..050d860790 100755 --- a/scripts/o3de.py +++ b/scripts/o3de.py @@ -40,8 +40,7 @@ def add_args(parser, subparsers) -> None: sys.path.remove(str(script_abs_dir.resolve())) from o3de import engine_template, global_project, register, print_registration, get_registration, download, \ - add_external_subdirectory, remove_external_subdirectory, add_gem_cmake, remove_gem_cmake, add_gem_project, \ - remove_gem_project, sha256 + add_gem_project, remove_gem_project, sha256 if script_abs_dir_removed: sys.path.insert(0, str(script_abs_dir)) @@ -65,18 +64,6 @@ def add_args(parser, subparsers) -> None: # download download.add_args(subparsers) - # add external subdirectories - add_external_subdirectory.add_args(subparsers) - - # remove external subdirectories - remove_external_subdirectory.add_args(subparsers) - - # add gems to cmake - add_gem_cmake.add_args(subparsers) - - # remove gems from cmake - remove_gem_cmake.add_args(subparsers) - # add a gem to a project add_gem_project.add_args(subparsers) diff --git a/scripts/o3de/o3de/add_external_subdirectory.py b/scripts/o3de/o3de/add_external_subdirectory.py deleted file mode 100644 index 29013d30c8..0000000000 --- a/scripts/o3de/o3de/add_external_subdirectory.py +++ /dev/null @@ -1,168 +0,0 @@ -# -# All or portions of this file Copyright (c) Amazon.com, Inc. or its affiliates or -# its licensors. -# -# For complete copyright and license terms please see the LICENSE at the root of this -# distribution (the "License"). All use of this software is governed by the License, -# or, if provided, by the license below or the license accompanying this file. Do not -# remove or modify any license notices. This file is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# -""" -Contains command to add an external_subdirectory to a project's cmake scripts -""" - -import argparse -import logging -import pathlib -import sys - -from o3de import manifest - -logger = logging.getLogger() -logging.basicConfig() - -def add_external_subdirectory(external_subdir: str or pathlib.Path, - engine_path: str or pathlib.Path = None) -> int: - """ - add external subdirectory to a cmake - :param external_subdir: external subdirectory to add to cmake - :param engine_path: optional engine path, defaults to this engine - :return: 0 for success or non 0 failure code - """ - external_subdir = pathlib.Path(external_subdir).resolve() - if not external_subdir.is_dir(): - logger.error(f'Add External Subdirectory Failed: {external_subdir} does not exist.') - return 1 - - external_subdir_cmake = external_subdir / 'CMakeLists.txt' - if not external_subdir_cmake.is_file(): - logger.error(f'Add External Subdirectory Failed: {external_subdir} does not contain a CMakeLists.txt.') - return 1 - - json_data = manifest.load_o3de_manifest() - engine_object = manifest.find_engine_data(json_data, engine_path) - if not engine_object: - logger.error(f'Add External Subdirectory Failed: {engine_path} not registered.') - return 1 - - engine_object.setdefault('external_subdirectories', []) - while external_subdir.as_posix() in engine_object['external_subdirectories']: - engine_object['external_subdirectories'].remove(external_subdir.as_posix()) - - def parse_cmake_file(cmake: str or pathlib.Path, - files: set): - cmake_path = pathlib.Path(cmake).resolve() - cmake_file = cmake_path - if cmake_path.is_dir(): - files.add(cmake_path) - cmake_file = cmake_path / 'CMakeLists.txt' - elif cmake_path.is_file(): - cmake_path = cmake_path.parent - else: - return - - with cmake_file.open('r') as s: - lines = s.readlines() - for line in lines: - line = line.strip() - start = line.find('include(') - if start == 0: - end = line.find(')', start) - if end > start + len('include('): - try: - include_cmake_file = pathlib.Path(engine_path / line[start + len('include('): end]).resolve() - except FileNotFoundError as e: - pass - else: - parse_cmake_file(include_cmake_file, files) - else: - start = line.find('add_subdirectory(') - if start == 0: - end = line.find(')', start) - if end > start + len('add_subdirectory('): - try: - include_cmake_file = pathlib.Path( - cmake_path / line[start + len('add_subdirectory('): end]).resolve() - except FileNotFoundError as e: - pass - else: - parse_cmake_file(include_cmake_file, files) - - cmake_files = set() - parse_cmake_file(engine_path, cmake_files) - for external in engine_object["external_subdirectories"]: - parse_cmake_file(external, cmake_files) - - if external_subdir in cmake_files: - manifest.save_o3de_manifest(json_data) - logger.warning(f'External subdirectory {external_subdir.as_posix()} already included by add_subdirectory().') - return 1 - - engine_object['external_subdirectories'].insert(0, external_subdir.as_posix()) - engine_object['external_subdirectories'] = sorted(engine_object['external_subdirectories']) - - manifest.save_o3de_manifest(json_data) - - return 0 - - -def _run_add_external_subdirectory(args: argparse) -> int: - if args.override_home_folder: - manifest.override_home_folder = args.override_home_folder - - return add_external_subdirectory(args.external_subdirectory) - - -def add_parser_args(parser): - """ - add_parser_args is called to add arguments to each command such that it can be - invoked locally or added by a central python file. - Ex. Directly run from this file alone with: python add-external-subdirectory.py "/home/foo/external-subdir" - :param parser: the caller passes an argparse parser like instance to this method - """ - parser.add_argument('external_subdirectory', metavar='external_subdirectory', type=str, - help='add an external subdirectory to cmake') - - parser.add_argument('-ohf', '--override-home-folder', type=str, required=False, - help='By default the home folder is the user folder, override it to this folder.') - - parser.set_defaults(func=_run_add_external_subdirectory) - - -def add_args(subparsers) -> None: - """ - add_args is called to add subparsers arguments to each command such that it can be - a central python file such as o3de.py. - It can be run from the o3de.py script as follows - call add_args and execute: python o3de.py add_external_subdirectory "/home/foo/external-subdir" - :param subparsers: the caller instantiates subparsers and passes it in here - """ - add_external_subdirectory_subparser = subparsers.add_parser('add-external-subdirectory') - add_parser_args(add_external_subdirectory_subparser) - - -def main(): - """ - Runs add_external_subdirectory.py script as standalone script - """ - # parse the command line args - the_parser = argparse.ArgumentParser() - - # add subparsers - - # add args to the parser - add_parser_args(the_parser) - - # parse args - the_args = the_parser.parse_args() - - # run - ret = the_args.func(the_args) if hasattr(the_args, 'func') else 1 - - # return - sys.exit(ret) - - -if __name__ == "__main__": - main() diff --git a/scripts/o3de/o3de/add_gem_cmake.py b/scripts/o3de/o3de/add_gem_cmake.py deleted file mode 100644 index 523fb8dce8..0000000000 --- a/scripts/o3de/o3de/add_gem_cmake.py +++ /dev/null @@ -1,138 +0,0 @@ -# -# All or portions of this file Copyright (c) Amazon.com, Inc. or its affiliates or -# its licensors. -# -# For complete copyright and license terms please see the LICENSE at the root of this -# distribution (the "License"). All use of this software is governed by the License, -# or, if provided, by the license below or the license accompanying this file. Do not -# remove or modify any license notices. This file is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# -""" -Contains command to add a gem to a project's cmake scripts -""" - -import argparse -import logging -import pathlib -import sys - -from o3de import add_external_subdirectory, manifest, validation - -logger = logging.getLogger() -logging.basicConfig() - -def add_gem_to_cmake(gem_name: str = None, - gem_path: str or pathlib.Path = None, - engine_name: str = None, - engine_path: str or pathlib.Path = None) -> int: - """ - add a gem to a cmake as an external subdirectory for an engine - :param gem_name: name of the gem to add to cmake - :param gem_path: the path of the gem to add to cmake - :param engine_name: name of the engine to add to cmake - :param engine_path: the path of the engine to add external subdirectory to, default to this engine - :return: 0 for success or non 0 failure code - """ - if not gem_name and not gem_path: - logger.error('Must specify either a Gem name or Gem Path.') - return 1 - - if gem_name and not gem_path: - gem_path = manifest.get_registered(gem_name=gem_name) - - if not gem_path: - logger.error(f'Gem Path {gem_path} has not been registered.') - return 1 - - gem_path = pathlib.Path(gem_path).resolve() - gem_json = gem_path / 'gem.json' - if not gem_json.is_file(): - logger.error(f'Gem json {gem_json} is not present.') - return 1 - if not validation.valid_o3de_gem_json(gem_json): - logger.error(f'Gem json {gem_json} is not valid.') - return 1 - - if not engine_name and not engine_path: - engine_path = manifest.get_this_engine_path() - - if engine_name and not engine_path: - engine_path = manifest.get_registered(engine_name=engine_name) - - if not engine_path: - logger.error(f'Engine Path {engine_path} has not been registered.') - return 1 - - engine_json = engine_path / 'engine.json' - if not engine_json.is_file(): - logger.error(f'Engine json {engine_json} is not present.') - return 1 - if not validation.valid_o3de_engine_json(engine_json): - logger.error(f'Engine json {engine_json} is not valid.') - return 1 - - return add_external_subdirectory.add_external_subdirectory(external_subdir=gem_path, engine_path=engine_path) - -def _run_add_gem_to_cmake(args: argparse) -> int: - if args.override_home_folder: - manifest.override_home_folder = args.override_home_folder - - return add_gem_to_cmake(gem_name=args.gem_name, gem_path=args.gem_path) - - -def add_parser_args(parser): - """ - add_parser_args is called to add arguments to each command such that it can be - invoked locally or added by a central python file. - Ex. Directly run from this file alone with: python add_gem_cmake.py --gem-path "/path/to/gem" - :param parser: the caller passes an argparse parser like instance to this method - """ - group = parser.add_mutually_exclusive_group(required=True) - group.add_argument('-gp', '--gem-path', type=str, required=False, - help='The path to the gem.') - group.add_argument('-gn', '--gem-name', type=str, required=False, - help='The name of the gem.') - - parser.add_argument('-ohf', '--override-home-folder', type=str, required=False, - help='By default the home folder is the user folder, override it to this folder.') - - parser.set_defaults(func=_run_add_gem_to_cmake) - - -def add_args(subparsers) -> None: - """ - add_args is called to add subparsers arguments to each command such that it can be - a central python file such as o3de.py. - It can be run from the o3de.py script as follows - call add_args and execute: python o3de.py add-gem-to-cmake --gem-path "/path/to/gem" - :param subparsers: the caller instantiates subparsers and passes it in here - """ - add_gem_cmake_subparser = subparsers.add_parser('add-gem-to-cmake') - add_parser_args(add_gem_cmake_subparser) - - -def main(): - """ - Runs add_gem_cmake.py script as standalone script - """ - # parse the command line args - the_parser = argparse.ArgumentParser() - - # add subparsers - - # add args to the parser - add_parser_args(the_parser) - - # parse args - the_args = the_parser.parse_args() - - # run - ret = the_args.func(the_args) if hasattr(the_args, 'func') else 1 - - # return - sys.exit(ret) - - -if __name__ == "__main__": - main() diff --git a/scripts/o3de/o3de/add_gem_project.py b/scripts/o3de/o3de/add_gem_project.py index 78dffc4477..8eb1468485 100644 --- a/scripts/o3de/o3de/add_gem_project.py +++ b/scripts/o3de/o3de/add_gem_project.py @@ -19,7 +19,7 @@ import os import pathlib import sys -from o3de import add_gem_cmake, cmake, manifest, validation +from o3de import cmake, manifest, validation logger = logging.getLogger() logging.basicConfig() @@ -239,9 +239,6 @@ def add_gem_to_project(gem_name: str = None, # add the dependency ret_val = add_gem_dependency(project_server_dependencies_file, gem_target) - if not ret_val and add_to_cmake: - ret_val = add_gem_cmake.add_gem_to_cmake(gem_path=gem_path, engine_path=engine_path) - return ret_val diff --git a/scripts/o3de/o3de/manifest.py b/scripts/o3de/o3de/manifest.py index 6c14c2533f..bc27d9116c 100644 --- a/scripts/o3de/o3de/manifest.py +++ b/scripts/o3de/o3de/manifest.py @@ -131,7 +131,7 @@ def get_o3de_manifest() -> pathlib.Path: json_data.update({'default_restricted_folder': default_restricted_folder.as_posix()}) json_data.update({'projects': []}) - json_data.update({'gems': []}) + json_data.update({'external_subdirectories': []}) json_data.update({'templates': []}) json_data.update({'restricted': []}) json_data.update({'repos': []}) @@ -172,8 +172,15 @@ def get_o3de_manifest() -> pathlib.Path: return manifest_path -def load_o3de_manifest() -> dict: - with get_o3de_manifest().open('r') as f: +def load_o3de_manifest(manifest_path: pathlib.Path = None) -> dict: + """ + Loads supplied manifest file or ~/.o3de/o3de_manifest.json if None + + :param manifest_path: optional path to manifest file to load + """ + if not manifest_path: + manifest_path = get_o3de_manifest() + with manifest_path.open('r') as f: try: json_data = json.load(f) except json.JSONDecodeError as e: @@ -183,8 +190,16 @@ def load_o3de_manifest() -> dict: return json_data -def save_o3de_manifest(json_data: dict) -> None: - with get_o3de_manifest().open('w') as s: +def save_o3de_manifest(json_data: dict, manifest_path: pathlib.Path = None) -> None: + """ + Save the json dictionary to the supplied manifest file or ~/.o3de/o3de_manifest.json if None + + :param json_data: dictionary to save in json format at the file path + :param manifest_path: optional path to manifest file to save + """ + if not manifest_path: + manifest_path = get_o3de_manifest() + with manifest_path.open('w') as s: try: s.write(json.dumps(json_data, indent=4)) except OSError as e: @@ -198,36 +213,44 @@ def get_this_engine() -> dict: return engine_data -def get_engines() -> dict: +def get_engines() -> list: json_data = load_o3de_manifest() return json_data['engines'] -def get_projects() -> dict: +def get_projects() -> list: json_data = load_o3de_manifest() return json_data['projects'] -def get_gems() -> dict: - json_data = load_o3de_manifest() - return json_data['gems'] +def get_gems() -> list: + def is_gem_subdirectory(subdir): + return (pathlib.Path(subdir) / 'gem.json').exists() + + external_subdirs = get_external_subdirectories() + return list(filter(is_gem_subdirectory, external_subdirs)) if external_subdirs else [] -def get_templates() -> dict: +def get_templates() -> list: json_data = load_o3de_manifest() return json_data['templates'] -def get_restricted() -> dict: +def get_restricted() -> list: json_data = load_o3de_manifest() return json_data['restricted'] -def get_repos() -> dict: +def get_external_subdirectories() -> list: json_data = load_o3de_manifest() - return json_data['repos'] + return json_data['external_subdirectories'] +def get_repos() -> list: + json_data = load_o3de_manifest() + return json_data['repos'] + +# engine.json queries def get_engine_projects() -> list: engine_path = get_this_engine_path() engine_object = get_engine_json_data(engine_path=engine_path) @@ -264,6 +287,21 @@ def get_engine_external_subdirectories() -> list: engine_object['external_subdirectories'])) if 'external_subdirectories' in engine_object else [] +# project.json queries +def get_project_gems(project_path: pathlib.Path) -> list: + def is_gem_subdirectory(subdir): + return (pathlib.Path(subdir) / 'gem.json').exists() + + external_subdirs = get_project_external_subdirectories() + return list(filter(is_gem_subdirectory, external_subdirs)) if external_subdirs else [] + + +def get_project_external_subdirectories(project_path: pathlib.Path) -> list: + project_object = get_project_json_data(project_path=project_path) + return list(map(lambda rel_path: (pathlib.Path(project_path) / rel_path).as_posix(), + project_object['external_subdirectories'])) if 'external_subdirectories' in project_object else [] + + def get_all_projects() -> list: engine_projects = get_engine_projects() projects_data = get_projects() diff --git a/scripts/o3de/o3de/register.py b/scripts/o3de/o3de/register.py index c44af03b30..1b30448db9 100644 --- a/scripts/o3de/o3de/register.py +++ b/scripts/o3de/o3de/register.py @@ -1,3 +1,4 @@ + # # All or portions of this file Copyright (c) Amazon.com, Inc. or its affiliates or # its licensors. @@ -23,7 +24,7 @@ import sys import urllib.parse import urllib.request -from o3de import add_gem_cmake, get_registration, manifest, remove_external_subdirectory, repo, utils, validation +from o3de import get_registration, manifest, repo, utils, validation logger = logging.getLogger() logging.basicConfig() @@ -183,7 +184,8 @@ def register_all_projects_in_folder(projects_path: str or pathlib.Path, def register_all_gems_in_folder(gems_path: str or pathlib.Path, remove: bool = False, - engine_path: str or pathlib.Path = None) -> int: + engine_path: pathlib.Path = None, + project_path: pathlib.Path = None) -> int: return register_all_o3de_objects_of_type_in_folder(gems_path, 'gem', remove, False, engine_path=engine_path) @@ -283,106 +285,122 @@ def register_engine_path(json_data: dict, return add_engine_name_to_path(json_data, engine_path, force) -def register_gem_path(json_data: dict, - gem_path: str or pathlib.Path, - remove: bool = False, - engine_path: str or pathlib.Path = None) -> int: - if not gem_path: - logger.error(f'Gem path cannot be empty.') +def register_o3de_object_path(json_data: dict, + o3de_object_path: str or pathlib.Path, + o3de_object_key: str, + o3de_json_filename: str, + validation_func: callable, + remove: bool = False, + engine_path: pathlib.Path = None, + project_path: pathlib.Path = None) -> int: + # save_path variable is used to save the changes to the store the path to the file to save + # if the registration is for the project or engine + save_path = None + + if not o3de_object_path: + logger.error(f'o3de object path cannot be empty.') return 1 - gem_path = pathlib.Path(gem_path).resolve() + o3de_object_path = pathlib.Path(o3de_object_path).resolve() + + if engine_path and project_path: + logger.error(f'Both a project path: {project_path} and engine path: {engine_path} has been supplied.' + 'A subdirectory can only be registered to either the engine path or project in one command') + + manifest_data = None if engine_path: - engine_data = manifest.find_engine_data(json_data, engine_path) - if not engine_data: - logger.error(f'Engine path {engine_path} is not registered.') + manifest_data = manifest.get_engine_json_data(json_data, engine_path) + if not manifest_data: + logger.error(f'Cannot load engine.json data at path {engine_path}') return 1 - engine_data['gems'] = list(filter(lambda p: gem_path != pathlib.Path(p), engine_data['gems'])) + save_path = engine_path / 'engine.json' + elif project_path: + manifest_data = manifest.get_project_json_data(json_data, project_path) + if not manifest_data: + logger.error(f'Cannot load project.json data at path {project_path}') + return 1 - if remove: - logger.warn(f'Removing Gem path {gem_path}.') - return 0 + save_path = project_path / 'project.json' else: - json_data['gems'] = list(filter(lambda p: gem_path != pathlib.Path(p), json_data['gems'])) + manifest_data = json_data - if remove: - logger.warn(f'Removing Gem path {gem_path}.') - return 0 + paths_to_remove = [o3de_object_path] + if save_path: + try: + paths_to_remove.append(o3de_object_path.relative_to(save_path.parent)) + except ValueError: + pass # It is OK relative path cannot be formed + manifest_data[o3de_object_key] = list(filter(lambda p: pathlib.Path(p) not in paths_to_remove, + manifest_data.setdefault(o3de_object_key, []))) - if not gem_path.is_dir(): - logger.error(f'Gem path {gem_path} does not exist.') + if remove: + if save_path: + manifest.save_o3de_manifest(manifest_data, save_path) + return 0 + + if not o3de_object_path.is_dir(): + logger.error(f'o3de object path {o3de_object_path} does not exist.') return 1 - gem_json = gem_path / 'gem.json' - if not validation.valid_o3de_gem_json(gem_json): - logger.error(f'Gem json {gem_json} is not valid.') + manifest_json_path = o3de_object_path / o3de_json_filename + if validation_func and not validation_func(manifest_json_path): + logger.error(f'o3de json {manifest_json_path} is not valid.') return 1 - if engine_path: - engine_data['gems'].insert(0, gem_path.as_posix()) - else: - json_data['gems'].insert(0, gem_path.as_posix()) + # if there is a save path make it relative the directory containing o3de object json file + if save_path: + try: + o3de_object_path = o3de_object_path.relative_to(save_path.parent) + except ValueError: + pass # It is OK relative path cannot be formed + manifest_data[o3de_object_key].insert(0, o3de_object_path.as_posix()) + if save_path: + manifest.save_o3de_manifest(manifest_data, save_path) return 0 +def register_external_subdirectory(json_data: dict, + external_subdir_path: str or pathlib.Path, + remove: bool = False, + engine_path: pathlib.Path = None, + project_path: pathlib.Path = None) -> int: + """ + :return An integer return code indicating whether registration or removal of the external subdirectory + completed successfully + """ + return register_o3de_object_path(json_data, external_subdir_path, 'external_subdirectories', '', None, remove, + engine_path, project_path) + + +def register_gem_path(json_data: dict, + gem_path: str or pathlib.Path, + remove: bool = False, + engine_path: pathlib.Path = None, + project_path: pathlib.Path = None) -> int: + return register_o3de_object_path(json_data, gem_path, 'external_subdirectories', 'gem.json', + validation.valid_o3de_gem_json, remove, engine_path, project_path) + + def register_project_path(json_data: dict, project_path: str or pathlib.Path, remove: bool = False, engine_path: str or pathlib.Path = None) -> int: - if not project_path: - logger.error(f'Project path cannot be empty.') - return 1 - project_path = pathlib.Path(project_path).resolve() - - if engine_path: - engine_data = manifest.find_engine_data(json_data, engine_path) - if not engine_data: - logger.error(f'Engine path {engine_path} is not registered.') - return 1 + result = register_o3de_object_path(json_data, project_path, 'projects', 'project.json', + validation.valid_o3de_project_json, remove, engine_path, None) - engine_data['projects'] = list(filter(lambda p: project_path != pathlib.Path(p), engine_data['projects'])) + if result != 0: + return result - if remove: - logger.warn(f'Engine {engine_path} removing Project path {project_path}.') - return 0 - else: - json_data['projects'] = list(filter(lambda p: project_path != pathlib.Path(p), json_data['projects'])) - - if remove: - logger.warn(f'Removing Project path {project_path}.') - return 0 - - if not project_path.is_dir(): - logger.error(f'Project path {project_path} does not exist.') + # registering a project has the additional step of setting the project.json 'engine' field + this_engine_json = manifest.get_engine_json_data(engine_path=manifest.get_this_engine_path()) + if not this_engine_json: return 1 - - project_json = project_path / 'project.json' - if not validation.valid_o3de_project_json(project_json): - logger.error(f'Project json {project_json} is not valid.') + project_json_data = manifest.get_project_json_data(project_path=project_path) + if not project_json_data: return 1 - if engine_path: - engine_data['projects'].insert(0, project_path.as_posix()) - else: - json_data['projects'].insert(0, project_path.as_posix()) - - # registering a project has the additional step of setting the project.json 'engine' field - this_engine_json = manifest.get_this_engine_path() / 'engine.json' - with this_engine_json.open('r') as f: - try: - this_engine_json = json.load(f) - except json.JSONDecodeError as e: - logger.error(f'Engine json failed to load: {str(e)}') - return 1 - with project_json.open('r') as f: - try: - project_json_data = json.load(f) - except json.JSONDecodeError as e: - logger.error(f'Project json failed to load: {str(e)}') - return 1 - update_project_json = False try: update_project_json = project_json_data['engine'] != this_engine_json['engine_name'] @@ -399,6 +417,7 @@ def register_project_path(json_data: dict, logger.error(f'Project json failed to save: {str(e)}') return 1 + return 0 @@ -406,88 +425,16 @@ def register_template_path(json_data: dict, template_path: str or pathlib.Path, remove: bool = False, engine_path: str or pathlib.Path = None) -> int: - if not template_path: - logger.error(f'Template path cannot be empty.') - return 1 - template_path = pathlib.Path(template_path).resolve() - - if engine_path: - engine_data = manifest.find_engine_data(json_data, engine_path) - if not engine_data: - logger.error(f'Engine path {engine_path} is not registered.') - return 1 - - engine_data['templates'] = list(filter(lambda p: template_path != pathlib.Path(p), engine_data['templates'])) - - if remove: - logger.warn(f'Engine {engine_path} removing Template path {template_path}.') - return 0 - else: - json_data['templates'] = list(filter(lambda p: template_path != pathlib.Path(p), json_data['templates'])) - - if remove: - logger.warn(f'Removing Template path {template_path}.') - return 0 - - if not template_path.is_dir(): - logger.error(f'Template path {template_path} does not exist.') - return 1 - - template_json = template_path / 'template.json' - if not validation.valid_o3de_template_json(template_json): - logger.error(f'Template json {template_json} is not valid.') - return 1 - - if engine_path: - engine_data['templates'].insert(0, template_path.as_posix()) - else: - json_data['templates'].insert(0, template_path.as_posix()) - - return 0 + return register_o3de_object_path(json_data, template_path, 'templates', 'template.json', + validation.valid_o3de_template_json, remove, engine_path, None) def register_restricted_path(json_data: dict, restricted_path: str or pathlib.Path, remove: bool = False, engine_path: str or pathlib.Path = None) -> int: - if not restricted_path: - logger.error(f'Restricted path cannot be empty.') - return 1 - restricted_path = pathlib.Path(restricted_path).resolve() - - if engine_path: - engine_data = manifest.find_engine_data(json_data, engine_path) - if not engine_data: - logger.error(f'Engine path {engine_path} is not registered.') - return 1 - - engine_data['restricted'] = list(filter(lambda p: restricted_path != pathlib.Path(p), engine_data['restricted'])) - - if remove: - logger.warn(f'Engine {engine_path} removing Restricted path {restricted_path}.') - return 0 - else: - json_data['restricted'] = list(filter(lambda p: restricted_path != pathlib.Path(p), json_data['restricted'])) - - if remove: - logger.warn(f'Removing Restricted path {restricted_path}.') - return 0 - - if not restricted_path.is_dir(): - logger.error(f'Restricted path {restricted_path} does not exist.') - return 1 - - restricted_json = restricted_path / 'restricted.json' - if not validation.valid_o3de_restricted_json(restricted_json): - logger.error(f'Restricted json {restricted_json} is not valid.') - return 1 - - if engine_path: - engine_data['restricted'].insert(0, restricted_path.as_posix()) - else: - json_data['restricted'].insert(0, restricted_path.as_posix()) - - return 0 + return register_o3de_object_path(json_data, restricted_path, 'restricted', 'restricted.json', + validation.valid_o3de_restricted_json, remove, engine_path, None) def register_repo(json_data: dict, @@ -581,6 +528,7 @@ def register_default_restricted_folder(json_data: dict, def register(engine_path: str or pathlib.Path = None, project_path: str or pathlib.Path = None, gem_path: str or pathlib.Path = None, + external_subdir_path: str or pathlib.Path = None, template_path: str or pathlib.Path = None, restricted_path: str or pathlib.Path = None, repo_uri: str or pathlib.Path = None, @@ -589,15 +537,18 @@ def register(engine_path: str or pathlib.Path = None, default_gems_folder: str or pathlib.Path = None, default_templates_folder: str or pathlib.Path = None, default_restricted_folder: str or pathlib.Path = None, + external_subdir_engine_path: pathlib.Path = None, + external_subdir_project_path: pathlib.Path = None, remove: bool = False, force: bool = False ) -> int: """ - Adds/Updates entries to the .o3de/o3de_manifest.json + Adds/Updates entries to the ~/.o3de/o3de_manifest.json :param engine_path: if engine folder is supplied the path will be added to the engine if it can, if not global :param project_path: project folder :param gem_path: gem folder + :param external_subdir_path: external subdirectory :param template_path: template folder :param restricted_path: restricted folder :param repo_uri: repo uri @@ -606,6 +557,10 @@ def register(engine_path: str or pathlib.Path = None, :param default_gems_folder: default gems folder :param default_templates_folder: default templates folder :param default_restricted_folder: default restricted code folder + :param external_subdir_engine_path: Path to the engine to use when registering an external subdirectory. + The registration occurs in the engine.json file in this case + :param external_subdir_engine_path: Path to the project to use when registering an external subdirectory. + The registrations occurs in the project.json in this case :param remove: add/remove the entries :param force: force update of the engine_path for specified "engine_name" from the engine.json file @@ -627,7 +582,14 @@ def register(engine_path: str or pathlib.Path = None, if not gem_path: logger.error(f'Gem path cannot be empty.') return 1 - result = register_gem_path(json_data, gem_path, remove, engine_path) + result = register_gem_path(json_data, gem_path, remove, + external_subdir_engine_path, external_subdir_project_path) + elif isinstance(external_subdir_path, str) or isinstance(external_subdir_path, pathlib.PurePath): + if not external_subdir_path: + logger.error(f'External Subdirectory path is None.') + return 1 + result = register_external_subdirectory(json_data, external_subdir_path, remove, + external_subdir_engine_path, external_subdir_project_path) elif isinstance(template_path, str) or isinstance(template_path, pathlib.PurePath): if not template_path: @@ -685,32 +647,6 @@ def remove_invalid_o3de_objects() -> None: if not validation.valid_o3de_engine_json(pathlib.Path(engine_path).resolve() / 'engine.json'): logger.warn(f"Engine path {engine_path} is invalid.") register(engine_path=engine_path, remove=True) - else: - for project in engine_object['projects']: - if not validation.valid_o3de_project_json(pathlib.Path(project).resolve() / 'project.json'): - logger.warn(f"Project path {project} is invalid.") - register(engine_path=engine_path, project_path=project, remove=True) - - for gem_path in engine_object['gems']: - if not validation.valid_o3de_gem_json(pathlib.Path(gem_path).resolve() / 'gem.json'): - logger.warn(f"Gem path {gem_path} is invalid.") - register(engine_path=engine_path, gem_path=gem_path, remove=True) - - for template_path in engine_object['templates']: - if not validation.valid_o3de_template_json(pathlib.Path(template_path).resolve() / 'template.json'): - logger.warn(f"Template path {template_path} is invalid.") - register(engine_path=engine_path, template_path=template_path, remove=True) - - for restricted in engine_object['restricted']: - if not validation.valid_o3de_restricted_json(pathlib.Path(restricted).resolve() / 'restricted.json'): - logger.warn(f"Restricted path {restricted} is invalid.") - register(engine_path=engine_path, restricted_path=restricted, remove=True) - - for external in engine_object['external_subdirectories']: - external = pathlib.Path(external).resolve() - if not external.is_dir(): - logger.warn(f"External subdirectory {external} is invalid.") - remove_external_subdirectory.remove_external_subdirectory(external) for project in json_data['projects']: if not validation.valid_o3de_project_json(pathlib.Path(project).resolve() / 'project.json'): @@ -722,6 +658,12 @@ def remove_invalid_o3de_objects() -> None: logger.warn(f"Gem path {gem} is invalid.") register(gem_path=gem, remove=True) + for external in json_data['external_subdirectories']: + external = pathlib.Path(external).resolve() + if not external.is_dir(): + logger.warn(f"External subdirectory {external} is invalid.") + register(engine_path=engine_path, external_subdir_path=external, remove=True) + for template in json_data['templates']: if not validation.valid_o3de_template_json(pathlib.Path(template).resolve() / 'template.json'): logger.warn(f"Template path {template} is invalid.") @@ -804,6 +746,7 @@ def _run_register(args: argparse) -> int: return register(engine_path=args.engine_path, project_path=args.project_path, gem_path=args.gem_path, + external_subdir_path=args.external_subdirectory, template_path=args.template_path, restricted_path=args.restricted_path, repo_uri=args.repo_uri, @@ -812,6 +755,8 @@ def _run_register(args: argparse) -> int: default_gems_folder=args.default_gems_folder, default_templates_folder=args.default_templates_folder, default_restricted_folder=args.default_restricted_folder, + external_subdir_engine_path=args.external_subdirectory_engine_path, + external_subdir_project_path=args.external_subdirectory_project_path, remove=args.remove, force=args.force) @@ -833,6 +778,8 @@ def add_parser_args(parser): help='Project path to register/remove.') group.add_argument('-gp', '--gem-path', type=str, required=False, help='Gem path to register/remove.') + group.add_argument('-es', '--external-subdirectory', type=str, required=False, + help='External subdirectory path to register/remove.') group.add_argument('-tp', '--template-path', type=str, required=False, help='Template path to register/remove.') group.add_argument('-rp', '--restricted-path', type=str, required=False, @@ -872,6 +819,14 @@ def add_parser_args(parser): help='Remove entry.') parser.add_argument('-f', '--force', action='store_true', default=False, help='For the update of the registration field being modified.') + + external_subdir_group = parser.add_argument_group(title='external-subdirectory', + description='path arguments to use with the --external-subdirectory option') + external_subdir_path_group = external_subdir_group.add_mutually_exclusive_group() + external_subdir_path_group.add_argument('-esep', '--external-subdirectory-engine-path', type=pathlib.Path, + help='If supplied, registers the external subdirectory with the engine.json at' \ + ' the engine-path location') + external_subdir_path_group.add_argument('-espp', '--external-subdirectory-project-path', type=pathlib.Path) parser.set_defaults(func=_run_register) diff --git a/scripts/o3de/o3de/remove_external_subdirectory.py b/scripts/o3de/o3de/remove_external_subdirectory.py deleted file mode 100644 index b433e9c398..0000000000 --- a/scripts/o3de/o3de/remove_external_subdirectory.py +++ /dev/null @@ -1,120 +0,0 @@ -# -# All or portions of this file Copyright (c) Amazon.com, Inc. or its affiliates or -# its licensors. -# -# For complete copyright and license terms please see the LICENSE at the root of this -# distribution (the "License"). All use of this software is governed by the License, -# or, if provided, by the license below or the license accompanying this file. Do not -# remove or modify any license notices. This file is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# -""" -Implemens functinality to remove external_subdirectories from the o3de_manifests.json -""" - -import argparse -import logging -import pathlib -import sys - -from o3de import manifest - -logger = logging.getLogger() -logging.basicConfig() - -def remove_external_subdirectory(external_subdir: str or pathlib.Path, - engine_path: str or pathlib.Path = None) -> int: - """ - remove external subdirectory from cmake - :param external_subdir: external subdirectory to add to cmake - :param engine_path: optional engine path, defaults to this engine - :return: 0 for success or non 0 failure code - """ - json_data = manifest.load_o3de_manifest() - engine_object = manifest.find_engine_data(json_data, engine_path) - if not engine_object or not 'external_subdirectories' in engine_object: - logger.error(f'Remove External Subdirectory Failed: {engine_path} not registered.') - return 1 - - external_subdir = pathlib.Path(external_subdir).resolve() - while external_subdir.as_posix() in engine_object['external_subdirectories']: - engine_object['external_subdirectories'].remove(external_subdir.as_posix()) - - manifest.save_o3de_manifest(json_data) - - return 0 - - -def _run_remove_external_subdirectory(args: argparse) -> int: - if args.override_home_folder: - manifest.override_home_folder = args.override_home_folder - - return remove_external_subdirectory(args.external_subdirectory) - - -def add_args(parser, subparsers) -> None: - """ - add_args is called to add expected parser arguments and subparsers arguments to each command such that it can be - invoked locally or added by a central python file. - Ex. Directly run from this file alone with: python register.py register --gem-path "C:/TestGem" - OR - o3de.py can downloadable commands by importing engine_template, - call add_args and execute: python o3de.py register --gem-path "C:/TestGem" - :param parser: the caller instantiates a parser and passes it in here - :param subparsers: the caller instantiates subparsers and passes it in here - """ - - -def add_parser_args(parser): - """ - add_parser_args is called to add arguments to each command such that it can be - invoked locally or added by a central python file. - Ex. Directly run from this file alone with: python remove_external_subdirectory.py "D:/subdir" - :param parser: the caller passes an argparse parser like instance to this method - """ - parser.add_argument('external_subdirectory', metavar='external_subdirectory', - type=str, - help='remove external subdirectory from cmake') - - parser.add_argument('-ohf', '--override-home-folder', type=str, required=False, - help='By default the home folder is the user folder, override it to this folder.') - - parser.set_defaults(func=_run_remove_external_subdirectory) - - -def add_args(subparsers) -> None: - """ - add_args is called to add subparsers arguments to each command such that it can be - a central python file such as o3de.py. - It can be run from the o3de.py script as follows - call add_args and execute: python o3de.py remove-external-subdirectory "D:/subdir" - :param subparsers: the caller instantiates subparsers and passes it in here - """ - remove_external_subdirectory_subparser = subparsers.add_parser('remove-external-subdirectory') - add_parser_args(remove_external_subdirectory_subparser) - - -def main(): - """ - Runs remove_external_subdirectory.py script as standalone script - """ - # parse the command line args - the_parser = argparse.ArgumentParser() - - # add subparsers - - # add args to the parser - add_parser_args(the_parser) - - # parse args - the_args = the_parser.parse_args() - - # run - ret = the_args.func(the_args) if hasattr(the_args, 'func') else 1 - - # return - sys.exit(ret) - - -if __name__ == "__main__": - main() diff --git a/scripts/o3de/o3de/remove_gem_cmake.py b/scripts/o3de/o3de/remove_gem_cmake.py deleted file mode 100644 index 3d988a579a..0000000000 --- a/scripts/o3de/o3de/remove_gem_cmake.py +++ /dev/null @@ -1,122 +0,0 @@ -# -# All or portions of this file Copyright (c) Amazon.com, Inc. or its affiliates or -# its licensors. -# -# For complete copyright and license terms please see the LICENSE at the root of this -# distribution (the "License"). All use of this software is governed by the License, -# or, if provided, by the license below or the license accompanying this file. Do not -# remove or modify any license notices. This file is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# -""" -Contains methods for removing a gem from a project's cmake scripts -""" - -import argparse -import logging -import pathlib -import sys - -from o3de import manifest, remove_external_subdirectory - -logger = logging.getLogger() -logging.basicConfig() - -def remove_gem_from_cmake(gem_name: str = None, - gem_path: str or pathlib.Path = None, - engine_name: str = None, - engine_path: str or pathlib.Path = None) -> int: - """ - remove a gem to cmake as an external subdirectory - :param gem_name: name of the gem to remove from cmake - :param gem_path: the path of the gem to add to cmake - :param engine_name: optional name of the engine to remove from cmake - :param engine_path: the path of the engine to remove external subdirectory from, defaults to this engine - :return: 0 for success or non 0 failure code - """ - if not gem_name and not gem_path: - logger.error('Must specify either a Gem name or Gem Path.') - return 1 - - if gem_name and not gem_path: - gem_path = manifest.get_registered(gem_name=gem_name) - - if not gem_path: - logger.error(f'Gem Path {gem_path} has not been registered.') - return 1 - - if not engine_name and not engine_path: - engine_path = manifest.get_this_engine_path() - - if engine_name and not engine_path: - engine_path = manifest.get_registered(engine_name=engine_name) - - if not engine_path: - logger.error(f'Engine Path {engine_path} is not registered.') - return 1 - - return remove_external_subdirectory.remove_external_subdirectory(external_subdir=gem_path, engine_path=engine_path) - - -def _run_remove_gem_from_cmake(args: argparse) -> int: - if args.override_home_folder: - manifest.override_home_folder = args.override_home_folder - - return remove_gem_from_cmake(args.gem_name, args.gem_path) - - -def add_parser_args(parser): - """ - add_parser_args is called to add arguments to each command such that it can be - invoked locally or added by a central python file. - Ex. Directly run from this file alone with: python remove_gem_cmake.py --gem-name Atom - :param parser: the caller passes an argparse parser like instance to this method - """ - group = parser.add_mutually_exclusive_group(required=True) - group.add_argument('-gp', '--gem-path', type=str, required=False, - help='The path to the gem.') - group.add_argument('-gn', '--gem-name', type=str, required=False, - help='The name of the gem.') - - parser.add_argument('-ohf', '--override-home-folder', type=str, required=False, - help='By default the home folder is the user folder, override it to this folder.') - - parser.set_defaults(func=_run_remove_gem_from_cmake) - - -def add_args(subparsers) -> None: - """ - add_args is called to add subparsers arguments to each command such that it can be - a central python file such as o3de.py. - It can be run from the o3de.py script as follows - call add_args and execute: python o3de.py remove-gem-from-cmake --gem-name Atom - :param subparsers: the caller instantiates subparsers and passes it in here - """ - remove_gem_from_cmake_subparser = subparsers.add_parser('remove-gem-from-cmake') - add_parser_args(remove_gem_from_cmake_subparser) - - -def main(): - """ - Runs remove_gem_cmake.py script as standalone script - """ - # parse the command line args - the_parser = argparse.ArgumentParser() - - # add subparsers - - # add args to the parser - add_parser_args(the_parser) - - # parse args - the_args = the_parser.parse_args() - - # run - ret = the_args.func(the_args) if hasattr(the_args, 'func') else 1 - - # return - sys.exit(ret) - - -if __name__ == "__main__": - main() diff --git a/scripts/o3de/o3de/remove_gem_project.py b/scripts/o3de/o3de/remove_gem_project.py index 671427db14..463cc69961 100644 --- a/scripts/o3de/o3de/remove_gem_project.py +++ b/scripts/o3de/o3de/remove_gem_project.py @@ -18,7 +18,7 @@ import os import pathlib import sys -from o3de import cmake, remove_gem_cmake +from o3de import cmake logger = logging.getLogger() logging.basicConfig() @@ -196,11 +196,6 @@ def remove_gem_from_project(gem_name: str = None, if error_code: ret_val = error_code - if remove_from_cmake: - error_code = remove_gem_cmake.remove_gem_from_cmake(gem_path=gem_path) - if error_code: - ret_val = error_code - return ret_val @@ -256,9 +251,6 @@ def add_parser_args(parser): default='Common', help='Optional list of platforms this gem should be removed from' ' Ex. --platforms Mac,Windows,Linux') - parser.add_argument('-r', '--remove-from-cmake', type=bool, required=False, - default=False, - help='Automatically call remove-from-cmake.') parser.add_argument('-ohf', '--override-home-folder', type=str, required=False, help='By default the home folder is the user folder, override it to this folder.')