Merge branch 'development' into o3de_sdk/installer_configs

Signed-off-by: Esteban Papp <81431996+amznestebanpapp@users.noreply.github.com>

# Conflicts:
#	Gems/AtomLyIntegration/CommonFeatures/Code/Source/Material/EditorMaterialSystemComponent.cpp
#	cmake/Platform/Common/Install_common.cmake

scripts/build/Jenkins/Jenkinsfile

@@ -416,16 +416,18 @@ def ExportTestResults(Map options, String platform, String type, String workspac
     }
 }
 
-def ExportTestScreenshots(Map options, String workspace, String platformName, String jobName, Map params) {
+def ExportTestScreenshots(Map options, String branchName, String platformName, String jobName, String workspace, Map params) {
     catchError(message: "Error exporting test screenshots (this won't fail the build)", buildResult: 'SUCCESS', stageResult: 'FAILURE') {
-        def screenshotsFolder = '${workspace}/${ENGINE_REPOSITORY_NAME}/AutomatedTesting/user/PythonTests/Automated/Screenshots'
-        def s3Uploader = '${workspace}/${ENGINE_REPOSITORY_NAME}/scripts/build/tools/upload_to_s3.py'
-        def command = '${options.PYTHON_DIR}/python.cmd -u ${s3Uploader} --base_dir ${screenshotsFolder} ' +
-                      '--file_regex "(.*zip$)" --bucket ${env.TEST_SCREENSHOT_BUCKET} ' +
-                      '--search_subdirectories True --key_prefix ${branchName}_${env.BUILD_NUMBER}' +
-                      '--extra-args {"ACL": "bucket-owner-full-control"}'
-        bat label: "Uploading test screenshots for ${jobName}",
-            script: command
+        dir("${workspace}/${ENGINE_REPOSITORY_NAME}") {
+            def screenshotsFolder = "AutomatedTesting/user/PythonTests/Automated/Screenshots"
+            def s3Uploader = "scripts/build/tools/upload_to_s3.py"
+            def command = "${options.PYTHON_DIR}/python.cmd -u ${s3Uploader} --base_dir ${screenshotsFolder} " +
+                          '--file_regex \\"(.*zip\$)\\" ' +
+                          "--bucket ${env.TEST_SCREENSHOT_BUCKET} " +
+                          "--search_subdirectories True --key_prefix ${branchName}_${env.BUILD_NUMBER} " +
+                          '--extra_args {\\"ACL\\":\\"bucket-owner-full-control\\"}'
+            palSh(command, "Uploading test screenshots for ${jobName}")
+        }
     }
 }
 
@@ -484,10 +486,10 @@ def CreateExportTestResultsStage(Map pipelineConfig, String platformName, String
     }
 }
 
-def CreateExportTestScreenshotsStage(Map pipelineConfig, String platformName, String jobName, Map environmentVars, Map params) {
+def CreateExportTestScreenshotsStage(Map pipelineConfig, String branchName, String platformName, String jobName, Map environmentVars, Map params) {
     return {
         stage("${jobName}_screenshots") {
-            ExportTestScreenshots(pipelineConfig, platformName, jobName, environmentVars['WORKSPACE'], params)
+            ExportTestScreenshots(pipelineConfig, branchName, platformName, jobName, environmentVars['WORKSPACE'], params)
         }
     }
 }
@@ -553,10 +555,10 @@ def CreateSingleNode(Map pipelineConfig, def platform, def build_job, Map envVar
                             CreateTestMetricsStage(pipelineConfig, branchName, envVars, build_job_name, output_directory, configuration).call()
                         }
                         if (params && params.containsKey('TEST_RESULTS') && params.TEST_RESULTS == 'True') {
-                            CreateExportTestResultsStage(pipelineConfig,  platform.key, build_job_name, envVars, params).call()
+                            CreateExportTestResultsStage(pipelineConfig, platform.key, build_job_name, envVars, params).call()
                         }
                         if (params && params.containsKey('TEST_SCREENSHOTS') && params.TEST_SCREENSHOTS == 'True' && currentResult == 'FAILURE') {
-                            CreateExportTestScreenshotsStage(pipelineConfig,  platform.key, build_job_name, envVars, params).call()
+                            CreateExportTestScreenshotsStage(pipelineConfig, branchName, platform.key, build_job_name, envVars, params).call()
                         }
                         CreateTeardownStage(envVars).call()
                     }

scripts/build/Platform/Linux/build_config.json

@@ -83,7 +83,7 @@
       "CMAKE_OPTIONS": "-G 'Ninja Multi-Config' -DCMAKE_C_COMPILER=clang-6.0 -DCMAKE_CXX_COMPILER=clang++-6.0 -DLY_PARALLEL_LINK_JOBS=4",
       "CMAKE_LY_PROJECTS": "AutomatedTesting",
       "CMAKE_TARGET": "all",
-      "CTEST_OPTIONS": "-E Gem::EMotionFX.Editor.Tests -LE (SUITE_sandbox|SUITE_awsi) -L FRAMEWORK_googletest",
+      "CTEST_OPTIONS": "-E Gem::EMotionFX.Editor.Tests -LE (SUITE_sandbox|SUITE_awsi) -L FRAMEWORK_googletest --no-tests=error",
       "TEST_RESULTS": "True"
     }
   },
@@ -96,7 +96,7 @@
       "CMAKE_OPTIONS": "-G 'Ninja Multi-Config' -DCMAKE_C_COMPILER=clang-6.0 -DCMAKE_CXX_COMPILER=clang++-6.0 -DLY_UNITY_BUILD=FALSE -DLY_PARALLEL_LINK_JOBS=4",
       "CMAKE_LY_PROJECTS": "AutomatedTesting",
       "CMAKE_TARGET": "all",
-      "CTEST_OPTIONS": "-E Gem::EMotionFX.Editor.Tests -LE (SUITE_sandbox|SUITE_awsi) -L FRAMEWORK_googletest",
+      "CTEST_OPTIONS": "-E Gem::EMotionFX.Editor.Tests -LE (SUITE_sandbox|SUITE_awsi) -L FRAMEWORK_googletest --no-tests=error",
       "TEST_RESULTS": "True"
     }
   },
@@ -145,7 +145,7 @@
       "CMAKE_OPTIONS": "-G 'Ninja Multi-Config' -DCMAKE_C_COMPILER=clang-6.0 -DCMAKE_CXX_COMPILER=clang++-6.0 -DLY_PARALLEL_LINK_JOBS=4",
       "CMAKE_LY_PROJECTS": "AutomatedTesting",
       "CMAKE_TARGET": "TEST_SUITE_periodic",
-      "CTEST_OPTIONS": "-L (SUITE_periodic)",
+      "CTEST_OPTIONS": "-L (SUITE_periodic) --no-tests=error",
       "TEST_RESULTS": "True"
     }
   },
@@ -165,7 +165,7 @@
       "CMAKE_OPTIONS": "-G 'Ninja Multi-Config' -DCMAKE_C_COMPILER=clang-6.0 -DCMAKE_CXX_COMPILER=clang++-6.0 -DLY_PARALLEL_LINK_JOBS=4 -DO3DE_HOME_PATH=\"${WORKSPACE}/home\" -DO3DE_REGISTER_ENGINE_PATH=\"${WORKSPACE}/o3de\" -DO3DE_REGISTER_THIS_ENGINE=TRUE",
       "CMAKE_LY_PROJECTS": "AutomatedTesting",
       "CMAKE_TARGET": "all",
-      "CTEST_OPTIONS": "-L (SUITE_sandbox)"
+      "CTEST_OPTIONS": "-L (SUITE_sandbox) --no-tests=error"
     }
   },
   "benchmark_test_profile": {
@@ -181,7 +181,7 @@
       "CMAKE_OPTIONS": "-G 'Ninja Multi-Config' -DCMAKE_C_COMPILER=clang-6.0 -DCMAKE_CXX_COMPILER=clang++-6.0 -DLY_PARALLEL_LINK_JOBS=4",
       "CMAKE_LY_PROJECTS": "AutomatedTesting",
       "CMAKE_TARGET": "TEST_SUITE_benchmark",
-      "CTEST_OPTIONS": "-L (SUITE_benchmark)",
+      "CTEST_OPTIONS": "-L (SUITE_benchmark) --no-tests=error",
       "TEST_RESULTS": "True"
     }
   },

scripts/build/Platform/Windows/build_config.json

@@ -117,7 +117,7 @@
       "CMAKE_LY_PROJECTS": "AutomatedTesting",
       "CMAKE_TARGET": "TEST_SUITE_smoke TEST_SUITE_main",
       "CMAKE_NATIVE_BUILD_ARGS": "/m /nologo",
-      "CTEST_OPTIONS": "-L \"(SUITE_smoke|SUITE_main)\" -LE \"(REQUIRES_gpu)\" -T Test",
+      "CTEST_OPTIONS": "-L \"(SUITE_smoke|SUITE_main)\" -LE \"(REQUIRES_gpu)\" -T Test --no-tests=error",
       "TEST_METRICS": "True",
       "TEST_RESULTS": "True"
     }
@@ -166,7 +166,7 @@
       "CMAKE_LY_PROJECTS": "AutomatedTesting",
       "CMAKE_TARGET": "TEST_SUITE_smoke TEST_SUITE_main",
       "CMAKE_NATIVE_BUILD_ARGS": "/m /nologo",
-      "CTEST_OPTIONS": "-L \"(SUITE_smoke|SUITE_main)\" -LE \"(REQUIRES_gpu)\" -T Test",
+      "CTEST_OPTIONS": "-L \"(SUITE_smoke|SUITE_main)\" -LE \"(REQUIRES_gpu)\" -T Test --no-tests=error",
       "TEST_METRICS": "True",
       "TEST_RESULTS": "True"
     }
@@ -187,7 +187,7 @@
       "CMAKE_LY_PROJECTS": "AutomatedTesting",
       "CMAKE_TARGET": "TEST_SUITE_smoke TEST_SUITE_main",
       "CMAKE_NATIVE_BUILD_ARGS": "/m /nologo",
-      "CTEST_OPTIONS": "-L \"(SUITE_smoke_REQUIRES_gpu|SUITE_main_REQUIRES_gpu)\" -T Test",
+      "CTEST_OPTIONS": "-L \"(SUITE_smoke_REQUIRES_gpu|SUITE_main_REQUIRES_gpu)\" -T Test --no-tests=error",
       "TEST_METRICS": "True",
       "TEST_RESULTS": "True",
       "TEST_SCREENSHOTS": "True"
@@ -238,7 +238,7 @@
       "CMAKE_LY_PROJECTS": "AutomatedTesting",
       "CMAKE_TARGET": "TEST_SUITE_awsi",
       "CMAKE_NATIVE_BUILD_ARGS": "/m /nologo",
-      "CTEST_OPTIONS": "-L \"(SUITE_awsi)\" -T Test",
+      "CTEST_OPTIONS": "-L \"(SUITE_awsi)\" -T Test --no-tests=error",
       "TEST_METRICS": "True",
       "TEST_RESULTS": "True"
     }
@@ -257,7 +257,7 @@
       "CMAKE_LY_PROJECTS": "AutomatedTesting",
       "CMAKE_TARGET": "TEST_SUITE_periodic",
       "CMAKE_NATIVE_BUILD_ARGS": "/m /nologo",
-      "CTEST_OPTIONS": "-L \"(SUITE_periodic)\" -T Test",
+      "CTEST_OPTIONS": "-L \"(SUITE_periodic)\" -T Test --no-tests=error",
       "TEST_METRICS": "True",
       "TEST_RESULTS": "True"
     }
@@ -279,7 +279,7 @@
       "CMAKE_LY_PROJECTS": "AutomatedTesting",
       "CMAKE_TARGET": "TEST_SUITE_sandbox",
       "CMAKE_NATIVE_BUILD_ARGS": "/m /nologo",
-      "CTEST_OPTIONS": "-L \"(SUITE_sandbox)\" -T Test",
+      "CTEST_OPTIONS": "-L \"(SUITE_sandbox)\" -T Test --no-tests=error",
       "TEST_METRICS": "True",
       "TEST_RESULTS": "True"
     }
@@ -298,7 +298,7 @@
       "CMAKE_LY_PROJECTS": "AutomatedTesting",
       "CMAKE_TARGET": "TEST_SUITE_benchmark",
       "CMAKE_NATIVE_BUILD_ARGS": "/m /nologo",
-      "CTEST_OPTIONS": "-L \"(SUITE_benchmark)\" -T Test",
+      "CTEST_OPTIONS": "-L \"(SUITE_benchmark)\" -T Test --no-tests=error",
       "TEST_METRICS": "True",
       "TEST_RESULTS": "True"
     }

scripts/o3de/o3de/download.py

@@ -90,7 +90,8 @@ def get_downloadable(engine_name: str = None,
 
 
 def download_o3de_object(object_name: str, default_folder_name: str, dest_path: str or pathlib.Path,
-                         object_type: str, downloadable_kwarg_key, skip_auto_register: bool) -> int:
+                         object_type: str, downloadable_kwarg_key, skip_auto_register: bool,
+                         download_progress_callback = None) -> int:
 
     download_path = manifest.get_o3de_cache_folder() / default_folder_name / object_name
     download_path.mkdir(parents=True, exist_ok=True)
@@ -104,7 +105,7 @@ def download_o3de_object(object_name: str, default_folder_name: str, dest_path:
     origin_uri = downloadable_object_data['originuri']
     parsed_uri = urllib.parse.urlparse(origin_uri)
 
-    download_zip_result = utils.download_zip_file(parsed_uri, download_zip_path)
+    download_zip_result = utils.download_zip_file(parsed_uri, download_zip_path, download_progress_callback)
     if download_zip_result != 0:
         return download_zip_result
 
@@ -147,33 +148,38 @@ def download_o3de_object(object_name: str, default_folder_name: str, dest_path:
 
 def download_engine(engine_name: str,
                     dest_path: str or pathlib.Path,
-                    skip_auto_register: bool) -> int:
-    return download_o3de_object(engine_name, 'engines', dest_path, 'engine', 'engine_name', skip_auto_register)
+                    skip_auto_register: bool,
+                    download_progress_callback = None) -> int:
+    return download_o3de_object(engine_name, 'engines', dest_path, 'engine', 'engine_name', skip_auto_register, download_progress_callback)
 
 
 def download_project(project_name: str,
                      dest_path: str or pathlib.Path,
-                     skip_auto_register: bool) -> int:
-    return download_o3de_object(project_name, 'projects', dest_path, 'project', 'project_name', skip_auto_register)
+                     skip_auto_register: bool,
+                     download_progress_callback = None) -> int:
+    return download_o3de_object(project_name, 'projects', dest_path, 'project', 'project_name', skip_auto_register, download_progress_callback)
 
 
 def download_gem(gem_name: str,
                  dest_path: str or pathlib.Path,
-                 skip_auto_register: bool) -> int:
-    return download_o3de_object(gem_name, 'gems', dest_path, 'gem', 'gem_name', skip_auto_register)
+                 skip_auto_register: bool,
+                 download_progress_callback = None) -> int:
+    return download_o3de_object(gem_name, 'gems', dest_path, 'gem', 'gem_name', skip_auto_register, download_progress_callback)
 
 
 def download_template(template_name: str,
                       dest_path: str or pathlib.Path,
-                      skip_auto_register: bool) -> int:
-    return download_o3de_object(template_name, 'templates', dest_path, 'template', 'template_name', skip_auto_register)
+                      skip_auto_register: bool,
+                      download_progress_callback = None) -> int:
+    return download_o3de_object(template_name, 'templates', dest_path, 'template', 'template_name', skip_auto_register, download_progress_callback)
 
 
 
 def download_restricted(restricted_name: str,
                         dest_path: str or pathlib.Path,
-                        skip_auto_register: bool) -> int:
-    return download_o3de_object(restricted_name, 'restricted', dest_path, 'restricted', 'restricted_name', skip_auto_register)
+                        skip_auto_register: bool,
+                        download_progress_callback = None) -> int:
+    return download_o3de_object(restricted_name, 'restricted', dest_path, 'restricted', 'restricted_name', skip_auto_register, download_progress_callback)
 
 
 def _run_download(args: argparse) -> int:

scripts/o3de/o3de/engine_template.py

@@ -407,15 +407,12 @@ def create_template(source_path: pathlib.Path,
         source_name = os.path.basename(source_path)
     sanitized_source_name = utils.sanitize_identifier_for_cpp(source_name)
 
-    # if no template path, error
+    # if no template path, use default_templates_folder path
     if not template_path:
-        logger.info(f'Template path empty. Using source name {source_name}')
-        template_path = pathlib.Path(source_name)
-    if not template_path.is_absolute():
         default_templates_folder = manifest.get_registered(default_folder='templates')
-        template_path = default_templates_folder / template_path
-        logger.info(f'Template path not a full path. Using default templates folder {template_path}')
-    if not force and template_path.is_dir():
+        template_path = default_templates_folder / source_name
+        logger.info(f'Template path empty. Using default templates folder {template_path}')
+    if not force and template_path.is_dir() and len(list(template_path.iterdir())):
         logger.error(f'Template path {template_path} already exists.')
         return 1
 
@@ -1105,7 +1102,7 @@ def create_from_template(destination_path: pathlib.Path,
         logger.error(f'Could not find the template {template_name}=>{template_path}')
         return 1
 
-    # the template.json should be in the template_path, make sure it's there a nd valid
+    # the template.json should be in the template_path, make sure it is valid
     template_json = template_path / 'template.json'
     if not validation.valid_o3de_template_json(template_json):
         logger.error(f'Template json {template_path} is invalid.')
@@ -1254,7 +1251,7 @@ def create_from_template(destination_path: pathlib.Path,
     # destination restricted path
     elif destination_restricted_path:
         if os.path.isabs(destination_restricted_path):
-            restricted_default_path = manifest.get_registered(default='restricted')
+            restricted_default_path = manifest.get_registered(default_folder='restricted')
             new_destination_restricted_path = restricted_default_path / destination_restricted_path
             logger.info(f'{destination_restricted_path} is not a full path, making it relative'
                         f' to default restricted path = {new_destination_restricted_path}')
@@ -1346,7 +1343,7 @@ def create_project(project_path: pathlib.Path,
     Template instantiation specialization that makes all default assumptions for a Project template instantiation,
      reducing the effort needed in instancing a project
     :param project_path: the project path, can be absolute or relative to default projects path
-    :param project_name: the project name, defaults to project_path basename if not provided 
+    :param project_name: the project name, defaults to project_path basename if not provided
     :param template_path: the path to the template you want to instance, can be absolute or relative to default templates path
     :param template_name: the name the registered template you want to instance, defaults to DefaultProject, resolves template_path
     :param project_restricted_path: path to the projects restricted folder, can be absolute or relative to the restricted='projects'
@@ -1523,13 +1520,9 @@ def create_project(project_path: pathlib.Path,
     if not project_path:
         logger.error('Project path cannot be empty.')
         return 1
-    if not os.path.isabs(project_path):
-        default_projects_folder = manifest.get_registered(default_folder='projects')
-        new_project_path = default_projects_folder / project_path
-        logger.info(f'Project Path {project_path} is not a full path, we must assume its relative'
-                    f' to default projects path = {new_project_path}')
-        project_path = new_project_path
-    if not force and os.path.isdir(project_path) and len(os.listdir(project_path)) > 0:
+
+    project_path = project_path.resolve()
+    if not force and project_path.is_dir() and len(list(project_path.iterdir())):
         logger.error(f'Project path {project_path} already exists and is not empty.')
         return 1
     elif not os.path.isdir(project_path):
@@ -1904,14 +1897,10 @@ def create_gem(gem_path: pathlib.Path,
     if not gem_path:
         logger.error('Gem path cannot be empty.')
         return 1
-    if not os.path.isabs(gem_path):
-        default_gems_folder = manifest.get_registered(default_folder='gems')
-        new_gem_path = default_gems_folder / gem_path
-        logger.info(f'Gem Path {gem_path} is not a full path, we must assume its relative'
-                    f' to default gems path = {new_gem_path}')
-        gem_path = new_gem_path
-    if not force and os.path.isdir(gem_path):
-        logger.error(f'Gem path {gem_path} already exists.')
+
+    gem_path = gem_path.resolve()
+    if not force and gem_path.is_dir() and len(list(gem_path.iterdir())):
+        logger.error(f'Gem path {gem_path} already exists and is not empty.')
         return 1
     else:
         os.makedirs(gem_path, exist_ok=force)
@@ -1936,16 +1925,18 @@ def create_gem(gem_path: pathlib.Path,
     # gem restricted path
     elif gem_restricted_path:
         if not os.path.isabs(gem_restricted_path):
-            default_gems_restricted_folder = manifest.get_registered(restricted_name='gems')
-            new_gem_restricted_path = default_gems_restricted_folder /gem_restricted_path
-            logger.info(f'Gem restricted path {gem_restricted_path} is not a full path, we must assume its'
-                        f' relative to default gems restricted path = {new_gem_restricted_path}')
-            gem_restricted_path = new_gem_restricted_path
-    elif template_restricted_path:
+            gem_restricted_default_path = manifest.get_registered(restricted_name='gems')
+            if gem_restricted_default_path:
+                new_gem_restricted_path = gem_restricted_default_path / gem_restricted_path
+                logger.info(f'Gem restricted path {gem_restricted_path} is not a full path, we must assume its'
+                            f' relative to default gems restricted path = {new_gem_restricted_path}')
+                gem_restricted_path = new_gem_restricted_path
+    else:
         gem_restricted_default_path = manifest.get_registered(restricted_name='gems')
-        logger.info(f'--gem-restricted-path is not specified, using default gem restricted path / gem name'
-                    f' = {gem_restricted_default_path}')
-        gem_restricted_path = gem_restricted_default_path
+        if gem_restricted_default_path:
+            logger.info(f'--gem-restricted-path is not specified, using default <gem restricted path> / <gem name>'
+                        f' = {gem_restricted_default_path}')
+            gem_restricted_path = gem_restricted_default_path / gem_name
 
     # gem restricted relative
     if not gem_restricted_platform_relative_path:
@@ -1964,7 +1955,7 @@ def create_gem(gem_path: pathlib.Path,
     replacements.append(("${NameUpper}", gem_name.upper()))
     replacements.append(("${NameLower}", gem_name.lower()))
     replacements.append(("${SanitizedCppName}", sanitized_cpp_name))
-    
+
 
     # module id is a uuid with { and -
     if module_id:
@@ -2244,14 +2235,14 @@ def add_args(subparsers) -> None:
     create_from_template_subparser = subparsers.add_parser('create-from-template')
     create_from_template_subparser.add_argument('-dp', '--destination-path', type=pathlib.Path, required=True,
                                                 help='The path to where you want the template instantiated,'
-                                                     ' can be absolute or dev root relative.'
+                                                     ' can be absolute or relative to the current working directory.'
                                                      'Ex. C:/o3de/Test'
                                                      'Test = <destination_name>')
 
     group = create_from_template_subparser.add_mutually_exclusive_group(required=True)
     group.add_argument('-tp', '--template-path', type=pathlib.Path, required=False,
                        help='The path to the template you want to instantiate, can be absolute'
-                            ' or dev root/Templates relative.'
+                            ' or relative to the current working directory.'
                             'Ex. C:/o3de/Template/TestTemplate'
                             'TestTemplate = <template_name>')
     group.add_argument('-tn', '--template-name', type=str, required=False,
@@ -2327,7 +2318,7 @@ def add_args(subparsers) -> None:
     create_project_subparser = subparsers.add_parser('create-project')
     create_project_subparser.add_argument('-pp', '--project-path', type=pathlib.Path, required=True,
                                           help='The location of the project you wish to create from the template,'
-                                               ' can be an absolute path or dev root relative.'
+                                               ' can be an absolute path or relative to the current working directory.'
                                                ' Ex. C:/o3de/TestProject'
                                                ' TestProject = <project_name> if --project-name not provided')
     create_project_subparser.add_argument('-pn', '--project-name', type=str, required=False,
@@ -2349,8 +2340,8 @@ def add_args(subparsers) -> None:
     group = create_project_subparser.add_mutually_exclusive_group(required=False)
     group.add_argument('-prp', '--project-restricted-path', type=pathlib.Path, required=False,
                        default=None,
-                       help='path to the projects restricted folder, can be absolute or relative'
-                            ' to the restricted="projects"')
+                       help='path to the projects restricted folder, can be absolute or relative to'
+                            ' the default restricted projects directory')
     group.add_argument('-prn', '--project-restricted-name', type=str, required=False,
                        default=None,
                        help='The name of the registered projects restricted path. If supplied this will resolve'
@@ -2360,7 +2351,7 @@ def add_args(subparsers) -> None:
     group.add_argument('-trp', '--template-restricted-path', type=pathlib.Path, required=False,
                        default=None,
                        help='The templates restricted path can be absolute or relative to'
-                            ' restricted="templates"')
+                             'the default restricted templates directory')
     group.add_argument('-trn', '--template-restricted-name', type=str, required=False,
                        default=None,
                        help='The name of the registered templates restricted path. If supplied this will resolve'
@@ -2423,7 +2414,7 @@ def add_args(subparsers) -> None:
     # creation of a gem from a template (like create from template but makes gem assumptions)
     create_gem_subparser = subparsers.add_parser('create-gem')
     create_gem_subparser.add_argument('-gp', '--gem-path', type=pathlib.Path, required=True,
-                                      help='The gem path, can be absolute or relative to default gems path')
+                                      help='The gem path, can be absolute or relative to the current working directory')
     create_gem_subparser.add_argument('-gn', '--gem-name', type=str,
                                           help='The name to use when substituting the ${Name} placeholder for the gem,'
                                                ' must be alphanumeric, '
@@ -2444,19 +2435,18 @@ def add_args(subparsers) -> None:
     group = create_gem_subparser.add_mutually_exclusive_group(required=False)
     group.add_argument('-grp', '--gem-restricted-path', type=pathlib.Path, required=False,
                        default=None,
-                       help='The path to the gem restricted to write to folder if any, can be'
-                            'absolute or dev root relative, default is dev root/restricted.')
+                       help='The gem restricted path, can be absolute or relative to'
+                            ' the default restricted gems directory')
     group.add_argument('-grn', '--gem-restricted-name', type=str, required=False,
                        default=None,
-                       help='The path to the gem restricted to write to folder if any, can be'
-                            'absolute or dev root relative, default is dev root/restricted. If supplied'
-                            ' this will resolve the --gem-restricted-path.')
+                       help='The name of the gem to look up the gem restricted path if any.'
+                            'If supplied this will resolve the --gem-restricted-path.')
 
     group = create_gem_subparser.add_mutually_exclusive_group(required=False)
     group.add_argument('-trp', '--template-restricted-path', type=pathlib.Path, required=False,
                        default=None,
                        help='The templates restricted path, can be absolute or relative to'
-                            ' the restricted="templates"')
+                            ' the default restricted templates directory')
     group.add_argument('-trn', '--template-restricted-name', type=str, required=False,
                        default=None,
                        help='The name of the registered templates restricted path. If supplied'

scripts/o3de/o3de/manifest.py

@@ -41,6 +41,11 @@ def get_o3de_folder() -> pathlib.Path:
     o3de_folder.mkdir(parents=True, exist_ok=True)
     return o3de_folder
 
+def get_o3de_user_folder() -> pathlib.Path:
+    o3de_user_folder = get_home_folder() / 'O3DE'
+    o3de_user_folder.mkdir(parents=True, exist_ok=True)
+    return o3de_user_folder
+
 
 def get_o3de_registry_folder() -> pathlib.Path:
     registry_folder = get_o3de_folder() / 'Registry'
@@ -67,19 +72,19 @@ def get_o3de_engines_folder() -> pathlib.Path:
 
 
 def get_o3de_projects_folder() -> pathlib.Path:
-    projects_folder = get_o3de_folder() / 'Projects'
+    projects_folder = get_o3de_user_folder() / 'Projects'
     projects_folder.mkdir(parents=True, exist_ok=True)
     return projects_folder
 
 
 def get_o3de_gems_folder() -> pathlib.Path:
-    gems_folder = get_o3de_folder() / 'Gems'
+    gems_folder = get_o3de_user_folder() / 'Gems'
     gems_folder.mkdir(parents=True, exist_ok=True)
     return gems_folder
 
 
 def get_o3de_templates_folder() -> pathlib.Path:
-    templates_folder = get_o3de_folder() / 'Templates'
+    templates_folder = get_o3de_user_folder() / 'Templates'
     templates_folder.mkdir(parents=True, exist_ok=True)
     return templates_folder
 
@@ -497,7 +502,10 @@ def get_gem_json_data(gem_name: str = None, gem_path: str or pathlib.Path = None
     if gem_name and not gem_path:
         gem_path = get_registered(gem_name=gem_name, project_path=project_path)
 
-    return get_json_data('gem', gem_path, validation.valid_o3de_gem_json)
+    if pathlib.Path(gem_path).is_file():
+        return get_json_data_file(gem_path, 'gem', validation.valid_o3de_gem_json)
+    else:
+        return get_json_data('gem', gem_path, validation.valid_o3de_gem_json)
 
 
 def get_template_json_data(template_name: str = None, template_path: str or pathlib.Path = None,

scripts/o3de/o3de/register.py

@@ -467,7 +467,7 @@ def register_restricted_path(json_data: dict,
 
 
 def register_repo(json_data: dict,
-                  repo_uri: str or pathlib.Path,
+                  repo_uri: str,
                   remove: bool = False) -> int:
     if not repo_uri:
         logger.error(f'Repo URI cannot be empty.')
@@ -480,9 +480,9 @@ def register_repo(json_data: dict,
         while repo_uri in json_data['repos']:
             json_data['repos'].remove(repo_uri)
     else:
-        repo_uri = pathlib.Path(repo_uri).resolve()
-        while repo_uri.as_posix() in json_data['repos']:
-            json_data['repos'].remove(repo_uri.as_posix())
+        repo_uri = pathlib.Path(repo_uri).resolve().as_posix()
+        while repo_uri in json_data['repos']:
+            json_data['repos'].remove(repo_uri)
 
     if remove:
         logger.warn(f'Removing repo uri {repo_uri}.')
@@ -566,7 +566,7 @@ def register(engine_path: pathlib.Path = None,
              external_subdir_path: pathlib.Path = None,
              template_path: pathlib.Path = None,
              restricted_path: pathlib.Path = None,
-             repo_uri: str or pathlib.Path = None,
+             repo_uri: str = None,
              default_engines_folder: pathlib.Path = None,
              default_projects_folder: pathlib.Path = None,
              default_gems_folder: pathlib.Path = None,
@@ -596,7 +596,7 @@ def register(engine_path: pathlib.Path = None,
     :param default_third_party_folder: default 3rd party cache folder
     :param external_subdir_engine_path: Path to the engine to use when registering an external subdirectory.
      The registration occurs in the engine.json file in this case
-    :param external_subdir_engine_path: Path to the project to use when registering an external subdirectory.
+    :param external_subdir_project_path: Path to the project to use when registering an external subdirectory.
      The registrations occurs in the project.json in this case
     :param remove: add/remove the entries
     :param force: force update of the engine_path for specified "engine_name" from the engine.json file
@@ -641,7 +641,7 @@ def register(engine_path: pathlib.Path = None,
             return 1
         result = result or register_restricted_path(json_data, restricted_path, remove, project_path, engine_path)
 
-    if isinstance(repo_uri, str) or isinstance(repo_uri, pathlib.PurePath):
+    if isinstance(repo_uri, str):
         if not repo_uri:
             logger.error(f'Repo URI cannot be empty.')
             return 1

scripts/o3de/o3de/repo.py

@@ -13,7 +13,7 @@ import shutil
 import urllib.parse
 import urllib.request
 import hashlib
-
+from datetime import datetime
 from o3de import manifest, utils, validation
 
 logger = logging.getLogger()
@@ -27,6 +27,7 @@ def process_add_o3de_repo(file_name: str or pathlib.Path,
         return 1
     cache_folder = manifest.get_o3de_cache_folder()
 
+    repo_data = {}
     with file_name.open('r') as f:
         try:
             repo_data = json.load(f)
@@ -34,63 +35,146 @@ def process_add_o3de_repo(file_name: str or pathlib.Path,
             logger.error(f'{file_name} failed to load: {str(e)}')
             return 1
 
-        # A repo may not contain all types of object.
-        manifest_download_list = []
+    with file_name.open('w') as f:
         try:
-            manifest_download_list.append((repo_data['engines'], 'engine.json'))
-        except KeyError:
-            pass
+            time_now = datetime.now()
+            # Convert to lower case because AM/PM is capitalized
+            time_str = time_now.strftime('%d/%m/%Y %I:%M%p').lower()
+            repo_data.update({'last_updated': time_str})
+            f.write(json.dumps(repo_data, indent=4) + '\n')
+        except Exception as e:
+            logger.error(f'{file_name} failed to save: {str(e)}')
+            return 1
+
+    # A repo may not contain all types of object.
+    manifest_download_list = []
+    try:
+        manifest_download_list.append((repo_data['engines'], 'engine.json'))
+    except KeyError:
+        pass
+    try:
+        manifest_download_list.append((repo_data['projects'], 'project.json'))
+    except KeyError:
+        pass
+    try:
+        manifest_download_list.append((repo_data['gems'], 'gem.json'))
+    except KeyError:
+        pass
+    try:
+        manifest_download_list.append((repo_data['templates'], 'template.json'))
+    except KeyError:
+        pass
+    try:
+        manifest_download_list.append((repo_data['restricted'], 'restricted.json'))
+    except KeyError:
+        pass
+
+    for o3de_object_uris, manifest_json in manifest_download_list:
+        for o3de_object_uri in o3de_object_uris:
+            manifest_json_uri = f'{o3de_object_uri}/{manifest_json}'
+            manifest_json_sha256 = hashlib.sha256(manifest_json_uri.encode())
+            cache_file = cache_folder / str(manifest_json_sha256.hexdigest() + '.json')
+            if not cache_file.is_file():
+                parsed_uri = urllib.parse.urlparse(manifest_json_uri)
+                download_file_result = utils.download_file(parsed_uri, cache_file)
+                if download_file_result != 0:
+                    return download_file_result
+
+    # Having a repo is also optional
+    repo_list = []
+    try:
+        repo_list.append(repo_data['repos'])
+    except KeyError:
+        pass
+
+    for repo in repo_list:
+        if repo not in repo_set:
+            repo_set.add(repo)
+            for o3de_object_uri in o3de_object_uris:
+                parsed_uri = urllib.parse.urlparse(f'{repo}/repo.json')
+                manifest_json_sha256 = hashlib.sha256(parsed_uri.geturl().encode())
+                cache_file = cache_folder / str(manifest_json_sha256.hexdigest() + '.json')
+                if cache_file.is_file():
+                    cache_file.unlink()
+                download_file_result = utils.download_file(parsed_uri, cache_file)
+                if download_file_result != 0:
+                    return download_file_result
+
+                return process_add_o3de_repo(parsed_uri.geturl(), repo_set)
+    return 0
+
+
+def get_gem_json_paths_from_cached_repo(repo_uri: str) -> set:
+    url = f'{repo_uri}/repo.json'
+    repo_sha256 = hashlib.sha256(url.encode())
+    cache_folder = manifest.get_o3de_cache_folder()
+    cache_filename = cache_folder / str(repo_sha256.hexdigest() + '.json')
+
+    gem_set = set()
+
+    file_name = pathlib.Path(cache_filename).resolve()
+    if not file_name.is_file():
+        logger.error(f'Could not find cached repo json file for {repo_uri}')
+        return gem_list
+
+    with file_name.open('r') as f:
         try:
-            manifest_download_list.append((repo_data['projects'], 'project.json'))
-        except KeyError:
-            pass
+            repo_data = json.load(f)
+        except json.JSONDecodeError as e:
+            logger.error(f'{file_name} failed to load: {str(e)}')
+            return gem_list
+
+        # Get list of gems, then add all json paths to the list if they exist in the cache
+        repo_gems = []
         try:
-            manifest_download_list.append((repo_data['gems'], 'gem.json'))
-        except KeyError:
-            pass
-        try:
-            manifest_download_list.append((repo_data['templates'], 'template.json'))
-        except KeyError:
-            pass
-        try:
-            manifest_download_list.append((repo_data['restricted'], 'restricted.json'))
+            repo_gems.append((repo_data['gems'], 'gem.json'))
         except KeyError:
             pass
 
-        for o3de_object_uris, manifest_json in manifest_download_list:
+        for o3de_object_uris, manifest_json in repo_gems:
             for o3de_object_uri in o3de_object_uris:
                 manifest_json_uri = f'{o3de_object_uri}/{manifest_json}'
                 manifest_json_sha256 = hashlib.sha256(manifest_json_uri.encode())
-                cache_file = cache_folder / str(manifest_json_sha256.hexdigest() + '.json')
-                if not cache_file.is_file():
-                    parsed_uri = urllib.parse.urlparse(manifest_json_uri)
-                    download_file_result = utils.download_file(parsed_uri, cache_file)
-                    if download_file_result != 0:
-                        return download_file_result
+                cache_gem_json_filepath = cache_folder / str(manifest_json_sha256.hexdigest() + '.json')
+                if cache_gem_json_filepath.is_file():
+                    gem_set.add(cache_gem_json_filepath)
+                else:
+                    logger.warn(f'Could not find cached gem json file {cache_gem_json_filepath} for {o3de_object_uri} in repo {repo_uri}')
 
-        # Having a repo is also optional
-        repo_list = []
-        try:
-            repo_list.append(repo_data['repos'])
-        except KeyError:
-            pass
+    return gem_set
 
-        for repo in repo_list:
-            if repo not in repo_set:
-                repo_set.add(repo)
-                for o3de_object_uri in o3de_object_uris:
-                    parsed_uri = urllib.parse.urlparse(f'{repo}/repo.json')
-                    manifest_json_sha256 = hashlib.sha256(parsed_uri.geturl().encode())
-                    cache_file = cache_folder / str(manifest_json_sha256.hexdigest() + '.json')
-                    if cache_file.is_file():
-                        cache_file.unlink()
-                    download_file_result = utils.download_file(parsed_uri, cache_file)
-                    if download_file_result != 0:
-                        return download_file_result
+def get_gem_json_paths_from_all_cached_repos() -> set:
+    json_data = manifest.load_o3de_manifest()
+    gem_set = set()
 
-                    return process_add_o3de_repo(parsed_uri.geturl(), repo_set)
-    return 0
+    for repo_uri in json_data['repos']:
+        gem_set.update(get_gem_json_paths_from_cached_repo(repo_uri)) 
 
+    return gem_set
+    
+
+def refresh_repo(repo_uri: str,
+                 cache_folder: str = None,
+                 repo_set: set = None) -> int:
+    if not cache_folder:
+        cache_folder = manifest.get_o3de_cache_folder()
+    if not repo_set:
+        repo_set = set()
+
+    parsed_uri = urllib.parse.urlparse(f'{repo_uri}/repo.json')
+    repo_sha256 = hashlib.sha256(parsed_uri.geturl().encode())
+    cache_file = cache_folder / str(repo_sha256.hexdigest() + '.json')
+
+    download_file_result = utils.download_file(parsed_uri, cache_file)
+    if download_file_result != 0:
+        return download_file_result
+
+    if not validation.valid_o3de_repo_json(cache_file):
+        logger.error(f'Repo json {repo_uri} is not valid.')
+        cache_file.unlink()
+        return 1
+
+    return process_add_o3de_repo(cache_file, repo_set)
 
 def refresh_repos() -> int:
     json_data = manifest.load_o3de_manifest()
@@ -109,22 +193,9 @@ def refresh_repos() -> int:
         if repo_uri not in repo_set:
             repo_set.add(repo_uri)
 
-            parsed_uri = urllib.parse.urlparse(f'{repo_uri}/repo.json')
-            repo_sha256 = hashlib.sha256(parsed_uri.geturl().encode())
-            cache_file = cache_folder / str(repo_sha256.hexdigest() + '.json')
-            if not cache_file.is_file():
-                download_file_result = utils.download_file(parsed_uri, cache_file)
-                if download_file_result != 0:
-                    return download_file_result
-
-                if not validation.valid_o3de_repo_json(cache_file):
-                    logger.error(f'Repo json {repo_uri} is not valid.')
-                    cache_file.unlink()
-                    return 1
-
-                last_failure = process_add_o3de_repo(cache_file, repo_set)
-                if last_failure:
-                    result = last_failure
+            last_failure = refresh_repo(repo_uri, cache_folder, repo_set)
+            if last_failure:
+                result = last_failure
 
     return result
 

scripts/o3de/o3de/utils.py

@@ -10,6 +10,7 @@ This file contains utility functions
 """
 import sys
 import uuid
+import os
 import pathlib
 import shutil
 import urllib.request
@@ -19,6 +20,29 @@ import zipfile
 logger = logging.getLogger()
 logging.basicConfig()
 
+COPY_BUFSIZE = 64 * 1024
+
+def copyfileobj(fsrc, fdst, callback, length=0):
+    # This is functionally the same as the python shutil copyfileobj but
+    # allows for a callback to return the download progress in blocks and allows
+    # to early out to cancel the copy.
+    if not length:
+        length = COPY_BUFSIZE
+
+    fsrc_read = fsrc.read
+    fdst_write = fdst.write
+
+    copied = 0
+    while True:
+        buf = fsrc_read(length)
+        if not buf:
+            break
+        fdst_write(buf)
+        copied += len(buf)
+        if callback(copied):
+            return 1
+    return 0
+
 def validate_identifier(identifier: str) -> bool:
     """
     Determine if the identifier supplied is valid.
@@ -93,18 +117,29 @@ def backup_folder(folder: str or pathlib.Path) -> None:
             if backup_folder_name.is_dir():
                 renamed = True
 
-
-def download_file(parsed_uri, download_path: pathlib.Path) -> int:
+def download_file(parsed_uri, download_path: pathlib.Path, download_progress_callback = None) -> int:
     """
     :param parsed_uri: uniform resource identifier to zip file to download
     :param download_path: location path on disk to download file
+    :download_progress_callback: callback called with the download progress as a percentage, returns true to request to cancel the download
     """
     if download_path.is_file():
         logger.warn(f'File already downloaded to {download_path}.')
     elif parsed_uri.scheme in ['http', 'https', 'ftp', 'ftps']:
         with urllib.request.urlopen(parsed_uri.geturl()) as s:
+            download_file_size = 0
+            try:
+                download_file_size = s.headers['content-length']
+            except KeyError:
+                pass
+            def download_progress(blocks):
+                if download_progress_callback and download_file_size:
+                    return download_progress_callback(int(blocks/int(download_file_size) * 100))
+                return False
             with download_path.open('wb') as f:
-                shutil.copyfileobj(s, f)
+                download_cancelled = copyfileobj(s, f, download_progress)
+                if download_cancelled:
+                    return 1
     else:
         origin_file = pathlib.Path(parsed_uri.geturl()).resolve()
         if not origin_file.is_file():
@@ -114,12 +149,12 @@ def download_file(parsed_uri, download_path: pathlib.Path) -> int:
     return 0
 
 
-def download_zip_file(parsed_uri, download_zip_path: pathlib.Path) -> int:
+def download_zip_file(parsed_uri, download_zip_path: pathlib.Path, download_progress_callback = None) -> int:
     """
     :param parsed_uri: uniform resource identifier to zip file to download
     :param download_zip_path: path to output zip file
     """
-    download_file_result = download_file(parsed_uri, download_zip_path)
+    download_file_result = download_file(parsed_uri, download_zip_path, download_progress_callback)
     if download_file_result != 0:
         return download_file_result