Merge pull request #4486 from aws-lumberyard-dev/Prism/ShowRepoList

repo.jsons can be listed and and access through CLI and is Integrated with Project Manager

Code/Tools/ProjectManager/Source/GemRepo/GemRepoInfo.h

@@ -30,7 +30,7 @@ namespace O3DE::ProjectManager
         bool operator<(const GemRepoInfo& gemRepoInfo) const;
 
         QString m_path = "";
-        QString m_name = "Unknown Gem Repo Name";
+        QString m_name = "Unknown Repo Name";
         QString m_creator = "Unknown Creator";
         bool m_isEnabled = false; //! Is the repo currently enabled for this engine?
         QString m_summary = "No summary provided.";

Code/Tools/ProjectManager/Source/GemRepo/GemRepoItemDelegate.cpp

@@ -8,6 +8,7 @@
 
 #include <GemRepo/GemRepoItemDelegate.h>
 #include <GemRepo/GemRepoModel.h>
+#include <ProjectManagerDefs.h>
 
 #include <QEvent>
 #include <QPainter>
@@ -95,7 +96,7 @@ namespace O3DE::ProjectManager
         painter->drawText(repoCreatorRect, Qt::TextSingleLine, repoCreator);
 
         // Repo update
-        QString repoUpdatedDate = GemRepoModel::GetLastUpdated(modelIndex).toString("dd/MM/yyyy hh:mmap");
+        QString repoUpdatedDate = GemRepoModel::GetLastUpdated(modelIndex).toString(RepoTimeFormat);
         repoUpdatedDate = standardFontMetrics.elidedText(repoUpdatedDate, Qt::TextElideMode::ElideRight, s_updatedMaxWidth);
 
         QRect repoUpdatedDateRect = GetTextRect(standardFont, repoUpdatedDate, s_fontSize);

Code/Tools/ProjectManager/Source/GemRepo/GemRepoScreen.cpp

@@ -13,6 +13,7 @@
 #include <GemRepo/GemRepoAddDialog.h>
 #include <GemRepo/GemRepoInspector.h>
 #include <PythonBindingsInterface.h>
+#include <ProjectManagerDefs.h>
 
 #include <QVBoxLayout>
 #include <QHBoxLayout>
@@ -104,9 +105,29 @@ namespace O3DE::ProjectManager
         {
             // Add all available repos to the model
             const QVector<GemRepoInfo> allGemRepoInfos = allGemRepoInfosResult.GetValue();
+            QDateTime oldestRepoUpdate;
+            if (!allGemRepoInfos.isEmpty())
+            {
+                oldestRepoUpdate = allGemRepoInfos[0].m_lastUpdated;
+            }
             for (const GemRepoInfo& gemRepoInfo : allGemRepoInfos)
             {
                 m_gemRepoModel->AddGemRepo(gemRepoInfo);
+
+                // Find least recently updated repo
+                if (gemRepoInfo.m_lastUpdated < oldestRepoUpdate)
+                {
+                    oldestRepoUpdate = gemRepoInfo.m_lastUpdated;
+                }
+            }
+
+            if (!allGemRepoInfos.isEmpty())
+            {
+                m_lastAllUpdateLabel->setText(tr("Last Updated: %1").arg(oldestRepoUpdate.toString(RepoTimeFormat)));
+            }
+            else
+            {
+                m_lastAllUpdateLabel->setText(tr("Last Updated: Never"));
             }
         }
         else

Code/Tools/ProjectManager/Source/ProjectManagerDefs.h

@@ -27,4 +27,6 @@ namespace O3DE::ProjectManager
     static const QString ProjectCMakeCommand = "cmake";
     static const QString ProjectCMakeBuildTargetEditor = "Editor";
 
+    static const QString RepoTimeFormat = "dd/MM/yyyy hh:mmap";
+
 } // namespace O3DE::ProjectManager

Code/Tools/ProjectManager/Source/PythonBindings.cpp

@@ -932,13 +932,55 @@ namespace O3DE::ProjectManager
         return AZ::Failure<AZStd::string>("Adding Gem Repo not implemented yet in o3de scripts.");
     }
 
-    GemRepoInfo PythonBindings::GemRepoInfoFromPath(pybind11::handle path, pybind11::handle pyEnginePath)
+    GemRepoInfo PythonBindings::GetGemRepoInfo(pybind11::handle repoUri)
     {
-        /* Placeholder Logic */
+        GemRepoInfo gemRepoInfo;
-        (void)path;
+        gemRepoInfo.m_repoLink = Py_To_String(repoUri);
-        (void)pyEnginePath;
 
-        return GemRepoInfo();
+        auto data = m_manifest.attr("get_repo_json_data")(repoUri);
+        if (pybind11::isinstance<pybind11::dict>(data))
+        {
+            try
+            {
+                // required
+                gemRepoInfo.m_repoLink = Py_To_String(data["repo_uri"]);
+                gemRepoInfo.m_name = Py_To_String(data["repo_name"]);
+                gemRepoInfo.m_creator = Py_To_String(data["origin"]);
+
+                // optional
+                gemRepoInfo.m_summary = Py_To_String_Optional(data, "summary", "No summary provided.");
+                gemRepoInfo.m_additionalInfo = Py_To_String_Optional(data, "additional_info", "");
+
+                auto repoPath = m_manifest.attr("get_repo_path")(repoUri);
+                gemRepoInfo.m_path = gemRepoInfo.m_directoryLink = Py_To_String(repoPath);
+
+                QString lastUpdated = Py_To_String_Optional(data, "last_updated", "");
+                gemRepoInfo.m_lastUpdated = QDateTime::fromString(lastUpdated, RepoTimeFormat);
+
+                if (data.contains("enabled"))
+                {
+                    gemRepoInfo.m_isEnabled = data["enabled"].cast<bool>();
+                }
+                else
+                {
+                    gemRepoInfo.m_isEnabled = false;
+                }
+
+                if (data.contains("gem_paths"))
+                {
+                    for (auto gemPath : data["gem_paths"])
+                    {
+                        gemRepoInfo.m_includedGemPaths.push_back(Py_To_String(gemPath));
+                    }
+                }
+            }
+            catch ([[maybe_unused]] const std::exception& e)
+            {
+                AZ_Warning("PythonBindings", false, "Failed to get GemRepoInfo for repo %s", Py_To_String(repoUri));
+            }
+        }
+
+        return gemRepoInfo;
     }
 
 //#define MOCK_GEM_REPO_INFO true
@@ -951,14 +993,10 @@ namespace O3DE::ProjectManager
         auto result = ExecuteWithLockErrorHandling(
             [&]
             {
-                /* Placeholder Logic, o3de scripts need method added
+                for (auto repoUri : m_manifest.attr("get_repos")())
-                * 
-                for (auto path : m_manifest.attr("get_gem_repos")())
                 {
-                    gemRepos.push_back(GemRepoInfoFromPath(path, pybind11::none()));
+                    gemRepos.push_back(GetGemRepoInfo(repoUri));
                 }
-                *
-                */
             });
         if (!result.IsSuccess())
         {

Code/Tools/ProjectManager/Source/PythonBindings.h

@@ -67,7 +67,7 @@ namespace O3DE::ProjectManager
         AZ::Outcome<void, AZStd::string> ExecuteWithLockErrorHandling(AZStd::function<void()> executionCallback);
         bool ExecuteWithLock(AZStd::function<void()> executionCallback);
         GemInfo GemInfoFromPath(pybind11::handle path, pybind11::handle pyProjectPath);
-        GemRepoInfo GemRepoInfoFromPath(pybind11::handle path, pybind11::handle pyEnginePath);
+        GemRepoInfo GetGemRepoInfo(pybind11::handle repoUri);
         ProjectInfo ProjectInfoFromPath(pybind11::handle path);
         ProjectTemplateInfo ProjectTemplateInfoFromPath(pybind11::handle path, pybind11::handle pyProjectPath);
         bool RegisterThisEngine();

scripts/o3de/o3de/manifest.py

@@ -13,8 +13,10 @@ import json
 import logging
 import os
 import pathlib
+import shutil
+import hashlib
 
-from o3de import validation
+from o3de import validation, utils
 
 logger = logging.getLogger()
 logging.basicConfig()
@@ -135,12 +137,12 @@ def get_o3de_manifest() -> pathlib.Path:
         json_data.update({'default_restricted_folder': default_restricted_folder.as_posix()})
         json_data.update({'default_third_party_folder': default_third_party_folder.as_posix()})
 
+        json_data.update({'engines': []})
         json_data.update({'projects': []})
         json_data.update({'external_subdirectories': []})
         json_data.update({'templates': []})
         json_data.update({'restricted': []})
         json_data.update({'repos': []})
-        json_data.update({'engines': []})
 
         default_restricted_folder_json = default_restricted_folder / 'restricted.json'
         if not default_restricted_folder_json.is_file():
@@ -197,7 +199,7 @@ def load_o3de_manifest(manifest_path: pathlib.Path = None) -> dict:
 
 def save_o3de_manifest(json_data: dict, manifest_path: pathlib.Path = None) -> bool:
     """
-        Save the json dictionary to the supplied manifest file or ~/.o3de/o3de_manifest.json if manifest_path is None
+    Save the json dictionary to the supplied manifest file or ~/.o3de/o3de_manifest.json if None
 
     :param json_data: dictionary to save in json format at the file path
     :param manifest_path: optional path to manifest file to save
@@ -213,7 +215,6 @@ def save_o3de_manifest(json_data: dict, manifest_path: pathlib.Path = None) -> b
             return False
 
 
-
 def get_gems_from_subdirectories(external_subdirs: list) -> list:
     '''
     Helper Method for scanning a set of external subdirectories for gem.json files
@@ -235,7 +236,6 @@ def get_gems_from_subdirectories(external_subdirs: list) -> list:
     return gem_directories
 
 
-# Data query methods
 def get_engines() -> list:
     json_data = load_o3de_manifest()
     engine_list = json_data['engines'] if 'engines' in json_data else []
@@ -421,39 +421,60 @@ def get_templates_for_generic_creation():  # temporary until we have a better wa
 
     return list(filter(filter_project_and_gem_templates_out, get_all_templates()))
 
-
+def get_json_file_path(object_typename: str,
-def get_engine_json_data(engine_name: str = None,
+                       object_path: str or pathlib.Path) -> pathlib.Path:
-                         engine_path: str or pathlib.Path = None) -> dict or None:
+    if not object_typename or not object_path:
-    if not engine_name and not engine_path:
+        logger.error('Must specify an object typename and object path.')
-        logger.error('Must specify either a Engine name or Engine Path.')
         return None
 
-    if engine_name and not engine_path:
+    object_path = pathlib.Path(object_path).resolve()
-        engine_path = get_registered(engine_name=engine_name)
+    return object_path / f'{object_typename}.json'
+
 
-    if not engine_path:
+def get_json_data_file(object_json: pathlib.Path,
-        logger.error(f'Engine Path {engine_path} has not been registered.')
+                       object_typename: str,
+                       object_validator: callable) -> dict or None:
+    if not object_typename:
+        logger.error('Missing object typename.')
         return None
 
-    engine_path = pathlib.Path(engine_path).resolve()
+    if not object_json or not object_json.is_file():
-    engine_json = engine_path / 'engine.json'
+        logger.error(f'Invalid {object_typename} json {object_json} supplied or file missing.')
-    if not engine_json.is_file():
-        logger.error(f'Engine json {engine_json} is not present.')
         return None
-    if not validation.valid_o3de_engine_json(engine_json):
+
-        logger.error(f'Engine json {engine_json} is not valid.')
+    if not object_validator or not object_validator(object_json):
+        logger.error(f'{object_typename} json {object_json} is not valid or could not be validated.')
         return None
 
-    with engine_json.open('r') as f:
+    with object_json.open('r') as f:
         try:
-            engine_json_data = json.load(f)
+            object_json_data = json.load(f)
         except json.JSONDecodeError as e:
-            logger.warn(f'{engine_json} failed to load: {str(e)}')
+            logger.warn(f'{object_json} failed to load: {e}')
         else:
-            return engine_json_data
+            return object_json_data
 
     return None
 
+def get_json_data(object_typename: str,
+                  object_path: str or pathlib.Path,
+                  object_validator: callable) -> dict or None:
+    object_json = get_json_file_path(object_typename, object_path)
+
+    return get_json_data_file(object_json, object_typename, object_validator)
+
+
+def get_engine_json_data(engine_name: str = None,
+                         engine_path: str or pathlib.Path = None) -> dict or None:
+    if not engine_name and not engine_path:
+        logger.error('Must specify either a Engine name or Engine Path.')
+        return None
+
+    if engine_name and not engine_path:
+        engine_path = get_registered(engine_name=engine_name)
+
+    return get_json_data('engine', engine_path, validation.valid_o3de_engine_json)
+
 
 def get_project_json_data(project_name: str = None,
                           project_path: str or pathlib.Path = None) -> dict or None:
@@ -464,28 +485,7 @@ def get_project_json_data(project_name: str = None,
     if project_name and not project_path:
         project_path = get_registered(project_name=project_name)
 
-    if not project_path:
+    return get_json_data('project', project_path, validation.valid_o3de_project_json)
-        logger.error(f'Project Path {project_path} has not been registered.')
-        return None
-
-    project_path = pathlib.Path(project_path).resolve()
-    project_json = project_path / 'project.json'
-    if not project_json.is_file():
-        logger.error(f'Project json {project_json} is not present.')
-        return None
-    if not validation.valid_o3de_project_json(project_json):
-        logger.error(f'Project json {project_json} is not valid.')
-        return None
-
-    with project_json.open('r') as f:
-        try:
-            project_json_data = json.load(f)
-        except json.JSONDecodeError as e:
-            logger.warn(f'{project_json} failed to load: {str(e)}')
-        else:
-            return project_json_data
-
-    return None
 
 
 def get_gem_json_data(gem_name: str = None, gem_path: str or pathlib.Path = None,
@@ -497,28 +497,7 @@ def get_gem_json_data(gem_name: str = None, gem_path: str or pathlib.Path = None
     if gem_name and not gem_path:
         gem_path = get_registered(gem_name=gem_name, project_path=project_path)
 
-    if not gem_path:
+    return get_json_data('gem', gem_path, validation.valid_o3de_gem_json)
-        logger.error(f'Gem Path {gem_path} has not been registered.')
-        return None
-
-    gem_path = pathlib.Path(gem_path).resolve()
-    gem_json = gem_path / 'gem.json'
-    if not gem_json.is_file():
-        logger.error(f'Gem json {gem_json} is not present.')
-        return None
-    if not validation.valid_o3de_gem_json(gem_json):
-        logger.error(f'Gem json {gem_json} is not valid.')
-        return None
-
-    with gem_json.open('r') as f:
-        try:
-            gem_json_data = json.load(f)
-        except json.JSONDecodeError as e:
-            logger.warn(f'{gem_json} failed to load: {str(e)}')
-        else:
-            return gem_json_data
-
-    return None
 
 
 def get_template_json_data(template_name: str = None, template_path: str or pathlib.Path = None,
@@ -530,28 +509,7 @@ def get_template_json_data(template_name: str = None, template_path: str or path
     if template_name and not template_path:
         template_path = get_registered(template_name=template_name, project_path=project_path)
 
-    if not template_path:
+    return get_json_data('template', template_path, validation.valid_o3de_template_json)
-        logger.error(f'Template Path {template_path} has not been registered.')
-        return None
-
-    template_path = pathlib.Path(template_path).resolve()
-    template_json = template_path / 'template.json'
-    if not template_json.is_file():
-        logger.error(f'Template json {template_json} is not present.')
-        return None
-    if not validation.valid_o3de_template_json(template_json):
-        logger.error(f'Template json {template_json} is not valid.')
-        return None
-
-    with template_json.open('r') as f:
-        try:
-            template_json_data = json.load(f)
-        except json.JSONDecodeError as e:
-            logger.warn(f'{template_json} failed to load: {str(e)}')
-        else:
-            return template_json_data
-
-    return None
 
 
 def get_restricted_json_data(restricted_name: str = None, restricted_path: str or pathlib.Path = None,
@@ -563,29 +521,23 @@ def get_restricted_json_data(restricted_name: str = None, restricted_path: str o
     if restricted_name and not restricted_path:
         restricted_path = get_registered(restricted_name=restricted_name, project_path=project_path)
 
-    if not restricted_path:
+    return get_json_data('restricted', restricted_path, validation.valid_o3de_restricted_json)
-        logger.error(f'Restricted Path {restricted_path} has not been registered.')
-        return None
 
-    restricted_path = pathlib.Path(restricted_path).resolve()
+def get_repo_json_data(repo_uri: str) -> dict or None:
-    restricted_json = restricted_path / 'restricted.json'
+    if not repo_uri:
-    if not restricted_json.is_file():
+        logger.error('Must specify a Repo Uri.')
-        logger.error(f'Restricted json {restricted_json} is not present.')
-        return None
-    if not validation.valid_o3de_restricted_json(restricted_json):
-        logger.error(f'Restricted json {restricted_json} is not valid.')
         return None
 
-    with restricted_json.open('r') as f:
+    repo_json = get_repo_path(repo_uri=repo_uri)
-        try:
-            restricted_json_data = json.load(f)
-        except json.JSONDecodeError as e:
-            logger.warn(f'{restricted_json} failed to load: {str(e)}')
-        else:
-            return restricted_json_data
 
-    return None
+    return get_json_data_file(repo_json, "Repo", validation.valid_o3de_repo_json)
 
+def get_repo_path(repo_uri: str, cache_folder: str = None) -> pathlib.Path:
+    if not cache_folder:
+        cache_folder = get_o3de_cache_folder()
+
+    repo_sha256 = hashlib.sha256(repo_uri.encode())
+    return cache_folder / str(repo_sha256.hexdigest() + '.json')
 
 def get_registered(engine_name: str = None,
                    project_name: str = None,
@@ -721,9 +673,7 @@ def get_registered(engine_name: str = None,
     elif isinstance(repo_name, str):
         cache_folder = get_o3de_cache_folder()
         for repo_uri in json_data['repos']:
-            repo_uri = pathlib.Path(repo_uri).resolve()
+            cache_file = get_repo_path(repo_uri=repo_uri, cache_folder=cache_folder)
-            repo_sha256 = hashlib.sha256(repo_uri.encode())
-            cache_file = cache_folder / str(repo_sha256.hexdigest() + '.json')
             if cache_file.is_file():
                 repo = pathlib.Path(cache_file).resolve()
                 with repo.open('r') as f:

scripts/o3de/o3de/validation.py

@@ -27,7 +27,6 @@ def valid_o3de_repo_json(file_name: str or pathlib.Path) -> bool:
             test = json_data['origin']
         except (json.JSONDecodeError, KeyError) as e:
             return False
-
     return True