birkeh/o3de
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Merge pull request #4892 from aws-lumberyard-dev/TrackDownloadProgress

Browse Source 
Project Manager track progress of, and cancel downloads
This commit is contained in:
AMZN-Phil
2021-10-22 15:53:29 -07:00
committed by GitHub
parent 45b164e722 e1b46b7e5c
commit 69771ab2e1
9 changed files with 122 additions and 28 deletions
Download Patch File Download Diff File Expand all files Collapse all files
Code/Tools/ProjectManager/Source/DownloadController.cpp
+21 -7
View File
@@ -8,10 +8,12 @@
#include <DownloadController.h>
#include <DownloadWorker.h>
#include <PythonBindings.h>
#include <AzCore/std/algorithm.h>
#include <QMessageBox>
namespace O3DE::ProjectManager
{
DownloadController::DownloadController(QWidget* parent)
@@ -46,6 +48,24 @@ namespace O3DE::ProjectManager
}
}
void DownloadController::CancelGemDownload(const QString& gemName)
{
auto findResult = AZStd::find(m_gemNames.begin(), m_gemNames.end(), gemName);
if (findResult != m_gemNames.end())
{
if (findResult == m_gemNames.begin())
{
// HandleResults will remove the gem upon cancelling
PythonBindingsInterface::Get()->CancelDownload();
}
else
{
m_gemNames.erase(findResult);
}
}
}
void DownloadController::UpdateUIProgress(int progress)
{
m_lastProgress = progress;
@@ -75,10 +95,4 @@ namespace O3DE::ProjectManager
m_workerThread.wait();
}
}
void DownloadController::HandleCancel()
{
m_workerThread.quit();
emit Done(false);
}
} // namespace O3DE::ProjectManager
Code/Tools/ProjectManager/Source/DownloadController.h
+2 -2
View File
@@ -27,7 +27,8 @@ namespace O3DE::ProjectManager
explicit DownloadController(QWidget* parent = nullptr);
~DownloadController();
void AddGemDownload(const QString& m_gemName);
void AddGemDownload(const QString& gemName);
void CancelGemDownload(const QString& gemName);
bool IsDownloadQueueEmpty()
{
@@ -54,7 +55,6 @@ namespace O3DE::ProjectManager
public slots:
void UpdateUIProgress(int progress);
void HandleResults(const QString& result);
void HandleCancel();
signals:
void StartGemDownload(const QString& gemName);
Code/Tools/ProjectManager/Source/GemCatalog/GemCatalogHeaderWidget.cpp
+8 -1
View File
@@ -155,6 +155,11 @@ namespace O3DE::ProjectManager
update();
}
void CartOverlayWidget::OnCancelDownloadActivated(const QString& gemName)
{
m_downloadController->CancelGemDownload(gemName);
}
void CartOverlayWidget::CreateDownloadSection()
{
QWidget* widget = new QWidget();
@@ -235,7 +240,9 @@ namespace O3DE::ProjectManager
nameProgressLayout->addWidget(progress);
QSpacerItem* spacer = new QSpacerItem(0, 0, QSizePolicy::Expanding, QSizePolicy::Minimum);
nameProgressLayout->addSpacerItem(spacer);
QLabel* cancelText = new QLabel(tr("Cancel"));
QLabel* cancelText = new QLabel(QString("<a href=\"%1\">Cancel</a>").arg(downloadQueue[downloadingGemNumber]));
cancelText->setTextInteractionFlags(Qt::LinksAccessibleByMouse);
connect(cancelText, &QLabel::linkActivated, this, &CartOverlayWidget::OnCancelDownloadActivated);
nameProgressLayout->addWidget(cancelText);
downloadingItemLayout->addLayout(nameProgressLayout);
QProgressBar* downloadProgessBar = new QProgressBar();
Code/Tools/ProjectManager/Source/GemCatalog/GemCatalogHeaderWidget.h
+1
View File
@@ -41,6 +41,7 @@ namespace O3DE::ProjectManager
using GetTagIndicesCallback = AZStd::function<QVector<QModelIndex>()>;
void CreateGemSection(const QString& singularTitle, const QString& pluralTitle, GetTagIndicesCallback getTagIndices);
void CreateDownloadSection();
void OnCancelDownloadActivated(const QString& link);
QVBoxLayout* m_layout = nullptr;
GemModel* m_gemModel = nullptr;
Code/Tools/ProjectManager/Source/PythonBindings.cpp
+18 -1
View File
@@ -223,6 +223,7 @@ namespace RedirectOutput
}
} // namespace RedirectOutput
namespace O3DE::ProjectManager
{
PythonBindings::PythonBindings(const AZ::IO::PathView& enginePath)
@@ -1120,18 +1121,29 @@ namespace O3DE::ProjectManager
AZ::Outcome<void, AZStd::string> PythonBindings::DownloadGem(const QString& gemName, std::function<void(int)> gemProgressCallback)
{
// This process is currently limited to download a single gem at a time.
bool downloadSucceeded = false;
m_requestCancelDownload = false;
auto result = ExecuteWithLockErrorHandling(
[&]
{
auto downloadResult = m_download.attr("download_gem")(
QString_To_Py_String(gemName), // gem name
pybind11::none(), // destination path
false// skip auto register
false, // skip auto register
pybind11::cpp_function(
[this, gemProgressCallback](int progress)
{
gemProgressCallback(progress);
return m_requestCancelDownload;
}) // Callback for download progress and cancelling
);
downloadSucceeded = (downloadResult.cast<int>() == 0);
});
if (!result.IsSuccess())
{
return result;
@@ -1143,4 +1155,9 @@ namespace O3DE::ProjectManager
return AZ::Success();
}
void PythonBindings::CancelDownload()
{
m_requestCancelDownload = true;
}
}
Code/Tools/ProjectManager/Source/PythonBindings.h
+3
View File
@@ -63,6 +63,7 @@ namespace O3DE::ProjectManager
bool RemoveGemRepo(const QString& repoUri) override;
AZ::Outcome<QVector<GemRepoInfo>, AZStd::string> GetAllGemRepoInfos() override;
AZ::Outcome<void, AZStd::string> DownloadGem(const QString& gemName, std::function<void(int)> gemProgressCallback) override;
void CancelDownload() override;
private:
AZ_DISABLE_COPY_MOVE(PythonBindings);
@@ -91,5 +92,7 @@ namespace O3DE::ProjectManager
pybind11::handle m_editProjectProperties;
pybind11::handle m_download;
pybind11::handle m_pathlib;
bool m_requestCancelDownload = false;
};
}
Code/Tools/ProjectManager/Source/PythonBindingsInterface.h
+11
View File
@@ -196,7 +196,18 @@ namespace O3DE::ProjectManager
*/
virtual AZ::Outcome<QVector<GemRepoInfo>, AZStd::string> GetAllGemRepoInfos() = 0;
/**
* Downloads and registers a Gem.
* @param gemName the name of the Gem to download
* @param gemProgressCallback a callback function that is called with an int percentage download value
* @return an outcome with a string error message on failure.
*/
virtual AZ::Outcome<void, AZStd::string> DownloadGem(const QString& gemName, std::function<void(int)> gemProgressCallback) = 0;
/**
* Cancels the current download.
*/
virtual void CancelDownload() = 0;
};
using PythonBindingsInterface = AZ::Interface<IPythonBindings>;
scripts/o3de/o3de/download.py
+18 -12
View File
@@ -90,7 +90,8 @@ def get_downloadable(engine_name: str = None,
def download_o3de_object(object_name: str, default_folder_name: str, dest_path: str or pathlib.Path,
object_type: str, downloadable_kwarg_key, skip_auto_register: bool) -> int:
object_type: str, downloadable_kwarg_key, skip_auto_register: bool,
download_progress_callback = None) -> int:
download_path = manifest.get_o3de_cache_folder() / default_folder_name / object_name
download_path.mkdir(parents=True, exist_ok=True)
@@ -104,7 +105,7 @@ def download_o3de_object(object_name: str, default_folder_name: str, dest_path:
origin_uri = downloadable_object_data['originuri']
parsed_uri = urllib.parse.urlparse(origin_uri)
download_zip_result = utils.download_zip_file(parsed_uri, download_zip_path)
download_zip_result = utils.download_zip_file(parsed_uri, download_zip_path, download_progress_callback)
if download_zip_result != 0:
return download_zip_result
@@ -147,33 +148,38 @@ def download_o3de_object(object_name: str, default_folder_name: str, dest_path:
def download_engine(engine_name: str,
dest_path: str or pathlib.Path,
skip_auto_register: bool) -> int:
return download_o3de_object(engine_name, 'engines', dest_path, 'engine', 'engine_name', skip_auto_register)
skip_auto_register: bool,
download_progress_callback = None) -> int:
return download_o3de_object(engine_name, 'engines', dest_path, 'engine', 'engine_name', skip_auto_register, download_progress_callback)
def download_project(project_name: str,
dest_path: str or pathlib.Path,
skip_auto_register: bool) -> int:
return download_o3de_object(project_name, 'projects', dest_path, 'project', 'project_name', skip_auto_register)
skip_auto_register: bool,
download_progress_callback = None) -> int:
return download_o3de_object(project_name, 'projects', dest_path, 'project', 'project_name', skip_auto_register, download_progress_callback)
def download_gem(gem_name: str,
dest_path: str or pathlib.Path,
skip_auto_register: bool) -> int:
return download_o3de_object(gem_name, 'gems', dest_path, 'gem', 'gem_name', skip_auto_register)
skip_auto_register: bool,
download_progress_callback = None) -> int:
return download_o3de_object(gem_name, 'gems', dest_path, 'gem', 'gem_name', skip_auto_register, download_progress_callback)
def download_template(template_name: str,
dest_path: str or pathlib.Path,
skip_auto_register: bool) -> int:
return download_o3de_object(template_name, 'templates', dest_path, 'template', 'template_name', skip_auto_register)
skip_auto_register: bool,
download_progress_callback = None) -> int:
return download_o3de_object(template_name, 'templates', dest_path, 'template', 'template_name', skip_auto_register, download_progress_callback)
def download_restricted(restricted_name: str,
dest_path: str or pathlib.Path,
skip_auto_register: bool) -> int:
return download_o3de_object(restricted_name, 'restricted', dest_path, 'restricted', 'restricted_name', skip_auto_register)
skip_auto_register: bool,
download_progress_callback = None) -> int:
return download_o3de_object(restricted_name, 'restricted', dest_path, 'restricted', 'restricted_name', skip_auto_register, download_progress_callback)
def _run_download(args: argparse) -> int:
scripts/o3de/o3de/utils.py
+40 -5
View File
@@ -10,6 +10,7 @@ This file contains utility functions
"""
import sys
import uuid
import os
import pathlib
import shutil
import urllib.request
@@ -19,6 +20,29 @@ import zipfile
logger = logging.getLogger()
logging.basicConfig()
COPY_BUFSIZE = 64 * 1024
def copyfileobj(fsrc, fdst, callback, length=0):
# This is functionally the same as the python shutil copyfileobj but
# allows for a callback to return the download progress in blocks and allows
# to early out to cancel the copy.
if not length:
length = COPY_BUFSIZE
fsrc_read = fsrc.read
fdst_write = fdst.write
copied = 0
while True:
buf = fsrc_read(length)
if not buf:
break
fdst_write(buf)
copied += len(buf)
if callback(copied):
return 1
return 0
def validate_identifier(identifier: str) -> bool:
"""
Determine if the identifier supplied is valid.
@@ -93,18 +117,29 @@ def backup_folder(folder: str or pathlib.Path) -> None:
if backup_folder_name.is_dir():
renamed = True
def download_file(parsed_uri, download_path: pathlib.Path) -> int:
def download_file(parsed_uri, download_path: pathlib.Path, download_progress_callback = None) -> int:
"""
:param parsed_uri: uniform resource identifier to zip file to download
:param download_path: location path on disk to download file
:download_progress_callback: callback called with the download progress as a percentage, returns true to request to cancel the download
"""
if download_path.is_file():
logger.warn(f'File already downloaded to {download_path}.')
elif parsed_uri.scheme in ['http', 'https', 'ftp', 'ftps']:
with urllib.request.urlopen(parsed_uri.geturl()) as s:
download_file_size = 0
try:
download_file_size = s.headers['content-length']
except KeyError:
pass
def download_progress(blocks):
if download_progress_callback and download_file_size:
return download_progress_callback(int(blocks/int(download_file_size) * 100))
return False
with download_path.open('wb') as f:
shutil.copyfileobj(s, f)
download_cancelled = copyfileobj(s, f, download_progress)
if download_cancelled:
return 1
else:
origin_file = pathlib.Path(parsed_uri.geturl()).resolve()
if not origin_file.is_file():
@@ -114,12 +149,12 @@ def download_file(parsed_uri, download_path: pathlib.Path) -> int:
return 0
def download_zip_file(parsed_uri, download_zip_path: pathlib.Path) -> int:
def download_zip_file(parsed_uri, download_zip_path: pathlib.Path, download_progress_callback = None) -> int:
"""
:param parsed_uri: uniform resource identifier to zip file to download
:param download_zip_path: path to output zip file
"""
download_file_result = download_file(parsed_uri, download_zip_path)
download_file_result = download_file(parsed_uri, download_zip_path, download_progress_callback)
if download_file_result != 0:
return download_file_result