diff --git a/Code/Framework/AzQtComponents/AzQtComponents/Components/Style.cpp b/Code/Framework/AzQtComponents/AzQtComponents/Components/Style.cpp index 0f58f06420..47c0e66ff0 100644 --- a/Code/Framework/AzQtComponents/AzQtComponents/Components/Style.cpp +++ b/Code/Framework/AzQtComponents/AzQtComponents/Components/Style.cpp @@ -493,6 +493,29 @@ namespace AzQtComponents } } break; + case CE_MenuItem: + { + const QMenu* menu = qobject_cast(widget); + QAction* action = menu->activeAction(); + if (action) + { + QMenu* subMenu = action->menu(); + if (subMenu) + { + QVariant noHover = subMenu->property("noHover"); + if (noHover.isValid() && noHover.toBool()) + { + // First draw as standard to get the correct hover background for the complete control. + QProxyStyle::drawControl(element, option, painter, widget); + // Now draw the icon as non-hovered so control behaves as designed. + QStyleOptionMenuItem myOpt = *qstyleoption_cast(option); + myOpt.state &= ~QStyle::State_Selected; + return QProxyStyle::drawControl(element, &myOpt, painter, widget); + } + } + } + } + break; } return QProxyStyle::drawControl(element, option, painter, widget); diff --git a/Code/Framework/AzQtComponents/AzQtComponents/Images/Notifications/link.svg b/Code/Framework/AzQtComponents/AzQtComponents/Images/Notifications/link.svg index dfd21d157f..6f5608c092 100644 --- a/Code/Framework/AzQtComponents/AzQtComponents/Images/Notifications/link.svg +++ b/Code/Framework/AzQtComponents/AzQtComponents/Images/Notifications/link.svg @@ -1,4 +1,4 @@ - - + + diff --git a/Gems/AWSCore/Code/Include/Private/Editor/UI/AWSCoreEditorMenu.h b/Gems/AWSCore/Code/Include/Private/Editor/UI/AWSCoreEditorMenu.h index c892f86b66..ab03223323 100644 --- a/Gems/AWSCore/Code/Include/Private/Editor/UI/AWSCoreEditorMenu.h +++ b/Gems/AWSCore/Code/Include/Private/Editor/UI/AWSCoreEditorMenu.h @@ -46,6 +46,7 @@ namespace AWSCore void InitializeAWSDocActions(); void InitializeAWSGlobalDocsSubMenu(); void InitializeAWSFeatureGemActions(); + void AddSpaceForIcon(QMenu* menu); // AWSCoreEditorRequestBus interface implementation void SetAWSClientAuthEnabled() override; diff --git a/Gems/AWSCore/Code/Source/Editor/UI/AWSCoreEditorMenu.cpp b/Gems/AWSCore/Code/Source/Editor/UI/AWSCoreEditorMenu.cpp index c319788547..27e8710cbc 100644 --- a/Gems/AWSCore/Code/Source/Editor/UI/AWSCoreEditorMenu.cpp +++ b/Gems/AWSCore/Code/Source/Editor/UI/AWSCoreEditorMenu.cpp @@ -35,6 +35,9 @@ namespace AWSCore { + + static constexpr int IconSize = 16; + AWSCoreEditorMenu::AWSCoreEditorMenu(const QString& text) : QMenu(text) , m_resourceMappingToolWatcher(nullptr) @@ -43,6 +46,7 @@ namespace AWSCore InitializeResourceMappingToolAction(); this->addSeparator(); InitializeAWSFeatureGemActions(); + AddSpaceForIcon(this); AWSCoreEditorRequestBus::Handler::BusConnect(); } @@ -136,6 +140,8 @@ namespace AWSCore globalDocsMenu->addAction(AddExternalLinkAction(AWSAndScriptCanvasActionText, AWSAndScriptCanvasUrl, ":/Notifications/link.svg")); globalDocsMenu->addAction(AddExternalLinkAction(AWSAndComponentsActionText, AWSAndComponentsUrl, ":/Notifications/link.svg")); globalDocsMenu->addAction(AddExternalLinkAction(CallAWSResourcesActionText, CallAWSResourcesUrl, ":/Notifications/link.svg")); + + AddSpaceForIcon(globalDocsMenu); } void AWSCoreEditorMenu::InitializeAWSFeatureGemActions() @@ -170,6 +176,8 @@ namespace AWSCore AWSClientAuthPlatformSpecificActionText, AWSClientAuthPlatformSpecificUrl, ":/Notifications/link.svg")); subMenu->addAction(AddExternalLinkAction( AWSClientAuthAPIReferenceActionText, AWSClientAuthAPIReferenceUrl, ":/Notifications/link.svg")); + + AddSpaceForIcon(subMenu); } void AWSCoreEditorMenu::SetAWSMetricsEnabled() @@ -197,7 +205,9 @@ namespace AWSCore [configFilePath](){ QDesktopServices::openUrl(QUrl::fromLocalFile(configFilePath.c_str())); }); + subMenu->addAction(settingsAction); + AddSpaceForIcon(subMenu); } QMenu* AWSCoreEditorMenu::SetAWSFeatureSubMenu(const AZStd::string& menuText) @@ -209,6 +219,7 @@ namespace AWSCore { QMenu* subMenu = new QMenu(QObject::tr(menuText.c_str())); subMenu->setIcon(QIcon(QString(":/Notifications/checkmark.svg"))); + subMenu->setProperty("noHover", true); this->insertMenu(*itr, subMenu); this->removeAction(*itr); return subMenu; @@ -216,4 +227,11 @@ namespace AWSCore } return nullptr; } + + void AWSCoreEditorMenu::AddSpaceForIcon(QMenu *menu) + { + QSize size = menu->sizeHint(); + size.setWidth(size.width() + IconSize); + menu->setFixedSize(size); + } } // namespace AWSCore diff --git a/Gems/Atom/Feature/Common/Assets/Scripts/performance_metrics/timestamp_aggregator.py b/Gems/Atom/Feature/Common/Assets/Scripts/performance_metrics/timestamp_aggregator.py new file mode 100644 index 0000000000..0508395275 --- /dev/null +++ b/Gems/Atom/Feature/Common/Assets/Scripts/performance_metrics/timestamp_aggregator.py @@ -0,0 +1,62 @@ +#!/usr/bin/env python +""" +All or portions of this file Copyright (c) Amazon.com, Inc. or its affiliates or +its licensors. + +For complete copyright and license terms please see the LICENSE at the root of this +distribution (the "License"). All use of this software is governed by the License, +or, if provided, by the license below or the license accompanying this file. Do not +remove or modify any license notices. This file is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +""" + +from genericpath import isdir +from argparse import ArgumentParser +import json +from pathlib import Path +import os + +# this allows us to add additional data if necessary, e.g. frame_test_timestamps.json +is_timestamp_file = lambda file: file.name.startswith('frame') and file.name.endswith('_timestamps.json') +ns_to_ms = lambda time: time / 1e6 + +def main(logs_dir): + count = 0 + total = 0 + maximum = 0 + + print(f'Analyzing frame timestamp logs in {logs_dir}') + + # go through files in alphabetical order (remove sorted() if not necessary) + for file in sorted(logs_dir.iterdir(), key=lambda file: len(file.name)): + if file.is_dir() or not is_timestamp_file(file): + continue + + data = json.loads(file.read_text()) + entries = data['ClassData']['timestampEntries'] + timestamps = [entry['timestampResultInNanoseconds'] for entry in entries] + + frame_time = sum(timestamps) + frame_name = file.name.split('_')[0] + print(f'- Total time for frame {frame_name}: {ns_to_ms(frame_time)}ms') + + maximum = max(maximum, frame_time) + total += frame_time + count += 1 + + if count < 1: + print(f'No logs were found in {base_dir}') + exit(1) + + print(f'Avg. time across {count} frames: {ns_to_ms(total / count)}ms') + print(f'Max frame time: {ns_to_ms(maximum)}ms') + +if __name__ == '__main__': + parser = ArgumentParser(description='Gathers statistics from a group of pass timestamp logs') + parser.add_argument('path', help='Path to the directory containing the pass timestamp logs') + args = parser.parse_args() + + base_dir = Path(args.path) + if not base_dir.exists(): + raise FileNotFoundError('Invalid path provided') + main(base_dir) diff --git a/Gems/Atom/Feature/Common/Code/Source/DisplayMapper/DisplayMapperConfigurationDescriptor.cpp b/Gems/Atom/Feature/Common/Code/Source/DisplayMapper/DisplayMapperConfigurationDescriptor.cpp index a064b61a1a..ae48c7defb 100644 --- a/Gems/Atom/Feature/Common/Code/Source/DisplayMapper/DisplayMapperConfigurationDescriptor.cpp +++ b/Gems/Atom/Feature/Common/Code/Source/DisplayMapper/DisplayMapperConfigurationDescriptor.cpp @@ -50,6 +50,8 @@ namespace AZ if (auto behaviorContext = azrtti_cast(context)) { + behaviorContext->Class("OutputDeviceTransformType"); + behaviorContext->Class("AcesParameterOverrides") ->Attribute(AZ::Script::Attributes::Scope, AZ::Script::Attributes::ScopeFlags::Common) ->Attribute(AZ::Script::Attributes::Category, "render") @@ -58,6 +60,16 @@ namespace AZ ->Method("LoadPreset", &AcesParameterOverrides::LoadPreset) ->Property("overrideDefaults", BehaviorValueProperty(&AcesParameterOverrides::m_overrideDefaults)) ->Property("preset", BehaviorValueProperty(&AcesParameterOverrides::m_preset)) + ->Enum(OutputDeviceTransformType::NumOutputDeviceTransformTypes)>( + "OutputDeviceTransformType_NumOutputDeviceTransformTypes") + ->Enum(OutputDeviceTransformType::OutputDeviceTransformType_48Nits)>( + "OutputDeviceTransformType_48Nits") + ->Enum(OutputDeviceTransformType::OutputDeviceTransformType_1000Nits)>( + "OutputDeviceTransformType_1000Nits") + ->Enum(OutputDeviceTransformType::OutputDeviceTransformType_2000Nits)>( + "OutputDeviceTransformType_2000Nits") + ->Enum(OutputDeviceTransformType::OutputDeviceTransformType_4000Nits)>( + "OutputDeviceTransformType_4000Nits") ->Property("alterSurround", BehaviorValueProperty(&AcesParameterOverrides::m_alterSurround)) ->Property("applyDesaturation", BehaviorValueProperty(&AcesParameterOverrides::m_applyDesaturation)) ->Property("applyCATD60toD65", BehaviorValueProperty(&AcesParameterOverrides::m_applyCATD60toD65)) diff --git a/Gems/Multiplayer/Code/Source/MultiplayerSystemComponent.cpp b/Gems/Multiplayer/Code/Source/MultiplayerSystemComponent.cpp index e4256a875c..c43d136cd8 100644 --- a/Gems/Multiplayer/Code/Source/MultiplayerSystemComponent.cpp +++ b/Gems/Multiplayer/Code/Source/MultiplayerSystemComponent.cpp @@ -179,7 +179,10 @@ namespace Multiplayer AZ::TickBus::Handler::BusConnect(); AzFramework::SessionNotificationBus::Handler::BusConnect(); m_networkInterface = AZ::Interface::Get()->CreateNetworkInterface(AZ::Name(MPNetworkInterfaceName), sv_protocol, TrustZone::ExternalClientToServer, *this); - m_consoleCommandHandler.Connect(AZ::Interface::Get()->GetConsoleCommandInvokedEvent()); + if (AZ::Interface::Get()) + { + m_consoleCommandHandler.Connect(AZ::Interface::Get()->GetConsoleCommandInvokedEvent()); + } AZ::Interface::Register(this); AZ::Interface::Register(this); @@ -191,6 +194,8 @@ namespace Multiplayer { AZ::Interface::Unregister(this); AZ::Interface::Unregister(this); + m_consoleCommandHandler.Disconnect(); + AZ::Interface::Get()->DestroyNetworkInterface(AZ::Name(MPNetworkInterfaceName)); AzFramework::SessionNotificationBus::Handler::BusDisconnect(); AZ::TickBus::Handler::BusDisconnect(); } @@ -199,19 +204,10 @@ namespace Multiplayer { AZ::Interface::Get()->InitializeMultiplayer(MultiplayerAgentType::Client); + m_pendingConnectionTickets.push(config.m_playerSessionId); AZStd::string hostname = config.m_dnsName.empty() ? config.m_ipAddress : config.m_dnsName; const IpAddress ipAddress(hostname.c_str(), config.m_port, m_networkInterface->GetType()); - ConnectionId connectionId = m_networkInterface->Connect(ipAddress); - - AzNetworking::IConnection* connection = m_networkInterface->GetConnectionSet().GetConnection(connectionId); - if (connection->GetUserData() == nullptr) // Only add user data if the connect event handler has not already done so - { - connection->SetUserData(new ClientToServerConnectionData(connection, *this, config.m_playerSessionId)); - } - else - { - reinterpret_cast(connection->GetUserData())->SetProviderTicket(config.m_playerSessionId); - } + m_networkInterface->Connect(ipAddress); return true; } @@ -584,15 +580,16 @@ namespace Multiplayer datum.m_isInvited = false; datum.m_agentType = MultiplayerAgentType::Client; + AZStd::string providerTicket; if (connection->GetConnectionRole() == ConnectionRole::Connector) { AZLOG_INFO("New outgoing connection to remote address: %s", connection->GetRemoteAddress().GetString().c_str()); - AZ::CVarFixedString providerTicket; - if (connection->GetUserData() != nullptr) + if (!m_pendingConnectionTickets.empty()) { - providerTicket = reinterpret_cast(connection->GetUserData())->GetProviderTicket(); + providerTicket = m_pendingConnectionTickets.front(); + m_pendingConnectionTickets.pop(); } - connection->SendReliablePacket(MultiplayerPackets::Connect(0, providerTicket)); + connection->SendReliablePacket(MultiplayerPackets::Connect(0, providerTicket.c_str())); } else { @@ -622,7 +619,11 @@ namespace Multiplayer { if (connection->GetUserData() == nullptr) // Only add user data if the connect event handler has not already done so { - connection->SetUserData(new ClientToServerConnectionData(connection, *this)); + connection->SetUserData(new ClientToServerConnectionData(connection, *this, providerTicket)); + } + else + { + reinterpret_cast(connection->GetUserData())->SetProviderTicket(providerTicket); } AZStd::unique_ptr window = AZStd::make_unique(); @@ -646,14 +647,10 @@ namespace Multiplayer AZStd::string reasonString = ToString(reason); AZLOG_INFO("%s due to %s from remote address: %s", endpointString, reasonString.c_str(), connection->GetRemoteAddress().GetString().c_str()); - if (connection->GetConnectionRole() == ConnectionRole::Acceptor) - { - // The authority is shutting down its connection - m_shutdownEvent.Signal(m_networkInterface); - } - else if (GetAgentType() == MultiplayerAgentType::Client && connection->GetConnectionRole() == ConnectionRole::Connector) + // The client is disconnecting + if (GetAgentType() == MultiplayerAgentType::Client) { - // The client is disconnecting + AZ_Assert(connection->GetConnectionRole() == ConnectionRole::Connector, "Client connection role should only ever be Connector"); m_clientDisconnectedEvent.Signal(); } @@ -669,7 +666,7 @@ namespace Multiplayer if (m_agentType == MultiplayerAgentType::DedicatedServer || m_agentType == MultiplayerAgentType::ClientServer) { if (AZ::Interface::Get() != nullptr && - connection->GetConnectionRole() == ConnectionRole::Connector) + connection->GetConnectionRole() == ConnectionRole::Acceptor) { AzFramework::PlayerConnectionConfig config; config.m_playerConnectionId = aznumeric_cast(connection->GetConnectionId()); @@ -680,12 +677,15 @@ namespace Multiplayer // Signal to session management when there are no remaining players in a dedicated server for potential cleanup // We avoid this for client server as the host itself is a user - if (m_agentType == MultiplayerAgentType::DedicatedServer && connection->GetConnectionRole() == ConnectionRole::Connector) + if (m_agentType == MultiplayerAgentType::DedicatedServer && connection->GetConnectionRole() == ConnectionRole::Acceptor) { - if (AZ::Interface::Get() != nullptr - && m_networkInterface->GetConnectionSet().GetConnectionCount() == 0) + if (m_networkInterface->GetConnectionSet().GetConnectionCount() == 0) { - AZ::Interface::Get()->HandleDestroySession(); + m_shutdownEvent.Signal(m_networkInterface); + if (AZ::Interface::Get() != nullptr) + { + AZ::Interface::Get()->HandleDestroySession(); + } } } } diff --git a/Gems/Multiplayer/Code/Source/MultiplayerSystemComponent.h b/Gems/Multiplayer/Code/Source/MultiplayerSystemComponent.h index 0efef3ebe4..00313c6b65 100644 --- a/Gems/Multiplayer/Code/Source/MultiplayerSystemComponent.h +++ b/Gems/Multiplayer/Code/Source/MultiplayerSystemComponent.h @@ -146,6 +146,8 @@ namespace Multiplayer ConnectionAcquiredEvent m_connAcquiredEvent; ClientDisconnectedEvent m_clientDisconnectedEvent; + AZStd::queue m_pendingConnectionTickets; + AZ::TimeMs m_lastReplicatedHostTimeMs = AZ::TimeMs{ 0 }; HostFrameId m_lastReplicatedHostFrameId = InvalidHostFrameId; diff --git a/Gems/Multiplayer/Code/Tests/MultiplayerSystemTests.cpp b/Gems/Multiplayer/Code/Tests/MultiplayerSystemTests.cpp index 7b09b65de4..9579c84fc1 100644 --- a/Gems/Multiplayer/Code/Tests/MultiplayerSystemTests.cpp +++ b/Gems/Multiplayer/Code/Tests/MultiplayerSystemTests.cpp @@ -35,14 +35,16 @@ namespace UnitTest m_initHandler = Multiplayer::SessionInitEvent::Handler([this](AzNetworking::INetworkInterface* value) { TestInitEvent(value); }); m_mpComponent->AddSessionInitHandler(m_initHandler); - m_shutdownHandler = Multiplayer::SessionInitEvent::Handler([this](AzNetworking::INetworkInterface* value) { TestShutdownEvent(value); }); + m_shutdownHandler = Multiplayer::SessionShutdownEvent::Handler([this](AzNetworking::INetworkInterface* value) { TestShutdownEvent(value); }); m_mpComponent->AddSessionShutdownHandler(m_shutdownHandler); m_connAcquiredHandler = Multiplayer::ConnectionAcquiredEvent::Handler([this](Multiplayer::MultiplayerAgentDatum value) { TestConnectionAcquiredEvent(value); }); m_mpComponent->AddConnectionAcquiredHandler(m_connAcquiredHandler); + m_mpComponent->Activate(); } void TearDown() override { + m_mpComponent->Deactivate(); delete m_mpComponent; delete m_netComponent; AZ::NameDictionary::Destroy(); @@ -86,6 +88,7 @@ namespace UnitTest TEST_F(MultiplayerSystemTests, TestShutdownEvent) { + m_mpComponent->InitializeMultiplayer(Multiplayer::MultiplayerAgentType::DedicatedServer); IMultiplayerConnectionMock connMock1 = IMultiplayerConnectionMock(AzNetworking::ConnectionId(), AzNetworking::IpAddress(), AzNetworking::ConnectionRole::Acceptor); IMultiplayerConnectionMock connMock2 = IMultiplayerConnectionMock(AzNetworking::ConnectionId(), AzNetworking::IpAddress(), AzNetworking::ConnectionRole::Connector); m_mpComponent->OnDisconnect(&connMock1, AzNetworking::DisconnectReason::None, AzNetworking::TerminationEndpoint::Local); diff --git a/scripts/build/Platform/Windows/build_config.json b/scripts/build/Platform/Windows/build_config.json index e055f07482..e37e467fa0 100644 --- a/scripts/build/Platform/Windows/build_config.json +++ b/scripts/build/Platform/Windows/build_config.json @@ -189,7 +189,8 @@ "CMAKE_NATIVE_BUILD_ARGS": "/m /nologo", "CTEST_OPTIONS": "-L \"(SUITE_smoke_REQUIRES_gpu|SUITE_main_REQUIRES_gpu)\" -T Test", "TEST_METRICS": "True", - "TEST_RESULTS": "True" + "TEST_RESULTS": "True", + "TEST_SCREENSHOTS": "True" } }, "asset_profile_vs2019": { diff --git a/scripts/build/tools/upload_to_s3.py b/scripts/build/tools/upload_to_s3.py index 5dfe5eb66e..5666e6ec6b 100755 --- a/scripts/build/tools/upload_to_s3.py +++ b/scripts/build/tools/upload_to_s3.py @@ -17,12 +17,16 @@ python upload_to_s3.py --base_dir %WORKSPACE% --file_regex "(.*zip$|.*MD5$)" --b Use profile to upload all .zip and .MD5 files in %WORKSPACE% folder to bucket ly-packages-mainline: python upload_to_s3.py --base_dir %WORKSPACE% --profile profile --file_regex "(.*zip$|.*MD5$)" --bucket ly-packages-mainline +Another example usage for uploading all .png and .ppm files inside base_dir and only subdirectories within base_dir: +python upload_to_s3.py --base_dir %WORKSPACE%/path/to/files --file_regex "(.*png$|.*ppm$)" --bucket screenshot-test-bucket --search_subdirectories True --key_prefix Test + ''' import os import re import json +import time import boto3 from optparse import OptionParser @@ -34,6 +38,8 @@ def parse_args(): parser.add_option("--profile", dest="profile", default=None, help="The name of a profile to use. If not given, then the default profile is used.") parser.add_option("--bucket", dest="bucket", default=None, help="S3 bucket the files are uploaded to.") parser.add_option("--key_prefix", dest="key_prefix", default='', help="Object key prefix.") + parser.add_option("--search_subdirectories", dest="search_subdirectories", action='store_true', + help="Toggle for searching for files in subdirectories beneath base_dir, defaults to False") ''' ExtraArgs used to call s3.upload_file(), should be in json format. extra_args key must be one of: ACL, CacheControl, ContentDisposition, ContentEncoding, ContentLanguage, ContentType, Expires, GrantFullControl, GrantRead, GrantReadACP, GrantWriteACP, Metadata, RequestPayer, ServerSideEncryption, StorageClass, @@ -62,48 +68,82 @@ def get_client(service_name, profile_name): return client -def get_files_to_upload(base_dir, regex): +def get_files_to_upload(base_dir, regex, search_subdirectories): + """ + Uses a regex expression pattern to return a list of file paths for files to upload to the s3 bucket. + :param base_dir: path for the base directory, if using search_subdirectories=True ensure this is the parent. + :param regex: pattern to use for regex searching, ex. "(.*zip$|.*MD5$)" + :param search_subdirectories: boolean False for only getting files in base_dir, True to get all files in base_dir + and any subdirectory inside base_dir, defaults to False from the parse_args() function. + :return: a list of string file paths for files to upload to the s3 bucket matching the regex expression. + """ # Get all file names in base directory - files = [x for x in os.listdir(base_dir) if os.path.isfile(os.path.join(base_dir, x))] - # strip the surround quotes, if they exist + files = [os.path.join(base_dir, x) for x in os.listdir(base_dir) if os.path.isfile(os.path.join(base_dir, x))] + if search_subdirectories: # Get all file names in base directory and any subdirectories. + for subdirectory in os.walk(base_dir): + # Example output for subdirectory: + # ('C:\path\to\base_dir\', ['Subfolder1', 'Subfolder2'], ['file1', 'file2']) + subdirectory_file_path = subdirectory[0] + subdirectory_files = subdirectory[2] + if subdirectory_files: + subdirectory_file_paths = _build_file_paths(subdirectory_file_path, subdirectory_files) + files.extend(subdirectory_file_paths) + try: - regex = json.loads(regex) + regex = json.loads(regex) # strip the surround quotes, if they exist except: + print(f'WARNING: failed to call json.loads() for regex: "{regex}"') pass # Get all file names matching the regular expression, those file will be uploaded to S3 - files_to_upload = [x for x in files if re.match(regex, x)] - return files_to_upload + regex_files_to_upload = [x for x in files if re.match(regex, x)] + + return regex_files_to_upload -def s3_upload_file(client, base_dir, file, bucket, key_prefix=None, extra_args=None, max_retry=1): - print(('Uploading file {} to bucket {}.'.format(file, bucket))) - key = file if key_prefix is None else '{}/{}'.format(key_prefix, file) +def s3_upload_file(client, file, bucket, key_prefix=None, extra_args=None, max_retry=1): + key = file if key_prefix is None else f'{key_prefix}/{file}' + error_message = None + for x in range(max_retry): try: - client.upload_file( - os.path.join(base_dir, file), bucket, key, - ExtraArgs=extra_args - ) - print('Upload succeeded') + client.upload_file(file, bucket, key, ExtraArgs=extra_args) return True except Exception as err: - print(('exception while uploading: {}'.format(err))) - print('Retrying upload...') - print('Upload failed') + time.sleep(0.1) # Sleep for 100 milliseconds between retries. + error_message = err + + print(f'Upload failed - Exception while uploading: {error_message}') return False +def _build_file_paths(path_to_files, files_in_path): + """ + Given a path containing files, returns a list of strings representing complete paths to each file. + :param path_to_files: path to the location storing the files to create string paths for + :param files_in_path: list of files that are inside the path_to_files path string + :return: list of fully parsed file path strings from path_to_files path. + """ + parsed_file_paths = [] + + for file_in_path in files_in_path: + complete_file_path = os.path.join(path_to_files, file_in_path) + if os.path.isfile(complete_file_path): + parsed_file_paths.append(complete_file_path) + + return parsed_file_paths + + if __name__ == "__main__": options = parse_args() client = get_client('s3', options.profile) - files_to_upload = get_files_to_upload(options.base_dir, options.file_regex) + files_to_upload = get_files_to_upload(options.base_dir, options.file_regex, options.search_subdirectories) extra_args = json.loads(options.extra_args) if options.extra_args else None print(('Uploading {} files to bucket {}.'.format(len(files_to_upload), options.bucket))) failure = [] success = [] for file in files_to_upload: - if not s3_upload_file(client, options.base_dir, file, options.bucket, options.key_prefix, extra_args, 2): + if not s3_upload_file(client, file, options.bucket, options.key_prefix, extra_args, 2): failure.append(file) else: success.append(file)