Files
o3de/cmake/Tools/Platform/Android/android_deployment.py
T
Steve Pham f775ba7df8 Provide more informative error messages on android related environment / device related issues (#1261)
- If gradle is installed, but JAVA_HOME is not set properly, no detail message is given. Bubble up the error message as part of the description
- When deploying a newer API Level APK (30) to an API Level 29 device, a python callstack is given without any detail of the error. Now it will report the actual error that is from the adb call so the user can act upon it
2021-06-11 08:12:49 -07:00

558 lines
27 KiB
Python
Executable File

#
# All or portions of this file Copyright (c) Amazon.com, Inc. or its affiliates or
# its licensors.
#
# For complete copyright and license terms please see the LICENSE at the root of this
# distribution (the "License"). All use of this software is governed by the License,
# or, if provided, by the license below or the license accompanying this file. Do not
# remove or modify any license notices. This file is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
#
import datetime
import logging
import os
import json
import platform
import subprocess
import sys
import time
import pathlib
from distutils.version import LooseVersion
# Resolve the common python module
ROOT_DEV_PATH = os.path.realpath(os.path.join(os.path.dirname(__file__), '..', '..', '..', '..'))
if ROOT_DEV_PATH not in sys.path:
sys.path.append(ROOT_DEV_PATH)
from cmake.Tools import common
from cmake.Tools.Platform.Android import android_support
# The following is the list of known android external storage paths that we will attempt to verify on a device and
# return the first one that is detected
KNOWN_ANDROID_EXTERNAL_STORAGE_PATHS = [
'/sdcard/',
'/storage/emulated/0/',
'/storage/emulated/legacy/',
'/storage/sdcard0/',
'/storage/self/primary/',
]
ANDROID_TARGET_TIMESTAMP_FILENAME = 'deploy.timestamp'
class AndroidDeployment(object):
"""
Class to manage the deployment of game assets to an android device (Separately from the APK)
"""
DEPLOY_APK_ONLY = 'APK'
DEPLOY_ASSETS_ONLY = 'ASSETS'
DEPLOY_BOTH = 'BOTH'
def __init__(self, dev_root, build_dir, configuration, android_device_filter, clean_deploy, android_sdk_path, deployment_type, game_name=None, asset_mode=None, asset_type=None, embedded_assets=True, is_unit_test=False):
"""
Initialize the Android Deployment Worker
:param dev_root: The dev-root of the engine
:param android_device_filter: An optional list of devices to filter on the connected devices to deploy to. If not supplied, deploy to all devices
:param clean_deploy: Option to clean the target device's assets before deploying the game's assets from the host
:param android_sdk_path: Path to the android SDK (to use the adb tool)
:param deployment_type: The type of deployment (DEPLOY_APK_ONLY, DEPLOY_ASSETS_ONLY, or DEPLOY_BOTH)
:param game_name: The name of the game whose assets are being deployed. None if is_test_project is True
:param asset_mode: The asset mode of deployment (LOOSE, PAK, VFS). None if is_test_project is True
:param asset_type: The asset type. None if is_test_project is True
:param embedded_assets: Boolean to indicate if the assets are embedded in the APK or not
:param is_unit_test: Boolean to indicate if this is a unit test deployment
"""
self.dev_root = pathlib.Path(dev_root)
self.build_dir = self.dev_root / build_dir
self.configuration = configuration
self.game_name = game_name
self.asset_mode = asset_mode
self.asset_type = asset_type
self.clean_deploy = clean_deploy
self.embedded_assets = embedded_assets
self.deployment_type = deployment_type
self.is_test_project = is_unit_test
if not self.is_test_project:
if embedded_assets:
# If the assets are embedded, then warn that both APK and ASSETS will be deployed even if 'BOTH' is not specified
if deployment_type in (AndroidDeployment.DEPLOY_APK_ONLY, AndroidDeployment.DEPLOY_ASSETS_ONLY):
logging.warning(f"Deployment type of {deployment_type} set but the assets are embedded in the APK. Both the APK and the Assets will be deployed.")
if asset_mode == 'PAK':
self.local_asset_path = self.dev_root / 'Pak' / f'{game_name.lower()}_{asset_type}_paks'
else:
self.local_asset_path = self.dev_root / game_name / 'Cache' / asset_type
assert game_name is not None, f"'game_name' is required"
self.game_name = game_name
assert asset_mode is not None, f"'asset_mode' is required"
self.asset_mode = asset_mode
assert asset_type is not None, f"'asset_type' is required"
self.asset_type = asset_type
self.files_in_asset_path = list(self.local_asset_path.glob('**/*'))
self.android_settings = AndroidDeployment.read_android_settings(self.dev_root, game_name)
else:
self.local_asset_path = None
if asset_mode:
logging.warning(f"'asset_mode' argument '{asset_mode}' ignored for unit test deployment.")
if asset_type:
logging.warning(f"'asset_type' argument '{asset_type}' ignored for unit test deployment.")
self.files_in_asset_path = []
self.apk_path = self.build_dir / 'app' / 'build' / 'outputs' / 'apk' / configuration / f'app-{configuration}.apk'
self.android_device_filter = [android_device.strip() for android_device in android_device_filter.split(',')] if android_device_filter else []
self.adb_path = AndroidDeployment.resolve_adb_tool(pathlib.Path(android_sdk_path))
self.adb_started = False
@staticmethod
def read_android_settings(dev_root, game_name):
"""
Read and parse the project.json file into a dictionary to process the specific attributes needed for the manifest template
:param dev_root: The dev root we are working from
:param game_name: Name of the game under the dev root
:return: The android settings for the game project if any
"""
game_folder = dev_root / game_name
game_folder_project_properties_path = game_folder / 'project.json'
game_project_properties_content = game_folder_project_properties_path.resolve(strict=True)\
.read_text(encoding=common.DEFAULT_TEXT_READ_ENCODING,
errors=common.ENCODING_ERROR_HANDLINGS)
# Extract the key attributes we need to process and build up our environment table
game_project_json = json.loads(game_project_properties_content)
android_settings = game_project_json.get('android_settings', {})
return android_settings
@staticmethod
def resolve_adb_tool(android_sdk_path):
"""
Resolve the location of the adb tool based on the input Android SDK Path
:param android_sdk_path: The android SDK path to search for the adb tool
:return: The absolute path to the adb tool
"""
adb_target = 'adb.exe' if platform.system() == 'Windows' else 'adb'
check_adb_target = android_sdk_path / 'platform-tools' / adb_target
if not check_adb_target.exists():
raise common.LmbrCmdError(f"Invalid Android SDK path '{str(android_sdk_path)}': Unable to locate '{adb_target}'.")
return check_adb_target
def get_android_project_settings(self, key_name, default_value):
return self.android_settings.get(key_name, default_value)
def adb_call(self, arg_list, device_id=None):
"""
Wrapper to execute the adb command-line tool
:param arg_list: Argument list to send to the tool
:param device_id: Optional device id (from the 'get_target_android_devices' call) to invoke the call to.
:return: The stdout result of the call
"""
if isinstance(arg_list, str):
arg_list = [arg_list]
call_arguments = [str(self.adb_path.resolve())]
if device_id:
call_arguments.extend(['-s', device_id])
call_arguments.extend(arg_list)
try:
output = subprocess.check_output(call_arguments,
shell=True,
stderr=subprocess.PIPE).decode(common.DEFAULT_TEXT_READ_ENCODING,
common.ENCODING_ERROR_HANDLINGS)
return output
except subprocess.CalledProcessError as err:
raise common.LmbrCmdError(err.stderr.decode(common.DEFAULT_TEXT_READ_ENCODING,
common.ENCODING_ERROR_HANDLINGS))
def adb_shell(self, command, device_id):
"""
Special wrapper around calling "adb shell" which will invoke a shell command on the android device
:param command: The shell command to invoke on the android device
:param device_id: The device id (from the 'get_target_android_devices' call) to invoke the shell call on
:return: The stdout result of the call
"""
shell_command = ['shell', command]
return self.adb_call(shell_command, device_id=device_id)
def adb_ls(self, path, device_id, args=None):
"""
Request an 'ls' call on the android device
:param path: The path to perform the 'ls' call on
:param device_id: device id (from the 'get_target_android_devices' call) to invoke the shell call on
:param args: Additional args to pass into the l'ls' call
:return: Tuple of Boolean result of the call and the output of the call
"""
error_messages = [
'No such file or directory',
'Permission denied'
]
shell_command = ['ls']
if args:
shell_command.extend(args)
shell_command.append(path)
logging.debug(f"Testing {device_id}: ls {' '.join(shell_command)}")
raw_output = self.adb_shell(command=' '.join(shell_command),
device_id=device_id)
if not raw_output:
logging.debug('adb_ls: No output given')
return False, None
if raw_output is None or any([error for error in error_messages if error in raw_output]):
logging.debug('adb_ls: Error message found')
status = False
else:
logging.debug('adb_ls: Command was successful')
status = True
return status, raw_output
def get_target_android_devices(self):
"""
Gets all of the connected android devices with adb, filtered by the set optional device filter
:return: list of serial numbers of optionally filtered connected devices.
"""
connected_devices = []
# Call adb to get the device list and process the raw response
raw_devices_output = self.adb_call("devices")
if not raw_devices_output:
raise common.LmbrCmdError("Error getting connected devices through adb")
device_output_list = raw_devices_output.split(os.linesep)
for device_output in device_output_list:
if any(x in device_output for x in ['List', '*']):
logging.debug(f"Skipping the following line as it has 'List', '*' or 'emulator' in it: {device_output}")
continue
device_serial = device_output.split()
if device_serial:
if 'unauthorized' in device_output.lower():
logging.warning(f"Device {device_serial[0]} is not authorized for development access. Please reconnect the device and check for a confirmation dialog.")
elif device_serial[0] in self.android_device_filter or not self.android_device_filter:
connected_devices.append(device_serial[0])
else:
logging.debug(f"Skipping filtered out Device {device_serial[0]} .")
if not connected_devices:
raise common.LmbrCmdError("No connected android devices found")
return connected_devices
def check_known_android_paths(self, device_id):
"""
Look for a known android path that is writeable and return the first one that is found
:param device_id: The device id (from the 'get_target_android_devices' call) to invoke the shell call on
:return: The first available android path if found, None if not
"""
for path in KNOWN_ANDROID_EXTERNAL_STORAGE_PATHS:
logging.debug(f"Checking known path '{path}' on device '{device_id}'")
# Test the path by performing an 'ls' call on it and checking if an error is returned from the result
result, output = self.adb_ls(path=path,
args=None,
device_id=device_id)
if result:
return path[:-1]
return None
def detect_device_storage_path(self, device_id):
"""
Uses the device's environment variable "EXTERNAL_STORAGE" to determine the correct
path to public storage that has write permissions. If at any point does the env var
validation fail, fallback to checking known possible paths to external storage.
:param device_id:
:return: The first available storage device
"""
external_storage = self.adb_shell(command="set | grep EXTERNAL_STORAGE",
device_id=device_id)
if not external_storage:
logging.debug(f"Unable to get 'EXTERNAL_STORAGE' environment from device '{device_id}'. Falling back to known android paths.")
return self.check_known_android_paths(device_id)
# Given the 'EXTERNAL_STORAGE' environment, parse out the value and validate it
storage_path_key_value = external_storage.split('=')
if len(storage_path_key_value) != 2:
logging.debug(f"The value for 'EXTERNAL_STORAGE' environment from device '{device_id}' does not represent a valid key-value pair: {storage_path_key_value}. Falling back to known android paths")
return self.check_known_android_paths(device_id)
# Check the existence and permissions issue of the storage path
storage_path = storage_path_key_value[1].strip()
is_external_valid, _ = self.adb_ls(path=storage_path,
device_id=device_id)
if is_external_valid:
return storage_path
# The set external path has an issue, attempt to determine its real path through an adb shell call
logging.debug(f"The path specified in EXTERNAL_STORAGE seems to have permission issues, attempting to resolve with realpath for device {device_id}.")
real_path = self.adb_shell(command=f'realpath {storage_path}',
device_id=device_id)
if not real_path:
logging.debug(f"Unable to determine the real path '{storage_path}' (from EXTERNAL_STORAGE) for {self.game_name} on device {device_id}. Falling back to known android paths")
return self.check_known_android_paths(device_id)
real_path = real_path.strip()
is_external_valid, _ = self.adb_ls(path=real_path,
device_id=device_id)
if is_external_valid:
return real_path
logging.debug(f'Unable to validate the resolved EXTERNAL_STORAGE environment variable path for device {device_id}.')
return self.check_known_android_paths(device_id)
def get_device_file_timestamp(self, remote_file_path, device_id):
"""
Get the integer timestamp value of a file from a given device.
:param remote_file_path: The path to the timestamp file on the android device
:param device_id: The device id (from the 'get_target_android_devices' call) to invoke the shell call on
:return: The time value if found, None if not
"""
try:
timestamp_string = self.adb_shell(command=f'cat {remote_file_path}',
device_id=device_id).strip()
except (subprocess.CalledProcessError, AttributeError):
return None
if not timestamp_string:
return None
for fmt in ('%Y-%m-%d %H:%M:%S', '%Y-%m-%d %H:%M:%S.%f'):
try:
target_time = time.mktime(time.strptime(timestamp_string, fmt))
break
except ValueError:
target_time = None
return target_time
def update_device_file_timestamp(self, relative_assets_path, device_id):
"""
Update the device timestamp file on an android device to track files that need updating on pushes
:param relative_assets_path: The relative path to the assets on the android device
:param device_id: The device id (from the 'get_target_android_devices' call) to invoke the shell call on
"""
timestamp_str = str(datetime.datetime.now())
logging.debug(f"Updating timestamp on device {device_id} to {timestamp_str}")
local_timestamp_file_path = self.local_asset_path / ANDROID_TARGET_TIMESTAMP_FILENAME
local_timestamp_file_path.write_text(timestamp_str)
target_timestamp_file_path = f'{relative_assets_path}/{ANDROID_TARGET_TIMESTAMP_FILENAME}'
self.adb_call(arg_list=['push', str(local_timestamp_file_path), target_timestamp_file_path],
device_id=device_id)
@staticmethod
def should_copy_file(check_path, check_time):
"""
Check if a source file should be copied, by checking if its timestamp is newer than the 'check_time'
:param check_path: The path to the source file whose timestamp will be evaluated
:param check_time: The baseline 'check_time' value to compare the source file timestamp against
:return: True if the source file is newer than the baseline 'check_time', False if not
"""
if not check_path.is_file():
return False
stat_src = check_path.stat()
should_copy = stat_src.st_mtime >= check_time
return should_copy
def check_package_installed(self, package_name, target_device):
"""
Checks if the package for the game is currently installed or not
@param package_name: The name of the package to search for
@param target_device: The target device to search for the package on
@return: True if there an existing package on the device with the same package name, false if not
"""
output_result = self.adb_call(['shell', 'cmd', 'package', 'list', 'packages', package_name],
target_device)
return output_result != ''
def install_apk_to_device(self, target_device):
"""
Install the APK to a target device
@param target_device: The device id of the connected device to install to
"""
if self.is_test_project:
android_package_name = android_support.TEST_RUNNER_PACKAGE_NAME
else:
android_package_name = self.get_android_project_settings(key_name='package_name',
default_value='com.lumberyard.sdk')
if self.clean_deploy and self.check_package_installed(android_package_name, target_device):
logging.info(f"Device '{target_device}': Uninstalling pre-existing APK for {self.game_name} ...")
self.adb_call(arg_list=['uninstall', android_package_name],
device_id=target_device)
logging.info(f"Device '{target_device}': Installing APK for {self.game_name} ...")
self.adb_call(arg_list=['install', '-t', '-r', str(self.apk_path.resolve())],
device_id=target_device)
def install_assets_to_device(self, detected_storage, target_device):
"""
Install the assets for the game to a target device
@param detected_storage: The detected storage path on the target device
@param target_device: The ID of the target device
"""
assert not self.is_test_project
android_package_name = self.get_android_project_settings(key_name='package_name',
default_value='com.lumberyard.sdk')
relative_assets_path = f'Android/data/{android_package_name}/files'
output_target = f'{detected_storage}/{relative_assets_path}'
device_timestamp_file = f'{output_target}/{ANDROID_TARGET_TIMESTAMP_FILENAME}'
# Track the current timestamp if possible to see if we can incrementally push files rather
# than always pushing all files
target_timestamp = self.get_device_file_timestamp(remote_file_path=device_timestamp_file,
device_id=target_device)
if self.clean_deploy:
logging.info(f"Device '{target_device}': Cleaning target assets before deployment...")
self.adb_shell(command=f'rm -rf {output_target}',
device_id=target_device)
logging.info(f"Device '{target_device}': Target cleaned.")
settings_registry_src = self.build_dir / 'app/src/main/assets/Registry'
settings_registry_dst = f'{output_target}/Registry'
if self.clean_deploy or not target_timestamp:
logging.info(f"Device '{target_device}': Pushing {len(self.files_in_asset_path)} files from {str(self.local_asset_path.resolve())} to device ...")
paths_to_deploy = [(str(self.local_asset_path.resolve()), output_target),
(str(settings_registry_src), settings_registry_dst)]
for path_to_deploy, target_path in paths_to_deploy:
try:
self.adb_call(arg_list=['push', str(path_to_deploy), target_path],
device_id=target_device)
except subprocess.CalledProcessError as err:
# Something went wrong, clean up before leaving
self.adb_shell(command=f'rm -rf {output_target}',
device_id=target_device)
raise err
else:
# If no clean was specified, individually inspect all files to see if it needs to be updated
files_to_copy = []
for asset_file in self.files_in_asset_path:
# TODO: Check if the target exists in the destination as well?
if AndroidDeployment.should_copy_file(asset_file, target_timestamp):
files_to_copy.append(asset_file)
if len(files_to_copy) > 0:
logging.info(f"Copying {len(files_to_copy)} assets to device {target_device}")
for src_path in files_to_copy:
relative_path = os.path.relpath(str(src_path), str(self.local_asset_path)).replace('\\', '/')
target_path = f"{output_target}/{relative_path}"
self.adb_call(arg_list=['push', str(src_path), target_path],
device_id=target_device)
# Always update the settings registry
self.adb_call(arg_list=['push', str(settings_registry_src), settings_registry_dst],
device_id=target_device)
self.update_device_file_timestamp(relative_assets_path=output_target,
device_id=target_device)
def execute(self):
"""
Execute the asset deployment
"""
if self.is_test_project:
if not self.apk_path.is_file():
raise common.LmbrCmdError(f"Missing apk for {android_support.TEST_RUNNER_PROJECT} ({str(self.apk_path)}). Make sure it is built and is set as a signed APK.")
else:
if self.embedded_assets or self.deployment_type in (AndroidDeployment.DEPLOY_APK_ONLY, AndroidDeployment.DEPLOY_BOTH):
if not self.apk_path.is_file():
raise common.LmbrCmdError(f"Missing apk for game {self.game_name} ({str(self.apk_path)}). Make sure it is built and is set as a signed APK.")
if not self.embedded_assets or self.deployment_type in (AndroidDeployment.DEPLOY_ASSETS_ONLY, AndroidDeployment.DEPLOY_BOTH):
if not self.local_asset_path.is_dir():
raise common.LmbrCmdError(f"Missing {self.asset_type} assets for game {self.game_name} .")
try:
logging.debug("Starting ADB Server")
self.adb_call('start-server')
self.adb_started = True
# Get the list of target devices to deploy to
target_devices = self.get_target_android_devices()
if not target_devices:
raise common.LmbrCmdError("No connected and eligible android devices found")
for target_device in target_devices:
detected_storage = self.detect_device_storage_path(target_device)
if not detected_storage:
logging.warning(f"Unable to resolve storage path for device '{target_device}'. Skipping.")
continue
if self.is_test_project:
# If this is the unit test runner, then only install the APK, assets are not applicable
self.install_apk_to_device(target_device=target_device)
else:
# Otherwise install the apk and assets based on the deployment type
if self.embedded_assets or self.deployment_type in (AndroidDeployment.DEPLOY_APK_ONLY, AndroidDeployment.DEPLOY_BOTH):
self.install_apk_to_device(target_device=target_device)
if not self.embedded_assets and self.deployment_type in (AndroidDeployment.DEPLOY_ASSETS_ONLY, AndroidDeployment.DEPLOY_BOTH):
if self.deployment_type == AndroidDeployment.DEPLOY_ASSETS_ONLY:
# If we are deploying assets only without an APK, make sure the APK is even installed first
android_package_name = self.get_android_project_settings(key_name='package_name',
default_value='com.lumberyard.sdk')
if not self.check_package_installed(package_name=android_package_name,
target_device=target_device):
raise common.LmbrCmdError(f"Unable to locate APK for {self.game_name} on device '{target_device}'. Make sure it is installed "
f"first before installing the assets.")
self.install_assets_to_device(detected_storage=detected_storage,
target_device=target_device)
logging.info(f"{self.game_name} deployed to device {target_device}")
finally:
if self.adb_started:
self.adb_call('kill-server')