[development] Android compatibility fixes (#5424)

- Added support for NDK r23
 - Added support for the versioned Android 'cmdline-tools' package which replace the deprecated 'tools' package
 - Updated max Gradle version to latest point release of 7.0 to fix volume query bug during sync in some configurations
 - Updated min Android Gradle plugin version to latest point release of 4.2
 - Fixed issue with Gradle task chaining for some custom copy tasks
 - Replaced use of deprecated 'jcenter' repo with 'mavenCentral' in Android project generator template
 - Increased debug logging around adb calls in Android deployment script
 - Fixed a couple try/except cases that were looking for the wrong exception when invoking adb_* functions
 - Fixed issue where asset cache would get deleted when regenerating an existing Android Gradle project with --overwrite-existing
 - Fixed issues with running Android project generation scripts on Unix systems

Signed-off-by: AMZN-ScottR 24445312+AMZN-ScottR@users.noreply.github.com

Code/Tools/Android/ProjectBuilder/root.build.gradle.in

@@ -8,7 +8,7 @@
 buildscript {
     repositories {
         google()
-        jcenter()
+        mavenCentral()
     }
     dependencies {
         classpath 'com.android.tools.build:gradle:${ANDROID_GRADLE_PLUGIN_VERSION}'
@@ -21,7 +21,7 @@ buildscript {
 allprojects {
     repositories {
         google()
-        jcenter()
+        mavenCentral()
     }
 }
 

cmake/3rdParty/Platform/Android/VkValidation_android.cmake

@@ -6,4 +6,12 @@
 #
 #
 
-set(VKVALIDATION_RUNTIME_DEPENDENCIES $<$<NOT:$<CONFIG:Release>>:${LY_NDK_DIR}/sources/third_party/vulkan/src/build-android/jniLibs/arm64-v8a/libVkLayer_khronos_validation.so>)
+set(LY_ANDROID_VULKAN_VALIDATION_PATH "${LY_NDK_DIR}/sources/third_party/vulkan/src/build-android/jniLibs" CACHE PATH "Path to the Vulkan Validation Layers libs for Android")
+
+if(NOT EXISTS ${LY_ANDROID_VULKAN_VALIDATION_PATH})
+    message(FATAL_ERROR
+        "Unable to locate the Android Vulkan validation layer libs at ${LY_ANDROID_VULKAN_VALIDATION_PATH}.  "
+        "If using NDK r23 or above, these libs are distributed separately via https://github.com/KhronosGroup/Vulkan-ValidationLayers")
+endif()
+
+set(VKVALIDATION_RUNTIME_DEPENDENCIES $<$<NOT:$<CONFIG:Release>>:${LY_ANDROID_VULKAN_VALIDATION_PATH}/arm64-v8a/libVkLayer_khronos_validation.so>)

cmake/Platform/Android/Configurations_android.cmake

@@ -12,6 +12,11 @@ if(CMAKE_CXX_COMPILER_ID STREQUAL "Clang")
     include(cmake/Platform/Common/Configurations_common.cmake)
     include(cmake/Platform/Common/Clang/Configurations_clang.cmake)
 
+    set(_android_api_define)
+    if(${LY_TOOLCHAIN_NDK_PKG_MAJOR} VERSION_LESS "23")
+        set(_android_api_define __ANDROID_API__=${LY_TOOLCHAIN_NDK_API_LEVEL})
+    endif()
+
     ly_append_configurations_options(
         DEFINES
             LINUX64
@@ -22,9 +27,9 @@ if(CMAKE_CXX_COMPILER_ID STREQUAL "Clang")
             MOBILE
             _HAS_C9X
             ENABLE_TYPE_INFO
-            __ANDROID_API__=${LY_TOOLCHAIN_NDK_API_LEVEL}
             NDK_REV_MAJOR=${LY_TOOLCHAIN_NDK_PKG_MAJOR}
             NDK_REV_MINOR=${LY_TOOLCHAIN_NDK_PKG_MINOR}
+            ${_android_api_define}
 
         COMPILATION
             -femulated-tls           # All accesses to TLS variables are converted to calls to __emutls_get_address in the runtime library

cmake/Tools/Platform/Android/android_deployment.py

@@ -180,16 +180,20 @@ class AndroidDeployment(object):
             call_arguments.extend(['-s', device_id])
 
         call_arguments.extend(arg_list)
+        logging.debug(f"adb command: {subprocess.list2cmdline(call_arguments)}")
 
         try:
             output = subprocess.check_output(call_arguments,
                                              shell=True,
                                              stderr=subprocess.PIPE).decode(common.DEFAULT_TEXT_READ_ENCODING,
                                                                                common.ENCODING_ERROR_HANDLINGS)
+            logging.debug(f"adb output:\n{output}")
             return output
         except subprocess.CalledProcessError as err:
-            raise common.LmbrCmdError(err.stderr.decode(common.DEFAULT_TEXT_READ_ENCODING,
-                                                        common.ENCODING_ERROR_HANDLINGS))
+            std_out = err.stdout.decode(common.DEFAULT_TEXT_READ_ENCODING, common.ENCODING_ERROR_HANDLINGS)
+            std_err = err.stderr.decode(common.DEFAULT_TEXT_READ_ENCODING, common.ENCODING_ERROR_HANDLINGS)
+            logging.debug(f"adb returned non-zero.\noutput:\n{std_out}\nerror:\n{std_err}\n")
+            raise common.LmbrCmdError(std_err)
 
     def adb_shell(self, command, device_id):
         """
@@ -224,19 +228,15 @@ class AndroidDeployment(object):
 
         shell_command.append(path)
 
-        logging.debug(f"Testing {device_id}: ls {' '.join(shell_command)}")
         raw_output = self.adb_shell(command=' '.join(shell_command),
                                     device_id=device_id)
 
         if not raw_output:
-            logging.debug('adb_ls: No output given')
             return False, None
 
         if raw_output is None or any([error for error in error_messages if error in raw_output]):
-            logging.debug('adb_ls: Error message found')
             status = False
         else:
-            logging.debug('adb_ls: Command was successful')
             status = True
 
         return status, raw_output
@@ -347,7 +347,7 @@ class AndroidDeployment(object):
         try:
             timestamp_string = self.adb_shell(command=f'cat {remote_file_path}',
                                               device_id=device_id).strip()
-        except (subprocess.CalledProcessError, AttributeError):
+        except (common.LmbrCmdError, AttributeError):
             return None
 
         if not timestamp_string:
@@ -463,7 +463,7 @@ class AndroidDeployment(object):
                 try:
                     self.adb_call(arg_list=['push', str(path_to_deploy), target_path],
                                   device_id=target_device)
-                except subprocess.CalledProcessError as err:
+                except common.LmbrCmdError as err:
                     # Something went wrong, clean up before leaving
                     self.adb_shell(command=f'rm -rf {output_target}',
                                    device_id=target_device)

cmake/Tools/Platform/Android/android_support.py

@@ -30,10 +30,11 @@ if ROOT_DEV_PATH not in sys.path:
     sys.path.append(ROOT_DEV_PATH)
 
 from cmake.Tools import common
+from cmake.Tools.layout_tool import remove_link
 
 
 ANDROID_GRADLE_PLUGIN_COMPATIBILITY_MAP = {
-    '4.2.0': {'min_gradle_version': '6.7.1',
+    '4.2.2': {'min_gradle_version': '6.7.1',
               'sdk_build': '30.0.2',
               'default_ndk': '21.4.7075529',
               'min_cmake_version': '3.20'}
@@ -358,7 +359,7 @@ CUSTOM_GRADLE_COPY_NATIVE_CONFIG_FORMAT_STR = """
         into  'outputs/native-lib/{abi}'
     }}
 
-    compile{config}Sources.dependsOn copyNativeLibs{config}
+    merge{config}JniLibFolders.dependsOn copyNativeLibs{config}
 
     copyNativeLibs{config}.mustRunAfter {{
         tasks.findAll {{ task->task.name.contains('externalNativeBuild{config}') }}
@@ -388,7 +389,7 @@ CUSTOM_GRADLE_COPY_REGISTRY_FOLDER_FORMAT_STR = """
         include ('*.setreg')
     }}
 
-    compile{config}Sources.dependsOn copyRegistryFolder{config}
+    merge{config}Assets.dependsOn copyRegistryFolder{config}
 """
 
 CUSTOM_GRADLE_COPY_REGISTRY_FOLDER_DEPENDENCY_FORMAT_STR = """
@@ -470,7 +471,7 @@ class AndroidProjectGenerator(object):
 
     def __init__(self, engine_root, build_dir, android_sdk_path, build_tool, android_sdk_platform, android_native_api_level, android_ndk,
                  project_path, third_party_path, cmake_version, override_cmake_path, override_gradle_path, gradle_version, gradle_plugin_version,
-                 override_ninja_path, include_assets_in_apk, asset_mode, asset_type, signing_config, native_build_path, is_test_project=False,
+                 override_ninja_path, include_assets_in_apk, asset_mode, asset_type, signing_config, native_build_path, vulkan_validation_path, is_test_project=False,
                  overwrite_existing=True, unity_build_enabled=False):
         """
         Initialize the object with all the required parameters needed to create an Android Project. The parameters should be verified before initializing this object
@@ -494,6 +495,8 @@ class AndroidProjectGenerator(object):
         :param asset_mode:
         :param asset_type:
         :param signing_config:          Optional signing configuration arguments
+        :param native_build_path:       Override the native build staging path in gradle
+        :param vulkan_validation_path:  Override the path to where the Vulkan Validation Layers libraries are (required when using NDK r23+)
         :param is_test_project:         Flag to indicate if this is a unit test runner project. (If true, project_path, asset_mode, asset_type, and include_assets_in_apk are ignored)
         :param overwrite_existing:      Flag to overwrite existing project files when being generated, or skip if they already exist.
         """
@@ -534,6 +537,8 @@ class AndroidProjectGenerator(object):
 
         self.native_build_path = native_build_path
 
+        self.vulkan_validation_path = vulkan_validation_path
+
         self.asset_mode = asset_mode
 
         self.asset_type = asset_type
@@ -616,7 +621,7 @@ class AndroidProjectGenerator(object):
         gradle_wrapper_cmd.extend(['wrapper', '-p', str(self.build_dir.resolve())])
 
         proc_result = subprocess.run(gradle_wrapper_cmd,
-                                     shell=True)
+                                     shell=(platform.system() == 'Windows'))
         if proc_result.returncode != 0:
             raise common.LmbrCmdError("Gradle was unable to generate a gradle wrapper for this project (code {}): {}"
                                       .format(proc_result.returncode, proc_result.stderr or ""),
@@ -767,6 +772,8 @@ class AndroidProjectGenerator(object):
         # We must always delete 'src' any existing copied AzAndroid projects since building may pick up stale java sources
         lumberyard_app_src = az_android_dst_path / 'src'
         if lumberyard_app_src.exists():
+            # special case the 'assets' directory before cleaning the whole directory tree
+            remove_link(lumberyard_app_src / 'main' / 'assets')
             common.remove_dir_path(lumberyard_app_src)
 
         logging.debug("Copying AzAndroid to '%s'", az_android_dst_path.resolve())
@@ -817,6 +824,9 @@ class AndroidProjectGenerator(object):
                 f'"-DLY_3RDPARTY_PATH={template_third_party_path}"',
                 f'"-DLY_UNITY_BUILD={template_unity_build}"']
 
+            if self.vulkan_validation_path:
+                cmake_argument_list.append(f'"-DLY_ANDROID_VULKAN_VALIDATION_PATH={pathlib.PurePath(self.vulkan_validation_path).as_posix()}"')
+
             if not self.is_test_project:
                 cmake_argument_list.append(f'"-DLY_PROJECTS={pathlib.PurePath(self.project_path).as_posix()}"')
             else:
@@ -1548,17 +1558,29 @@ class AndroidSDKResolver(object):
             self.version = LooseVersion(available_update_components[1])
             self.available = available_update_components[2]
 
-    def __init__(self, android_sdk_path):
+    def __init__(self, android_sdk_path, command_line_tools_version):
 
         self.android_sdk_path = android_sdk_path or os.environ.get(ANDROID_SDK_ENV_NAME)
         if not self.android_sdk_path:
             raise common.LmbrCmdError(f"Android SDK path not set or it was not passed into the command to generate the android project")
         if not os.path.isdir(self.android_sdk_path):
             raise common.LmbrCmdError(f"Android SDK path {self.android_sdk_path} is not valid")
-        if platform.system() == 'Windows':
-            self.sdk_manager_path = pathlib.Path(self.android_sdk_path) / 'tools' / 'bin' / 'sdkmanager.bat'
+
+        sdk_root = pathlib.Path(self.android_sdk_path)
+
+        tools_path = sdk_root / 'cmdline-tools'
+        if tools_path.exists():
+            tools_path = tools_path / command_line_tools_version
+            if not tools_path.exists():
+                raise common.LmbrCmdError(f"The desired version of the Android 'cmdline-tools' ({command_line_tools_version}) is not detected")
         else:
-            raise common.LmbrCmdError(f"This tool is not supported on the current platform {platform.system()}")
+            tools_path =  sdk_root / 'tools'
+
+        ext = ''
+        if platform.system() == 'Windows':
+            ext = '.bat'
+        self.sdk_manager_path =  tools_path / 'bin' / f'sdkmanager{ext}'
+
         if not self.sdk_manager_path.is_file():
             raise common.LmbrCmdError(f"Android SDK path {self.android_sdk_path} is not valid or complete. Missing {self.sdk_manager_path}")
 

cmake/Tools/Platform/Android/generate_android_project.py

@@ -25,7 +25,7 @@ from cmake.Tools.Platform.Android import android_support
 
 GRADLE_ARGUMENT_NAME = '--gradle-install-path'
 GRADLE_MIN_VERSION = LooseVersion('6.5')
-GRADLE_MAX_VERSION = LooseVersion('7.0.0')
+GRADLE_MAX_VERSION = LooseVersion('7.0.2')
 GRADLE_VERSION_REGEX = re.compile(r"Gradle\s(\d+.\d+.?\d*)")
 GRADLE_EXECUTABLE = 'gradle.bat' if platform.system() == 'Windows' else 'gradle'
 
@@ -102,6 +102,7 @@ def build_optional_signing_profile(store_file, store_password, key_alias, key_pa
 ANDROID_SDK_ARGUMENT_NAME = '--android-sdk-path'
 ANDROID_SDK_PLATFORM_ARGUMENT_NAME = '--android-sdk-platform'
 ANDROID_SDK_PREFERRED_TOOL_VER = '--android-sdk-build-tool-version'
+ANDROID_SDK_COMMAND_LINE_TOOLS_VER = '--android-sdk-command-line-tools-version'
 
 ANDROID_NATIVE_API_LEVEL = '--android-native-api-level'
 
@@ -113,7 +114,7 @@ MIN_NATIVE_API_LEVEL = 24       # The minimum Native API level that is supported
 ANDROID_NDK_PLATFORM_ARGUMENT_NAME = '--android-ndk-version'
 
 ANDROID_GRADLE_PLUGIN_ARGUMENT_NAME = '--gradle-plugin-version'
-ANDROID_GRADLE_MIN_PLUGIN_VERSION = LooseVersion("4.2.0")
+ANDROID_GRADLE_MIN_PLUGIN_VERSION = LooseVersion("4.2.2")
 
 # Constants for asset-related options for APK generation
 INCLUDE_APK_ASSETS_ARGUMENT_NAME = "--include-apk-assets"
@@ -185,6 +186,11 @@ def main(args):
                         default=-1)
 
     # Override arguments
+    parser.add_argument(ANDROID_SDK_COMMAND_LINE_TOOLS_VER,
+                        default='latest',
+                        help='The android SDK command line tools version.',
+                        required=False)
+
     parser.add_argument(ANDROID_SDK_PREFERRED_TOOL_VER,
                         help='The android SDK build tools version.',
                         required=False)
@@ -217,6 +223,11 @@ def main(args):
                         default=None,
                         required=False)
 
+    parser.add_argument('--vulkan-validation-path',
+                        help='Override path to where the Vulkan Validation Layers libraries are.  Required for use with NDK r23+',
+                        default=None,
+                        required=False)
+
     # Asset Options
     parser.add_argument(INCLUDE_APK_ASSETS_ARGUMENT_NAME,
                         action='store_true',
@@ -304,7 +315,8 @@ def main(args):
                                   f"({android_gradle_plugin_version}).")
 
     # Use the SDK Resolver to make sure the build tools and ndk
-    android_sdk = android_support.AndroidSDKResolver(android_sdk_path=parsed_args.get_argument(ANDROID_SDK_ARGUMENT_NAME))
+    android_sdk = android_support.AndroidSDKResolver(android_sdk_path=parsed_args.get_argument(ANDROID_SDK_ARGUMENT_NAME),
+                                                command_line_tools_version=parsed_args.get_argument(ANDROID_SDK_COMMAND_LINE_TOOLS_VER))
 
     # If no SDK platform is provided, check for any installed one
     if android_sdk_platform_version < 0:
@@ -402,7 +414,8 @@ def main(args):
                                                         is_test_project=is_test_project,
                                                         overwrite_existing=parsed_args.overwrite_existing,
                                                         unity_build_enabled=parsed_args.enable_unity_build,
-                                                        native_build_path=parsed_args.native_build_path)
+                                                        native_build_path=parsed_args.native_build_path,
+                                                        vulkan_validation_path=parsed_args.vulkan_validation_path)
     generator.execute()
 
 

cmake/Tools/common.py

@@ -30,6 +30,9 @@ ENCODING_ERROR_HANDLINGS = 'ignore'     # What to do if we encounter any encodin
 DEFAULT_PAK_ROOT = 'Pak'                # The default Pak root folder under engine root where the game paks are built
 
 if platform.system() == 'Windows':
+    class PlatformError(WindowsError):
+        pass
+
     # Re-use microsoft error codes since this script is meant to only run on windows host platforms
     ERROR_CODE_FILE_NOT_FOUND = 2
     ERROR_CODE_ERROR_NOT_SUPPORTED = 50
@@ -37,6 +40,9 @@ if platform.system() == 'Windows':
     ERROR_CODE_CANNOT_COPY = 266
     ERROR_CODE_ERROR_DIRECTORY = 267
 else:
+    class PlatformError(Exception):
+        pass
+
     # Posix does not match any of the following errors to specific codes, so just the standard '1'
     ERROR_CODE_FILE_NOT_FOUND = 1
     ERROR_CODE_ERROR_NOT_SUPPORTED = 1
@@ -309,7 +315,7 @@ def verify_tool(override_tool_path, tool_name, tool_filename, argument_name, too
 
         # Extract the version and verify
         version_output = subprocess.check_output([tool_source, tool_version_argument],
-                                                 shell=True,
+                                                 shell=(platform.system() == 'Windows'),
                                                  stderr=subprocess.PIPE).decode(DEFAULT_TEXT_READ_ENCODING,
                                                                                 ENCODING_ERROR_HANDLINGS)
         version_match = tool_version_regex.search(version_output)
@@ -330,13 +336,13 @@ def verify_tool(override_tool_path, tool_name, tool_filename, argument_name, too
         return result_version, resolved_override_tool_path
 
     except CalledProcessError as e:
-        error_msg = e.output.decode(DEFAULT_TEXT_READ_ENCODING,
+        error_msg = e.stderr.decode(DEFAULT_TEXT_READ_ENCODING,
                                     ENCODING_ERROR_HANDLINGS)
         raise LmbrCmdError(f"{tool_name} cannot be resolved or there was a problem determining its version number. "
                            f"Either make sure its in the system path environment or a valid path is passed in "
                            f"through the {argument_name} argument.\n{error_msg}",
                            ERROR_CODE_ERROR_NOT_SUPPORTED)
-    except (WindowsError, RuntimeError) as e:
+    except (PlatformError, RuntimeError) as e:
         logging.error(f"Call to '{tool_source}' resulted in error: {e}")
         raise LmbrCmdError(f"{tool_name} cannot be resolved or there was a problem determining its version number. "
                            f"Either make sure its in the system path environment or a valid path is passed in "
@@ -552,7 +558,7 @@ class CommandLineExec(object):
                 call_args.append(str(arguments))
             logging.debug("exec(%s)", subprocess.list2cmdline(call_args))
             result = subprocess.run(call_args,
-                                    shell=True,
+                                    shell=(platform.system() == 'Windows'),
                                     capture_output=capture_stdout,
                                     stderr=subprocess.DEVNULL if not capture_stdout and suppress_stderr else None,
                                     encoding='utf-8',