Integrating up through commit 90f050496

2021-04-07 14:03:29 -07:00
parent 8f2ed080a9
commit c2cbd430fe
2694 changed files with 285622 additions and 176874 deletions
@@ -0,0 +1,44 @@
+@ECHO OFF
+REM
+REM All or portions of this file Copyright (c) Amazon.com, Inc. or its affiliates or
+REM its licensors.
+REM
+REM For complete copyright and license terms please see the LICENSE at the root of this
+REM distribution (the "License"). All use of this software is governed by the License,
+REM or, if provided, by the license below or the license accompanying this file. Do not
+REM remove or modify any license notices. This file is distributed on an "AS IS" BASIS,
+REM WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+REM
+
+SETLOCAL EnableDelayedExpansion
+
+CALL %~dp0gradle_windows.cmd
+IF NOT %ERRORLEVEL%==0 GOTO :error
+
+IF NOT EXIST "%LY_3RDPARTY_PATH%" (
+    ECHO [ci_build] LY_3RDPARTY_PATH is invalid or not set
+    GOTO :error
+)
+
+IF NOT EXIST "%LY_ANDROID_SDK%" (
+    SET LY_ANDROID_SDK=!LY_3RDPARTY_PATH!/android-sdk/platform-29
+)
+IF NOT EXIST "%LY_ANDROID_SDK%" (
+    ECHO [ci_build] FAIL: LY_ANDROID_SDK=!LY_ANDROID_SDK!
+    GOTO :error
+)
+
+SET PYTHON=python\python.cmd
+ECHO [ci_build] %PYTHON% Tools\build\JenkinsScripts\build\Platform\Android\run_test_on_android_simulator.py --android-sdk-path %LY_ANDROID_SDK% --build-path %OUTPUT_DIRECTORY% --build-config %CONFIGURATION%
+CALL %PYTHON% Tools\build\JenkinsScripts\build\Platform\Android\run_test_on_android_simulator.py --android-sdk-path %LY_ANDROID_SDK% --build-path %OUTPUT_DIRECTORY% --build-config %CONFIGURATION%
+IF NOT %ERRORLEVEL%==0 GOTO :popd_error
+
+EXIT /b 0
+
+:popd_error
+POPD
+
+:error
+ECHO ERROR
+
+EXIT /b 1
@@ -141,7 +141,28 @@
      "OUTPUT_DIRECTORY":"build\\android_gradle",
      "GAME_PROJECT": "AutomatedTesting",
      "ANDROID_NDK_PLATFORM": "21",
-      "ANDROID_SDK_PLATFORM": "29"
+      "ANDROID_SDK_PLATFORM": "29",
+      "SIGN_APK": "false",
+      "GRADLE_BUILD_CMD": "build",
+      "ADDITIONAL_GENERATE_ARGS": ""
+    }
+  },
+  "periodic_test_profile": {
+    "TAGS":[
+        "nightly",
+        "weekly-build-metrics"
+    ],
+    "COMMAND":"build_and_run_unit_tests.cmd",
+    "PARAMETERS": {
+      "CONFIGURATION":"profile",
+      "OUTPUT_DIRECTORY":"build\\android_unittest",
+      "GAME_PROJECT": "AutomatedTesting",
+      "ANDROID_NDK_PLATFORM": "21",
+      "ANDROID_SDK_PLATFORM": "29",
+      "SIGN_APK": "true",
+      "GRADLE_BUILD_CMD": "assemble",
+      "ADDITIONAL_GENERATE_ARGS": "--unit-test"
    }
  }
+
 }
@@ -10,6 +10,8 @@ REM remove or modify any license notices. This file is distributed on an "AS IS"
 REM WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 REM

+SETLOCAL EnableDelayedExpansion
+
 IF NOT EXIST "%LY_3RDPARTY_PATH%" (
    ECHO [ci_build] LY_3RDPARTY_PATH is invalid or not set
    GOTO :error
@@ -40,19 +42,27 @@ IF NOT EXIST "%LY_NINJA_PATH%" (
    GOTO :error
 )

+REM Make sure that Ninja in the Path variable
+ECHO Testing ninja on the current path variable
+call ninja --version
+IF %ERRORLEVEL%==0 GOTO ninja_on_path
+ECHO Ninja wasnt in the call path, add the value set by LY_NINJA_PATH 
+SET PATH=%PATH%;%LY_NINJA_PATH%
+
+:ninja_on_path
 IF NOT EXIST "%LY_ANDROID_SDK%" (
-    SET LY_ANDROID_SDK=%LY_3RDPARTY_PATH%/android-sdk/platform-29
+    SET LY_ANDROID_SDK=!LY_3RDPARTY_PATH!/android-sdk/platform-29
 )
 IF NOT EXIST "%LY_ANDROID_SDK%" (
-    ECHO [ci_build] FAIL: LY_ANDROID_SDK=%LY_ANDROID_SDK%
+    ECHO [ci_build] FAIL: LY_ANDROID_SDK=!LY_ANDROID_SDK!
    GOTO :error
 )

 IF NOT EXIST "%LY_ANDROID_NDK%" (
-    set LY_ANDROID_NDK=%LY_3RDPARTY_PATH%/android-ndk/r21d
+    set LY_ANDROID_NDK=!LY_3RDPARTY_PATH!/android-ndk/r21d
 )
 IF NOT EXIST "%LY_ANDROID_NDK%" (
-    ECHO [ci_build] LY_ANDROID_NDK=%LY_ANDROID_NDK%
+    ECHO [ci_build] LY_ANDROID_NDK=!LY_ANDROID_NDK!
    GOTO :error
 )

@@ -61,7 +71,7 @@ IF "%CLEAN_OUTPUT_DIRECTORY%"=="true" (
    IF EXIST %OUTPUT_DIRECTORY% (
        ECHO [ci_build] CLEAN_OUTPUT_DIRECTORY option set with value "%CLEAN_OUTPUT_DIRECTORY%"
        ECHO [ci_build] Deleting "%OUTPUT_DIRECTORY%"
-        DEL /s /q /f %OUTPUT_DIRECTORY%
+        DEL /s /q /f %OUTPUT_DIRECTORY% 1>nul
    )
 )

@@ -73,27 +83,134 @@ REM Jenkins reports MSB8029 when TMP/TEMP is not defined, define a dummy folder
 SET TMP=%cd%/temp
 SET TEMP=%cd%/temp
 IF NOT EXIST %TMP% (
-    mkdir temp
+    mkdir %TMP%
 )
-SET PYTHON=python\python.cmd
-ECHO [ci_build] %PYTHON% cmake\Tools\Platform\Android\generate_android_project.py --project-path=%GAME_PROJECT% --engine-root=. --build-dir=%OUTPUT_DIRECTORY% --gradle-install-path=%GRADLE_HOME% --cmake-install-path=%CMAKE_HOME% --ninja-install-path=%LY_NINJA_PATH% --third-party-path=%LY_3RDPARTY_PATH% --android-ndk-path=%LY_ANDROID_NDK% --android-sdk-path=%LY_ANDROID_SDK% --android-ndk-version=%ANDROID_NDK_PLATFORM% --android-sdk-version=%ANDROID_SDK_PLATFORM%
-CALL %PYTHON% cmake\Tools\Platform\Android\generate_android_project.py --project-path="%GAME_PROJECT%" --engine-root=. --build-dir="%OUTPUT_DIRECTORY%" --gradle-install-path="%GRADLE_HOME%" --cmake-install-path="%CMAKE_HOME%" --ninja-install-path="%LY_NINJA_PATH%" --third-party-path="%LY_3RDPARTY_PATH%" --android-ndk-path="%LY_ANDROID_NDK%" --android-sdk-path="%LY_ANDROID_SDK%" --android-ndk-version=%ANDROID_NDK_PLATFORM% --android-sdk-version=%ANDROID_SDK_PLATFORM%
-IF NOT %ERRORLEVEL%==0 GOTO :error

+REM Optionally sign the APK if we are generating an APK 
+SET GENERATE_SIGNED_APK=false
+IF "%SIGN_APK%"=="true" (
+    IF "%GRADLE_BUILD_CMD%"=="assemble" (
+        SET GENERATE_SIGNED_APK=true
+    )
+)
+
+SET PYTHON=python\python.cmd
+
+REM Regardless of whether or not we generate a signing key, apparently we must set variables outside of 
+REM an IF clause otherwise it will not work.
+
+REM First look for the JDK HOME in the environment variable
+IF EXIST "%JDK_HOME%" (
+    ECHO JDK Home found in Environment: !JDK_HOME!
+    GOTO JDK_FOUND
+)
+
+REM Next, look in the registry
+FOR /F "skip=2 tokens=1,2*" %%A IN ('REG QUERY "HKEY_LOCAL_MACHINE\SOFTWARE\JavaSoft\Java Development Kit\1.8" /v "JavaHome" 2^>nul') DO (
+    SET JDK_REG_VALUE=%%C
+)
+IF EXIST "%JDK_REG_VALUE%" (
+    SET JDK_HOME=!JDK_REG_VALUE!
+    ECHO JDK Home found in registry: !JDK_HOME!
+    GOTO JDK_FOUND
+)
+
+ECHO Unable to locate JDK_HOME
+GOTO error
+
+:JDK_FOUND
+
+SET JDK_BIN=%JDK_HOME%\bin
+IF NOT EXIST "%JDK_BIN%" (
+    ECHO The environment variable JDK_HOME is not set to a valid JDK 1.8 folder %JDK_BIN%
+    ECHO Make sure the variable is set to your local JDK 1.8 installation
+    GOTO error
+)
+
+SET KEYTOOL_PATH=%JDK_BIN%\keytool.exe
+IF NOT EXIST "%KEYTOOL_PATH%" (
+    ECHO The environment variable JDK_HOME is not set to a valid JDK 1.8 folder. Cannot find keytool at %JDK_BIN%\keytool.exe
+    ECHO Make sure the variable is set to your local JDK 1.8 installation
+    GOTO error
+)
+
+SET CI_ANDROID_KEYSTORE_FILE=ly-android-dev.keystore
+SET CI_ANDROID_KEYSTORE_ALIAS=ly-android
+SET CI_ANDROID_KEYSTORE_PASSWORD=lumberyard
+SET CI_ANDROID_KEYSTORE_DN=cn=LY Developer, ou=Lumberyard, o=Amazon, c=US
+SET CI_KEYSTORE_VALIDITY_DAYS=10000
+SET CI_KEYSTORE_CERT_DN=cn=LY Developer, ou=Lumberyard, o=Amazon, c=US
+
+REM Clear out any existing keystore file since the password/alias may have changed
+SET CI_ANDROID_KEYSTORE_FILE_ABS=%cd%\%OUTPUT_DIRECTORY%\%CI_ANDROID_KEYSTORE_FILE%
+
+
+IF "%GENERATE_SIGNED_APK%"=="true" (
+
+    REM Prepare a temporary keystore just for this unit test session
+
+    REM Generate the keystore file if needed
+    IF NOT EXIST "%CI_ANDROID_KEYSTORE_FILE_ABS%" (
+
+        ECHO [ci_build] Generating keystore file %CI_ANDROID_KEYSTORE_FILE%
+        ECHO [ci_build] "%KEYTOOL_PATH%" -genkeypair -v -keystore %CI_ANDROID_KEYSTORE_FILE_ABS% -storepass %CI_ANDROID_KEYSTORE_PASSWORD% -alias %CI_ANDROID_KEYSTORE_ALIAS% -keypass %CI_ANDROID_KEYSTORE_PASSWORD% -keyalg RSA -keysize 2048 -validity %CI_KEYSTORE_VALIDITY_DAYS% -dname "%CI_KEYSTORE_CERT_DN%"
+        CALL "%KEYTOOL_PATH%" -genkeypair -v -keystore %CI_ANDROID_KEYSTORE_FILE_ABS% -storepass %CI_ANDROID_KEYSTORE_PASSWORD% -alias %CI_ANDROID_KEYSTORE_ALIAS% -keypass %CI_ANDROID_KEYSTORE_PASSWORD% -keyalg RSA -keysize 2048 -validity %CI_KEYSTORE_VALIDITY_DAYS% -dname "%CI_KEYSTORE_CERT_DN%" 2> nul 
+        IF errorlevel 1 (
+            ECHO Unable to generate keystore file "%CI_ANDROID_KEYSTORE_FILE_ABS%"
+            GOTO error
+        )
+    ) ELSE (
+        ECHO Using keystore file at %CI_ANDROID_KEYSTORE_FILE_ABS%
+    )
+
+    ECHO [ci_build] %PYTHON% cmake\Tools\Platform\Android\generate_android_project.py --engine-root=. --build-dir=%OUTPUT_DIRECTORY% -g %GAME_PROJECT% --gradle-install-path=%GRADLE_HOME% --cmake-install-path=%CMAKE_HOME% --ninja-install-path=%LY_NINJA_PATH% --third-party-path=%LY_3RDPARTY_PATH% --android-ndk-path=%LY_ANDROID_NDK% --android-sdk-path=%LY_ANDROID_SDK% --android-ndk-version=%ANDROID_NDK_PLATFORM% --android-sdk-version=%ANDROID_SDK_PLATFORM% --signconfig-store-file %CI_ANDROID_KEYSTORE_FILE_ABS% --signconfig-store-password %CI_ANDROID_KEYSTORE_PASSWORD% --signconfig-key-alias %CI_ANDROID_KEYSTORE_ALIAS% --signconfig-key-password %CI_ANDROID_KEYSTORE_PASSWORD% %OPTIONAL_TEST_FLAG% %ADDITIONAL_GENERATE_ARGS% --overwrite-existing
+    CALL %PYTHON% cmake\Tools\Platform\Android\generate_android_project.py --engine-root=. --build-dir=%OUTPUT_DIRECTORY% -g %GAME_PROJECT% --gradle-install-path=%GRADLE_HOME% --cmake-install-path=%CMAKE_HOME% --ninja-install-path=%LY_NINJA_PATH% --third-party-path=%LY_3RDPARTY_PATH% --android-ndk-path=%LY_ANDROID_NDK% --android-sdk-path=%LY_ANDROID_SDK% --android-ndk-version=%ANDROID_NDK_PLATFORM% --android-sdk-version=%ANDROID_SDK_PLATFORM% --signconfig-store-file %CI_ANDROID_KEYSTORE_FILE_ABS% --signconfig-store-password %CI_ANDROID_KEYSTORE_PASSWORD% --signconfig-key-alias %CI_ANDROID_KEYSTORE_ALIAS% --signconfig-key-password %CI_ANDROID_KEYSTORE_PASSWORD% %ADDITIONAL_GENERATE_ARGS% --overwrite-existing
+) ELSE (
+    ECHO [ci_build] %PYTHON% cmake\Tools\Platform\Android\generate_android_project.py --engine-root=. --build-dir=%OUTPUT_DIRECTORY% -g %GAME_PROJECT% --gradle-install-path=%GRADLE_HOME% --cmake-install-path=%CMAKE_HOME% --ninja-install-path=%LY_NINJA_PATH% --third-party-path=%LY_3RDPARTY_PATH% --android-ndk-path=%LY_ANDROID_NDK% --android-sdk-path=%LY_ANDROID_SDK% --android-ndk-version=%ANDROID_NDK_PLATFORM% --android-sdk-version=%ANDROID_SDK_PLATFORM% %ADDITIONAL_GENERATE_ARGS% --overwrite-existing
+    CALL %PYTHON% cmake\Tools\Platform\Android\generate_android_project.py --engine-root=. --build-dir=%OUTPUT_DIRECTORY% -g %GAME_PROJECT% --gradle-install-path=%GRADLE_HOME% --cmake-install-path=%CMAKE_HOME% --ninja-install-path=%LY_NINJA_PATH% --third-party-path=%LY_3RDPARTY_PATH% --android-ndk-path=%LY_ANDROID_NDK% --android-sdk-path=%LY_ANDROID_SDK% --android-ndk-version=%ANDROID_NDK_PLATFORM% --android-sdk-version=%ANDROID_SDK_PLATFORM% %ADDITIONAL_GENERATE_ARGS%  --overwrite-existing
+)
+
+REM Validate the android project generation
+IF %ERRORLEVEL%==0 GOTO generate_project_success
+ECHO Error Generating Android Project
+goto error
+
+:generate_project_success
+
+REM Run the gradle build from the output directory
 PUSHD %OUTPUT_DIRECTORY%

 REM Stop any running or orphaned gradle daemon 
 ECHO [ci_build] gradlew --stop
-gradlew --stop
+CALL gradlew --stop

-ECHO [ci_build] gradlew --no-daemon -build%CONFIGURATION%
-gradlew --no-daemon build%CONFIGURATION%
-IF NOT %ERRORLEVEL%==0 GOTO :popd_error
+ECHO [ci_build] gradlew --no-daemon %GRADLE_BUILD_CMD%%CONFIGURATION%
+CALL gradlew --no-daemon %GRADLE_BUILD_CMD%%CONFIGURATION%
+
+IF %ERRORLEVEL%==0 GOTO gradle_build_success
+
+REM Do another build with the debug flag to try to get the failure reasons
+
+GOTO error
+
+ECHO Error building gradle. Rebuilding with debug information
+ECHO [ci_build] gradlew --debug --full-stacktrace --no-daemon %GRADLE_BUILD_CMD%%CONFIGURATION%
+CALL gradlew --debug --full-stacktrace --no-daemon %GRADLE_BUILD_CMD%%CONFIGURATION%
+
+ECHO [ci_build] gradlew --stop
+CALL gradlew --stop

 POPD

+GOTO error
+
+
+:gradle_build_success
+
 ECHO [ci_build] gradlew --stop
-gradlew --stop
+CALL gradlew --stop
+
+POPD

 EXIT /b 0

@@ -102,7 +219,4 @@ POPD

 :error

-ECHO [ci_build] gradlew --stop
-gradlew --stop
-
 EXIT /b 1
@@ -1,7 +1,7 @@
 {
    "ENV": {
        "GRADLE_HOME": "C:/Gradle/gradle-5.6.4",
-        "NODE_LABEL": "windows",
+        "NODE_LABEL": "windows-047e5cdf",
        "LY_3RDPARTY_PATH": "C:/ly/3rdParty",
        "TIMEOUT": 30,
        "WORKSPACE": "D:/workspace",
@@ -10,6 +10,9 @@
    "PIPELINE_ENV_OVERRIDE": {
        "daily-pipeline-metrics": {
            "CLEAN_WORKSPACE": true
+        },
+        "packaging": {
+            "CLEAN_WORKSPACE": true
        }
    }
 }
@@ -0,0 +1,544 @@
+#
+# All or portions of this file Copyright (c) Amazon.com, Inc. or its affiliates or
+# its licensors.
+#
+# For complete copyright and license terms please see the LICENSE at the root of this
+# distribution (the "License"). All use of this software is governed by the License,
+# or, if provided, by the license below or the license accompanying this file. Do not
+# remove or modify any license notices. This file is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#
+
+import argparse
+import os
+import pathlib
+import re
+import sys
+import subprocess
+import time
+import logging
+
+CURRENT_PATH = pathlib.Path(os.path.dirname(__file__)).absolute()
+
+ENGINE_ROOT = CURRENT_PATH.parent.parent.parent.parent.parent.parent
+
+
+class AndroidEmuError(Exception):
+    pass
+
+
+def get_android_sdk_path():
+    try:
+        android_sdk_path = pathlib.Path(os.getenv('LY_ANDROID_SDK'))
+        if not android_sdk_path:
+            raise AndroidEmuError(f"LY_ANDROID_SDK environment variable is not set")
+        if not android_sdk_path.is_dir():
+            raise AndroidEmuError(f"Android SDK Path ('{android_sdk_path}') set with the LY_ANDROID_SDK variable is invalid")
+        #TODO: Sanity check on necessary files
+        return android_sdk_path
+    except Exception as err:
+        raise AndroidEmuError(f"Unable to determine android SDK path: {err}")
+
+
+class Command(object):
+
+    def __init__(self, tool_name, tool_path, run_as_shell=True):
+
+        if not tool_path.is_file():
+            raise AndroidEmuError(f"Invalid path for {tool_name}. Cannot find ('{tool_path.absolute()}')")
+
+        self.tool_path = tool_path
+        self.run_as_shell = run_as_shell
+
+    def run_return_output(self, cmd_args):
+
+        args = [str(self.tool_path)]
+        if isinstance(cmd_args, str):
+            args.append(cmd_args)
+        elif isinstance(cmd_args, list):
+            args.extend(cmd_args)
+        else:
+            assert False, "run_return_output argument must be a string or list of strings"
+
+        full_cmd = subprocess.list2cmdline(args)
+        logging.debug(f"run_return_output: {full_cmd}")
+
+        run_result = subprocess.run(args,
+                                    capture_output=True,
+                                    encoding='UTF-8',
+                                    errors='ignore',
+                                    shell=self.run_as_shell)
+        if run_result.returncode != 0:
+            raise AndroidEmuError(f"Error executing command '{full_cmd}' (return code {run_result.returncode}): {run_result.stderr}")
+
+        return run_result.stdout
+
+    def run(self, cmd_args, cwd=None, suppress_output=False):
+        args = [str(self.tool_path)]
+        if isinstance(cmd_args, str):
+            args.append(cmd_args)
+        elif isinstance(cmd_args, list):
+            args.extend(cmd_args)
+        else:
+            assert False, "run_return_output argument must be a string or list of strings"
+
+        full_cmd = subprocess.list2cmdline(args)
+        logging.debug(f"run: {full_cmd}")
+
+        run_result = subprocess.run(args,
+                                    #stdout=subprocess.DEVNULL if suppress_output else subprocess.STDOUT,
+                                    capture_output=False,
+                                    shell=self.run_as_shell,
+                                    cwd=cwd)
+        if run_result.returncode != 0:
+            raise AndroidEmuError(f"Error executing command '{full_cmd}' (return code {run_result.returncode}): {run_result.stderr}")
+
+    def run_process(self, cmd_args):
+        args = [str(self.tool_path)]
+        if isinstance(cmd_args, str):
+            args.append(cmd_args)
+        elif isinstance(cmd_args, list):
+            args.extend(cmd_args)
+        else:
+            assert False, "run_return_output argument must be a string or list of strings"
+
+        full_cmd = subprocess.list2cmdline(args)
+        logging.debug(f"run_process: {full_cmd}")
+
+        process = subprocess.Popen(args,
+                                   shell=True,
+                                   stdout=subprocess.PIPE,
+                                   creationflags=subprocess.CREATE_NEW_PROCESS_GROUP | subprocess.NORMAL_PRIORITY_CLASS |
+                                                 subprocess.CREATE_NO_WINDOW,
+                                   encoding='UTF-8',
+                                   errors='ignore')
+        return process
+
+
+class AndroidEmulatorManager(object):
+
+    UNIT_TEST_AVD_NAME = "LY_UNITTEST_AVD"
+
+    UNIT_TEST_SYSTEM_IMAGE_PACKAGE = "android-30;google_apis;x86_64"
+
+    UNIT_TEST_DEVICE_TEMPLATE_NAME = "pixel_xl"
+
+    UNIT_TEST_DEVICE_SETTINGS_MAP = {
+        "disk.dataPartition.size": "32G",
+        "vm.heapSize": "1024",
+        "hw.ramSize": "2048",
+        "hw.sdCard": "no"
+    }
+    EMULATOR_STARTUP_TIMEOUT_SECS = 60*5  # Set the emulator startup timeout to 5 minutes
+
+    def __init__(self, base_android_sdk_path, hide_emulator_windows=True, force_avd_creation=False, emulator_startup_timeout=EMULATOR_STARTUP_TIMEOUT_SECS):
+
+        self.android_sdk_path = base_android_sdk_path
+
+        self.force_avd_creation = force_avd_creation
+
+        self.unit_test_avd_name = AndroidEmulatorManager.UNIT_TEST_AVD_NAME
+        self.unit_test_device_template_name = AndroidEmulatorManager.UNIT_TEST_DEVICE_TEMPLATE_NAME
+        self.unit_test_device_settings_map = AndroidEmulatorManager.UNIT_TEST_DEVICE_SETTINGS_MAP
+        self.unit_test_avd_system_image = AndroidEmulatorManager.UNIT_TEST_SYSTEM_IMAGE_PACKAGE
+        self.hide_emulator_windows = hide_emulator_windows
+
+        self.emulator_startup_timeout = emulator_startup_timeout
+        self.emulator_cmd = Command("Emulator", self.android_sdk_path / 'emulator' / 'emulator.exe')
+
+        self.avd_manager_cmd = Command("AVD Manager", self.android_sdk_path / 'tools' / 'bin' / 'avdmanager.bat')
+
+        self.sdk_manager_cmd = Command("SDK Manager", self.android_sdk_path / 'tools' / 'bin' / 'sdkmanager.bat')
+
+        self.adb_cmd = Command("ADB", self.android_sdk_path / 'platform-tools' / 'adb.exe')
+
+    def collect_android_sdk_list(self):
+        """
+        Use the SDK Manager to get the list of installed, available, and updateable packages
+        :return:  tuple of 3 lists: installed, available, and updateable packages
+        """
+
+        result_str = self.sdk_manager_cmd.run_return_output(['--list'])
+
+        # the result will be listed out in 3 sections: Installed packages, Available Packages, and Available updates
+        # and each item is represented by 3 columns separated by a '|' character
+        installed_packages = []
+        available_packages = []
+        available_updates = []
+
+        current_append_list = None
+        for avd_item in result_str.split('\n'):
+            avd_item_stripped = avd_item.strip()
+            if not avd_item_stripped:
+                continue
+            if '|' not in avd_item_stripped:
+                if avd_item_stripped.upper() == 'INSTALLED PACKAGES:':
+                    current_append_list = installed_packages
+                elif avd_item_stripped.upper() == 'AVAILABLE PACKAGES:':
+                    current_append_list = available_packages
+                elif avd_item_stripped.upper() == 'AVAILABLE UPDATES:':
+                    current_append_list = available_updates
+                else:
+                    current_append_list = None
+                continue
+            item_parts = [split.strip() for split in avd_item_stripped.split('|')]
+            if len(item_parts) < 3:
+                continue
+            elif item_parts[1].upper() in ('VERSION', 'INSTALLED', '-------'):
+                continue
+            elif current_append_list is None:
+                continue
+
+            if current_append_list is not None:
+                current_append_list.append(item_parts)
+
+        return installed_packages, available_packages, available_updates
+
+    def install_system_package_if_necessary(self):
+        """
+        Make sure that we have the correct system image installed, and install if not
+        """
+
+        installed_packages, available_packages, _ = self.collect_android_sdk_list()
+
+        unit_test_sdk_package_name = f'system-images;{self.unit_test_avd_system_image}'
+        detected_sdk_package_version = None
+        for package_line_items in installed_packages:
+            if package_line_items[0] == unit_test_sdk_package_name:
+                detected_sdk_package_version = package_line_items[0]
+
+        if detected_sdk_package_version:
+            # Already installed
+            logging.info(f"Detected installed system image {self.unit_test_avd_system_image} version {detected_sdk_package_version}")
+            return
+
+        # Make sure its an available image to install
+        detected_available_sdk_package_version = None
+        for package_line_items in available_packages:
+            if package_line_items[0] == unit_test_sdk_package_name:
+                detected_available_sdk_package_version = package_line_items[0]
+        if not detected_available_sdk_package_version:
+            raise AndroidEmuError(f"Unable to install required system image {self.unit_test_avd_system_image}, not found by the Android SDK Manager")
+
+        # Install the package
+        logging.info(f"Installing system image {self.unit_test_avd_system_image}...")
+        self.sdk_manager_cmd.run(['--install', unit_test_sdk_package_name])
+        logging.info(f"Installed Completed")
+
+    def find_device_id_by_name(self, device_name):
+        """
+        Find a device id (from AVD Manager) by the device name
+        :param device_name: Name to lookup
+        :return: The device id
+        """
+        result_str = self.avd_manager_cmd.run_return_output(['list', 'device'])
+        result_lines = [result_line.strip() for result_line in result_str.split('\n')]
+
+        result_line_count = len(result_lines)
+        current_index = 0
+
+        device_to_id_map = {}
+
+        while current_index < result_line_count:
+
+            current_line = result_lines[current_index]
+            current_index += 1
+
+            # This assumes the pattern "id: <id> or "<device name>"
+            if current_line.startswith('id:') and 'or' in current_line:
+                id_and_name_combo = current_line.split('or')
+                id_and_value_combo = id_and_name_combo[0].split(' ')
+                name = id_and_name_combo[1].replace('"', '').strip().upper()
+                id = id_and_value_combo[1]
+                device_to_id_map[name] = id
+
+            if current_line.startswith('Available Android targets:'):
+                break
+
+        device_id = device_to_id_map.get(device_name.upper())
+        if not device_id:
+            raise AndroidEmuError(f"Unable to locate device id for '{device_name}'")
+        return device_id
+
+    def query_installed_avds(self):
+        """
+        Get maps of all valid and invalid AVDs installed on the current system
+        :return: tuple of 2 maps (AVD Name -> Path): Valid and invalid
+        """
+        result_str = self.avd_manager_cmd.run_return_output(['list', 'avd'])
+        result_lines = [result_line.strip() for result_line in result_str.split('\n')]
+
+        line_count = len(result_lines)
+        current_index = 0
+        current_name = None
+        current_path = None
+
+        valid_avd_to_path_map = {}
+        invalid_avd_to_path_map = {}
+        current_avd_to_path_map = valid_avd_to_path_map
+
+        while current_index < line_count:
+            current_line = result_lines[current_index]
+            current_index += 1
+
+            if current_line.startswith('Name:'):
+                name = current_line[6:].strip()
+                if current_name is not None:
+                    current_avd_to_path_map[current_name] = current_path
+                current_path = None
+                current_name = name
+
+            elif current_line.startswith('Path:'):
+                current_path = current_line[6:].strip()
+            elif current_line.startswith('Device:'):
+                pass
+            elif 'could not be loaded:' in current_line:
+                if current_name is not None:
+                    current_avd_to_path_map[current_name] = current_path
+                current_avd_to_path_map = invalid_avd_to_path_map
+                current_path = None
+                current_name = None
+
+        if current_name is not None:
+            current_avd_to_path_map[current_name] = current_path
+
+        return valid_avd_to_path_map, invalid_avd_to_path_map
+
+    def create_unitest_avd(self):
+        """Create the unit test AVD"""
+
+        self.install_system_package_if_necessary()
+
+        device_id = self.find_device_id_by_name(self.unit_test_device_template_name)
+
+        self.avd_manager_cmd.run(['--silent',
+                                  'create', 'avd',
+                                  '--name', self.unit_test_avd_name,
+                                  '--package', f'system-images;{self.unit_test_avd_system_image}',
+                                  '--device', device_id])
+
+        valid_avd_map, _ = self.query_installed_avds()
+        unit_test_avd_path = valid_avd_map.get(self.unit_test_avd_name)
+        if not unit_test_avd_path:
+            raise AndroidEmuError(f"Unable to create unit test AVD {self.unit_test_avd_name}")
+
+        unit_test_avd_config_path = pathlib.Path(unit_test_avd_path) / 'config.ini'
+        if not unit_test_avd_config_path.is_file():
+            raise AndroidEmuError(f"Unable to create unit test AVD {self.unit_test_avd_name}: The expected config file '{unit_test_avd_config_path}' does not exist.")
+
+        config_content_full = unit_test_avd_config_path.read_text(encoding='UTF-8', errors='ignore')
+        for item, value in self.unit_test_device_settings_map.items():
+            regex_friendly_str = item.replace('.', '\\.')
+            repl_pattern = f"{regex_friendly_str}\\s*=\\s*[\\d]+"
+            repl_value = f"{item}={value}"
+            if re.search(repl_pattern, config_content_full):
+                config_content_full = re.sub(repl_pattern, repl_value, config_content_full)
+            else:
+                if not config_content_full.endswith('\n'):
+                    config_content_full += '\n'
+                config_content_full += f"{repl_value}\n"
+
+        unit_test_avd_config_path.write_text(config_content_full)
+
+    def query_emulator_device_id(self):
+        result_str = self.adb_cmd.run_return_output(['devices', '-l'])
+        emulators = []
+        for result_line in result_str.split('\n'):
+            if not result_line.startswith('emulator-'):
+                continue
+            emulator = result_line[:result_line.find(' ')].strip()
+            emulators.append(emulator)
+
+        if len(emulators) > 1:
+            logging.warning(f"Found multiple emulators connect ({','.join(emulators)}). Defaulting to {emulators[0]}")
+
+        return emulators[0] if len(emulators) > 0 else None
+
+    def install_unit_test_avd(self):
+        """
+        Install the unit test AVD (Android Virtual Device)
+        """
+
+        valid_avd_map, invalid_avd_map = self.query_installed_avds()
+
+        if not self.unit_test_avd_name in valid_avd_map:
+            create_avd = True
+        elif self.force_avd_creation or self.unit_test_avd_name in invalid_avd_map:
+            logging.info(f"Deleting AVD {self.unit_test_avd_name}..")
+            self.avd_manager_cmd.run(['delete', 'avd', '--name', self.unit_test_avd_name])
+            create_avd = True
+        else:
+            create_avd = False
+
+        if create_avd:
+            self.create_unitest_avd()
+
+    def uninstall_unit_test_avd(self):
+        """
+        Uninstall the unit test AVD
+        """
+        logging.info(f"Uninstalling AVD {self.unit_test_avd_name}..")
+        self.avd_manager_cmd.run(['delete', 'avd', '--name', self.unit_test_avd_name])
+
+    def launch_emulator_process(self):
+        """
+        Launch the emulator process for the unit test avd and return the process handle and its device id
+        :return: tuple of the process handle and the device id for the emulator
+        """
+        emulator_device_id = None
+        process = None
+        try:
+            # Launch the emulator process
+            emulator_process_args = [
+                "-avd",
+                self.unit_test_avd_name
+            ]
+            if self.hide_emulator_windows:
+                emulator_process_args.append("-no-window")
+            process = self.emulator_cmd.run_process(emulator_process_args)
+
+            # Wait for the emulator to signal that its bootup is complete
+            boot_completed = False
+            start_time = time.time()
+            timeout_secs = 360
+            while process.poll() is None:
+                elapsed_time = time.time() - start_time
+                if elapsed_time > timeout_secs > 0:
+                    break
+                line = process.stdout.readline()
+                print(line, end='')
+                if "boot completed" in line:
+                    boot_completed = True
+                    break
+            if not boot_completed:
+                raise AndroidEmuError("Bootup of emulator timed out")
+
+            # query ADB to get the emulator ID
+            emulator_device_id = self.query_emulator_device_id()
+
+            return process, emulator_device_id
+
+        except Exception:
+            if process:
+                if emulator_device_id:
+                    self.terminate_emulator_process(emulator_device_id)
+                else:
+                    process.kill()
+            raise
+
+    def terminate_emulator_process(self, device_id):
+        # Terminate the emulator
+        kill_emu_args = [
+            '-s', device_id,
+            'emu', 'kill'
+        ]
+        self.adb_cmd.run(kill_emu_args)
+
+    def run_emulation_process(self, process_func):
+        """
+        Execute a function that relies on the session based android simulator.
+
+        :param process_func:    The process function to execute. Function requires one argument which will be the device id
+        :return: The return value of the process function
+        """
+
+        emulator_device_id = None
+        try:
+            emulator_process, emulator_device_id = self.launch_emulator_process()
+
+            return process_func(emulator_device_id)
+
+        finally:
+            if emulator_device_id is not None:
+                self.terminate_emulator_process(emulator_device_id)
+
+
+def process_unit_test_on_simulator(base_android_sdk_path, build_path, build_config):
+    """
+    Run the android unit tests on a sessioned simulator
+
+    :param base_android_sdk_path:   The path to where the Android SDK exists
+    :param build_path:              The build path relative to the engine root where the android unit test project is configured and built
+    :param build_config:            The configuration of the build unit test APK to run
+    """
+
+    python_cmd = Command("Python", ENGINE_ROOT / 'python' / 'python.cmd')
+
+    android_script_root = ENGINE_ROOT / 'cmake' / 'Tools' / 'Platform' / 'Android'
+    assert android_script_root.is_dir(), "Missing the android scripts path in the engine folder hierarchy"
+    deploy_android_py_path = android_script_root / 'deploy_android.py'
+    assert deploy_android_py_path.is_file(), "Missing the android deployment script in the engine folder hierarchy"
+    launch_android_ptest_py_path = android_script_root / 'launch_android_test.py'
+    assert launch_android_ptest_py_path.is_file(), "Missing the android unit test launcher script in the engine folder hierarchy"
+
+    def _install_and_run_unit_tests(emulator_id):
+
+        # install unit test on the emulator
+        install_apk_args = [
+            str(deploy_android_py_path),
+            '-b', build_path,
+            '-c', build_config,
+            '--device-id-filter', emulator_id,
+            '--clean'
+        ]
+        python_cmd.run(cmd_args=install_apk_args,
+                       cwd=os.path.normpath(str(ENGINE_ROOT)))
+
+        try:
+            # Launch the unit test on the emulator
+            launch_apk_args = [
+                str(launch_android_ptest_py_path),
+                '-b', build_path,
+                '-c', build_config,
+                '--device-serial', emulator_id
+            ]
+            python_cmd.run(cmd_args=launch_apk_args,
+                           cwd=os.path.normpath(str(ENGINE_ROOT)))
+            return True
+        except AndroidEmuError:
+            print("\n\n")
+            raise AndroidEmuError("Unit Tests Failed")
+
+    # Prepare the emulator manager
+    manager = AndroidEmulatorManager(base_android_sdk_path=base_android_sdk_path,
+                                     force_avd_creation=True)
+
+    # First Install or overwrite the unit test emulator
+    manager.install_unit_test_avd()
+
+    # Run the emulator-dependent process based on the session AVD created by the manager
+    manager.run_emulation_process(_install_and_run_unit_tests)
+
+    # Uninstall the AVD when done
+    manager.uninstall_unit_test_avd()
+
+
+if __name__ == '__main__':
+
+    parser = argparse.ArgumentParser(description="Install and an android unit test APK on a android simulator.")
+
+    parser.add_argument('--android-sdk-path',
+                        help='Path to the Android SDK')
+    parser.add_argument('--build-path',
+                        help='The build path (relative to the engine root) where the project was generated and the APK is built',
+                        required=True)
+    parser.add_argument('--build-config',
+                        help='The build config of the built APK',
+                        required=True)
+    parser.add_argument('--debug',
+                        help='Enable debug messages from this script',
+                        action="store_true")
+
+    parsed_args = parser.parse_args(sys.argv[1:])
+    logging.basicConfig(format='%(levelname)s: %(message)s', level=logging.DEBUG if parsed_args.debug else logging.INFO)
+
+    try:
+        base_android_sdk_path = pathlib.Path(parsed_args.android_sdk_path) if parsed_args.android_sdk_path else get_android_sdk_path()
+        process_unit_test_on_simulator(base_android_sdk_path=base_android_sdk_path,
+                                       build_path=parsed_args.build_path,
+                                       build_config=parsed_args.build_config)
+        exit(0)
+    except AndroidEmuError as e:
+        print(e)
+        exit(1)
+
@@ -14,11 +14,14 @@ set -o errexit # exit on the first failure encountered

 # Delete output directory if CLEAN_OUTPUT_DIRECTORY env variable is set
 if [[ $CLEAN_OUTPUT_DIRECTORY == "true" ]]; then
-    if [[ -d Cache ]]; then
-        echo "[ci_build] CLEAN_OUTPUT_DIRECTORY option set with value \"${CLEAN_OUTPUT_DIRECTORY}\""
-        echo "[ci_build] Deleting \"Cache\""
-        rm -rf Cache
-    fi
+    for project in $(echo $CMAKE_LY_PROJECTS | sed "s/;/ /g")
+    do
+        if [[ -d "$project/Cache" ]]; then
+            echo "[ci_build] CLEAN_OUTPUT_DIRECTORY option set with value \"${CLEAN_OUTPUT_DIRECTORY}\""
+            echo "[ci_build] Deleting \"$project/Cache\""
+            rm -rf $project/Cache
+        fi
+    done
 fi

 if [[ ! -d $OUTPUT_DIRECTORY ]]; then
@@ -9,6 +9,9 @@
    "PIPELINE_ENV_OVERRIDE": {
        "daily-pipeline-metrics": {
            "CLEAN_WORKSPACE": true
+        },
+        "packaging": {
+            "CLEAN_WORKSPACE": true
        }
    }
 }
@@ -14,11 +14,14 @@ set -o errexit # exit on the first failure encountered

 # Delete output directory if CLEAN_OUTPUT_DIRECTORY env variable is set
 if [[ $CLEAN_OUTPUT_DIRECTORY == "true" ]]; then
-    if [[ -d Cache ]]; then
-        echo "[ci_build] CLEAN_OUTPUT_DIRECTORY option set with value \"${CLEAN_OUTPUT_DIRECTORY}\""
-        echo "[ci_build] Deleting \"Cache\""
-        rm -rf Cache
-    fi
+    for project in $(echo $CMAKE_LY_PROJECTS | sed "s/;/ /g")
+    do
+        if [[ -d "$project/Cache" ]]; then
+            echo "[ci_build] CLEAN_OUTPUT_DIRECTORY option set with value \"${CLEAN_OUTPUT_DIRECTORY}\""
+            echo "[ci_build] Deleting \"$project/Cache\""
+            rm -rf $project/Cache
+        fi
+    done
 fi

 if [[ ! -d $OUTPUT_DIRECTORY ]]; then
@@ -9,6 +9,9 @@
    "PIPELINE_ENV_OVERRIDE": {
        "daily-pipeline-metrics": {
            "CLEAN_WORKSPACE": true
+        },
+        "packaging": {
+            "CLEAN_WORKSPACE": true
        }
    }
 }
@@ -14,11 +14,13 @@ SETLOCAL EnableDelayedExpansion

 REM Delete output directory if CLEAN_OUTPUT_DIRECTORY env variable is set
 IF "%CLEAN_OUTPUT_DIRECTORY%"=="true" (
-    IF EXIST Cache (
-        ECHO [ci_build] CLEAN_OUTPUT_DIRECTORY option set with value "%CLEAN_OUTPUT_DIRECTORY%"
-        ECHO [ci_build] Deleting "Cache"
-        DEL /s /q /f Cache
-    )
+    FOR %%P in (%CMAKE_LY_PROJECTS%) do (
+        IF EXIST %%P\Cache (
+            ECHO [ci_build] CLEAN_OUTPUT_DIRECTORY option set with value "%CLEAN_OUTPUT_DIRECTORY%"
+            ECHO [ci_build] Deleting "%%P\Cache"
+            DEL /s /q /f %%P\Cache 1>nul
+        )
+    )    
 )

 IF NOT EXIST %OUTPUT_DIRECTORY% (
@@ -4,6 +4,7 @@
      "default"
    ],
    "steps": [
+      "scrubbing",
      "validation"
    ]
  },
@@ -27,18 +28,14 @@
    ]
  },
  "scrubbing": {
-    "TAGS": [
-        "weekly-build-metrics"
-    ],
+    "TAGS": [],
    "COMMAND": "python_windows.cmd",
    "PARAMETERS": {
      "SCRIPT_PATH": "Tools/build/JenkinsScripts/build/scrubbing_job.py"
    }
  },
  "validation": {
-    "TAGS": [
-      "weekly-build-metrics"
-    ],
+    "TAGS": [],
    "COMMAND": "python_windows.cmd",
    "PARAMETERS": {
      "SCRIPT_PATH": "scripts/commit_validation/validate_file_or_folder.py"
@@ -100,7 +97,7 @@
      "CMAKE_LY_PROJECTS": "AutomatedTesting",
      "CMAKE_TARGET": "TEST_SUITE_smoke TEST_SUITE_main",
      "CMAKE_NATIVE_BUILD_ARGS": "/m /nologo",
-      "CTEST_OPTIONS": "-L \"(SUITE_smoke|SUITE_main)\" -T Test"
+      "CTEST_OPTIONS": "-L \"(SUITE_smoke|SUITE_main)\" -LE \"(REQUIRES_gpu)\" -T Test"
    }
  },
  "profile_vs2019": {
@@ -151,9 +148,11 @@
    }
  },
  "test_gpu_profile_vs2019": {
-    "TAGS": [],
-    "PIPELINE_ENV": {
-      "NODE_LABEL": "windows-gpu"
+    "TAGS":[
+        "nightly"
+    ],
+    "PIPELINE_ENV":{
+        "NODE_LABEL":"windows-gpu"
    },
    "COMMAND": "build_test_windows.cmd",
    "PARAMETERS": {
@@ -19,7 +19,7 @@ IF "%CLEAN_OUTPUT_DIRECTORY%"=="true" (
    IF EXIST %OUTPUT_DIRECTORY% (
        ECHO [ci_build] CLEAN_OUTPUT_DIRECTORY option set with value "%CLEAN_OUTPUT_DIRECTORY%"
        ECHO [ci_build] Deleting "%OUTPUT_DIRECTORY%"
-        DEL /s /q /f %OUTPUT_DIRECTORY%
+        DEL /s /q /f %OUTPUT_DIRECTORY% 1>nul
    )
 )

@@ -1,6 +1,6 @@
 {
    "ENV": {
-        "NODE_LABEL": "windows",
+        "NODE_LABEL": "windows-047e5cdf",
        "LY_3RDPARTY_PATH": "C:/ly/3rdParty",
        "TIMEOUT": 30,
        "WORKSPACE": "D:/workspace",
@@ -9,6 +9,9 @@
    "PIPELINE_ENV_OVERRIDE": {
        "daily-pipeline-metrics": {
            "CLEAN_WORKSPACE": true
+        },
+        "packaging": {
+            "CLEAN_WORKSPACE": true
        }
    }
 }
@@ -9,6 +9,9 @@
    "PIPELINE_ENV_OVERRIDE": {
        "daily-pipeline-metrics": {
            "CLEAN_WORKSPACE": true
+        },
+        "packaging": {
+            "CLEAN_WORKSPACE": true
        }
    }
 }
@@ -146,16 +146,23 @@ def gather_build_metrics(current_dir, build_config_filename, platform):
            print(f'[ci_build_metrics] {reason}', flush=True)
            continue

-        # Clean the output
-        output_directory = build_parameters['OUTPUT_DIRECTORY']
+        # Clean the build output
+        output_directory = build_parameters['OUTPUT_DIRECTORY'] if 'OUTPUT_DIRECTORY' in build_parameters else None
        if not output_directory:
            metrics['result'] = -1 
            reason = f'OUTPUT_DIRECTORY entry in {build_config_abspath} is missing.'
            metrics['reason'] = reason
            print(f'[ci_build_metrics] {reason}', flush=True)
            continue
+        folders_of_interest = [output_directory]

-        folders_of_interest = ['Cache', 'AssetProcessorTemp', output_directory]
+        # Clean the AP output
+        cmake_ly_projects = build_parameters['CMAKE_LY_PROJECTS'] if 'CMAKE_LY_PROJECTS' in build_parameters else None
+        if cmake_ly_projects:
+            projects = cmake_ly_projects.split(';')
+            for project in projects:
+                folders_of_interest.append(os.path.join(project, 'user', 'AssetProcessorTemp'))
+                folders_of_interest.append(os.path.join(project, 'Cache'))
        
        metrics['build_metrics'] = []
        build_metrics = metrics['build_metrics']
@@ -1,13 +0,0 @@
-Contact: l-infrastructure@amazon.com
-
-Last Updated: 2015.02.13
-
-
-
-Description:
-
-
-
-LYKey.ppk is the keyfile for the EC2 instance hosted on the AWS account lumberyard-distribution@amazon.com.
-
-https://wiki.labcollab.net/confluence/display/lmbr/Build+Distribution+Proprosal
@@ -0,0 +1,309 @@
+"""
+
+ All or portions of this file Copyright (c) Amazon.com, Inc. or its affiliates or
+ its licensors.
+
+ For complete copyright and license terms please see the LICENSE at the root of this
+ distribution (the "License"). All use of this software is governed by the License,
+ or, if provided, by the license below or the license accompanying this file. Do not
+ remove or modify any license notices. This file is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+"""
+
+import BuildThirdPartyArgs
+import BuildThirdPartyUtils
+import SDKPackager
+import ThirdPartySDKAWS
+import json
+import os
+import re
+import sys
+import urlparse
+
+importDir = os.path.dirname(os.path.abspath(__file__))
+sys.path.append(os.path.join(importDir, "..")) #Required for AWS_PyTools
+from AWS_PyTools import LyChecksum
+from AWS_PyTools import LyCloudfrontOps
+
+# Some files are in the 3rd party folder and not associated with any SDK/Versions.
+fileIgnorelist = [
+    "boost/Boost_Autoexp.dat",
+    "boost/CryEngine Customizations.txt",
+    "boost/CryREADME.txt",
+    "boost/LICENSE_1_0.txt",
+    "boost/lumberyard-1.61.0.patch",
+    "Qwt/license.txt",
+    "lz4/git checkout.txt"
+    "3rdParty.txt"
+]
+
+
+class SDK(object):
+
+    def __init__(self):
+        self.fileList = []
+        self.path = ""
+        self.versionFolder = ""
+
+
+def getListFromFile(file):
+    openFile = open(file, 'r')
+    filePaths = openFile.readlines()
+    openFile.close()
+    return filePaths
+
+
+def addUntrackedSDKs(sdks):
+    # Not all SDKs are represented in SetupAssistantConfig.json yet.
+
+    untrackedSDKs = {
+        "OpenEXR20": ["OpenEXR/2.0", "2.0"],
+        "OpenEXR22": ["OpenEXR/2.2", "2.2"],
+    }
+
+    for sdkName, untrackedSDK in untrackedSDKs.iteritems():
+        sdk = SDK()
+        sdk.path = untrackedSDK[0]
+        sdk.versionFolder = untrackedSDK[1]
+        sdks[sdkName] = sdk
+
+
+def getSDKsToPathsDict(thirdPartyVersionsFile):
+    versionsList = getListFromFile(thirdPartyVersionsFile)
+    sdks = {}
+    for version in versionsList:
+        match = re.match(r"(.*)(\.package.dir=)(.*)", version)
+        if not match:
+            BuildThirdPartyUtils.printError('SDK version file {0} has invalid formatting on line {1}'.format(
+                    thirdPartyVersionsFile,
+                    version))
+
+        sdkName = match.group(1)
+        # Store the SDK Path with forward slashes to make matching easier.
+        sdkPath = match.group(3).replace('\\', '/')
+        sdk = SDK()
+        sdk.path = sdkPath
+
+        versionMatch = re.search(r"([^/\\\\]*)$", sdkPath)
+        if not versionMatch:
+            BuildThirdPartyUtils.printError('SDK version file {0} has invalid formatting on line {1}'.format(
+                    thirdPartyVersionsFile,
+                    version))
+
+        sdk.versionFolder = versionMatch.group(1)
+        sdks[sdkName] = sdk
+
+    addUntrackedSDKs(sdks)
+    return sdks
+
+
+def populateSDKFilePaths(sdks, sdkFileListFile):
+    fileList = getListFromFile(sdkFileListFile)
+    lastSDKName = ""
+    for file in fileList:
+        slashesFixed = file.replace('\\', '/')
+        match = re.search("3rdParty/(.*)", slashesFixed)
+        if not match:
+            BuildThirdPartyUtils.printError("Could not find third party folder in file path {0}".format(file))
+
+        localPath = match.group(1)
+        sdkFound = False
+
+        # Files are generally grouped by SDK in the file list,
+        # caching the last SDK used can save a search through the loop.
+        if lastSDKName:
+             if localPath.startswith(sdks[lastSDKName].path):
+                sdk.fileList.append(file)
+                sdkFound = True
+                continue
+
+        for sdkName, sdk in sdks.iteritems():
+            if localPath.startswith(sdk.path):
+                sdk.fileList.append(file)
+                sdkFound = True
+                lastSDKName = sdkName
+                break
+
+        # Some files are loose and not trackable within the current system. For now we're going to ignore them,
+        # and they will need to be included in the package manually.
+        for ignore in fileIgnorelist:
+            if localPath == ignore:
+                sdkFound = True
+                break
+
+        if not sdkFound:
+            BuildThirdPartyUtils.printError("File {0} is not associated with any known SDKs".format(
+                    file,
+                    sdkFileListFile))
+
+
+def checkForSDKStagingErrors(bucket, baseBucketPath, sdkPath, sdkPlatform, filesetHash):
+    tmpDirPath = SDKPackager.getTempDir(sdkPath, sdkPlatform)
+
+    filelistFileName = SDKPackager.getFilelistFileName(sdkPlatform)
+    filelistLocalPath = os.path.join(tmpDirPath, filelistFileName)
+    filelistStagingPath = ThirdPartySDKAWS.getS3StagingPath(baseBucketPath, sdkPath) + filelistFileName
+
+    if not os.path.exists(tmpDirPath):
+        os.makedirs(tmpDirPath)
+    bucket.download_file(filelistStagingPath, filelistLocalPath)
+
+    filelistData = open(filelistLocalPath, 'r')
+    filelistJsonData = json.load(filelistData)
+    filelistData.close()
+
+    assert(filelistData != ""), "Failed to load from " + filelistLocalPath
+
+    filelistChecksum = filelistJsonData["filelist"]["checksum"]
+
+    if filelistChecksum != filesetHash.hexdigest():
+        # If the checksums don't match, then the SDK has been modified without changing the version. This is an error.
+        return True
+    return False
+
+
+def checkForExistingSDK(ignoreExisting,
+                        bucket,
+                        baseBucketPath,
+                        sdkName,
+                        versionFolder,
+                        sdkPath,
+                        sdkPlatform,
+                        filesetHash):
+    returnCode = 0
+    statusMessage = None
+    # Check for existing manifest
+    manifestExists, filelistExists = ThirdPartySDKAWS.getSDKStagingStatus(bucket,
+                                                                          baseBucketPath,
+                                                                          sdkPath,
+                                                                          sdkPlatform)
+    if ignoreExisting:
+        return statusMessage, returnCode
+
+    # If the manifest or filelist is missing, but one is available, then the SDK likely failed to upload.
+    # In this case, just continue on and generate the SDK package.
+    if manifestExists and filelistExists:
+        # If the manifest and filelist both exist, this SDK.version.package has been uploaded already.
+        stagingError = checkForSDKStagingErrors(bucket,
+                                                baseBucketPath,
+                                                sdkPath,
+                                                sdkPlatform,
+                                                filesetHash)
+
+        if stagingError:
+            statusMessage = "ERROR: The file list manifests do not match for SDK {0}, Version {1}, Platform {2}".format(sdkName,
+                                                                                                                 versionFolder,
+                                                                                                                 sdkPlatform)
+            returnCode = 1
+        else:
+            statusMessage = "\tEverything is up to date for SDK {0}, Version {1}, Platform {2}".format(sdkName,
+                                                                                                     versionFolder,
+                                                                                                     sdkPlatform)
+    if statusMessage:
+        print statusMessage
+    return statusMessage, returnCode
+
+
+def getListFromJsonFile(jsonFile, root):
+    if not jsonFile:
+        return []
+    if not os.path.isfile(jsonFile):
+        print "{} is not a valid file, please check the filename specified.".format(jsonFile)
+        exit(1)
+    with open(jsonFile, 'r') as source:
+        source_json = json.load(source)
+        try:
+            sdks_list = source_json[root]
+            return sdks_list
+        except KeyError:
+            print "Unknown json root {}, please check the json root specified.".format(root)
+            exit(1)
+
+
+############################
+
+
+def main():
+    print "Building third party packages"
+    args = BuildThirdPartyArgs.createArgs()
+
+    print "Parsing file lists"
+    ignoreExistingSDKList = getListFromJsonFile(args.ignoreExistingList, args.sdkPlatform)
+    sdkBlacklist = getListFromJsonFile(args.sdkBlacklist, "Blacklist")
+    sdks = getSDKsToPathsDict(args.thirdPartyVersions)
+    populateSDKFilePaths(sdks, args.sdkFilelist)
+
+    cloudfrontDist = LyCloudfrontOps.getCloudfrontDistribution(args.cloudfrontDomain, args.awsProfile)
+    bucket = LyCloudfrontOps.getBucket(cloudfrontDist, args.awsProfile)
+    baseBucketPath = LyCloudfrontOps.buildBucketPath(urlparse.urljoin(args.cloudfrontDomain, args.stagingFolderPath), cloudfrontDist)
+    baseCloudfrontUrl = args.cloudfrontDomain # we assume that the domain ends in a trailing '/'
+    if args.stagingFolderPath:
+        baseCloudfrontUrl += args.stagingFolderPath # we assume that the folder path ends in a trailing '/'
+
+    returnCode = 0
+    sdksGenerated = []
+
+    sdkCount = len(sdks)
+    currentSDK = -1
+    for sdkName, sdk in sdks.iteritems():
+        currentSDK += 1
+        print "{0}/{1} - Processing {2}".format(currentSDK, sdkCount, sdkName)
+
+        # Don't process SDKs in the blacklist.
+        if sdkName in sdkBlacklist:
+            print "\tSkipping blacklist SDK {0}".format(sdkName)
+            continue
+        # Hash all files for the SDK
+        filesetHash = LyChecksum.generateFilesetChecksum(sdk.fileList)
+
+        if args.internalPackage:
+            ignoreExisting = True
+        else:
+            ignoreExisting = sdkName in ignoreExistingSDKList
+
+        statusMessage, sdkReturnCode = checkForExistingSDK(ignoreExisting,
+                                                           bucket,
+                                                           baseBucketPath,
+                                                           sdkName,
+                                                           sdk.versionFolder,
+                                                           sdk.path,
+                                                           args.sdkPlatform,
+                                                           filesetHash)
+        # The final return should be the highest reported return code.
+        returnCode = max(returnCode, sdkReturnCode)
+        if statusMessage:
+            continue
+        manifestPath, filelistPath, zipFiles = SDKPackager.generateSDKPackage(baseCloudfrontUrl,
+                                                                              sdkName,
+                                                                              sdk.versionFolder,
+                                                                              sdk.path,
+                                                                              args.sdkPlatform,
+                                                                              filesetHash,
+                                                                              sdk.fileList,
+                                                                              args.archiveMaxSize)
+        if not args.skipUpload:
+            ThirdPartySDKAWS.uploadSDKToStaging(bucket,
+                                                baseBucketPath,
+                                                sdkName,
+                                                sdk.versionFolder,
+                                                sdk.path,
+                                                args.sdkPlatform,
+                                                manifestPath,
+                                                filelistPath,
+                                                zipFiles)
+        print "{0}/{1} - Completed Processing {2}".format(currentSDK+1, sdkCount, sdkName)
+        sdksGenerated.append(sdkName)
+
+    # If any SDKs were updated, and we were told to make a file to output
+    # the list of updated SDKs to, then make said file and write out the list
+    if len(sdksGenerated) > 0 and args.updatesFile:
+        try:
+            with open(args.updatesFile, 'w') as out:
+                out.writelines('\n'.join(sdksGenerated))
+        except:
+            BuildThirdPartyUtils.printError("Failed to write list of updated SDKs to {0}. Backup zip files will not be created properly for this build.".format(args.updatesFile))
+
+    return returnCode
+
+if __name__ == "__main__":
+    sys.exit(main())
@@ -0,0 +1,29 @@
+"""
+
+ All or portions of this file Copyright (c) Amazon.com, Inc. or its affiliates or
+ its licensors.
+
+ For complete copyright and license terms please see the LICENSE at the root of this
+ distribution (the "License"). All use of this software is governed by the License,
+ or, if provided, by the license below or the license accompanying this file. Do not
+ remove or modify any license notices. This file is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+"""
+
+import sys
+
+def printError(message):
+    print(message)
+    sys.exit(1)
+
+
+def reportIterationStatus(index, count, reportFrequency, message):
+    # Reporting on the first, last, and at a frequency that is a good balance of not spamming the logs
+    frequency = count / reportFrequency
+    lastPercent = float(index-1) / float(count)
+    percentComplete = float(index) / float(count)
+    lastReportSlice = int(lastPercent * frequency)
+    thisReportSlice = int(percentComplete * frequency)
+    shouldPrint = lastReportSlice != thisReportSlice or index == 1 or index == count
+    if shouldPrint:
+        print "\t{0}% complete, {1}/{2} {3}".format(int(percentComplete*100), index, count, message)
@@ -0,0 +1,207 @@
+"""
+
+ All or portions of this file Copyright (c) Amazon.com, Inc. or its affiliates or
+ its licensors.
+
+ For complete copyright and license terms please see the LICENSE at the root of this
+ distribution (the "License"). All use of this software is governed by the License,
+ or, if provided, by the license below or the license accompanying this file. Do not
+ remove or modify any license notices. This file is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+"""
+
+import zipfile
+import os.path
+import re
+import json
+import sys
+import ThirdPartySDKAWS
+import BuildThirdPartyUtils
+import tempfile
+
+importDir = os.path.dirname(os.path.abspath(__file__))
+sys.path.append(os.path.join(importDir, "..")) #Required for AWS_PyTools
+from AWS_PyTools import LyChecksum
+
+class SDKZipFile(object):
+    def __init__(self):
+        self.file = None
+        self.filePath = None
+        self.contents = []
+        self.compressedSize = 0
+        self.uncompressedSize = 0
+        self.compressedHash = 0
+
+
+def getFilelistVersion():
+    return "1.0.0"
+
+
+def getFilelistFileName(sdkPlatform):
+    return "filelist." + getFilelistVersion() + "." + sdkPlatform + ".json"
+
+
+def getManifestVersion():
+    return "1.0.0"
+
+
+def getManifestFileName(sdkPlatform):
+    return "manifest." + getManifestVersion() + "." + sdkPlatform + ".json"
+
+
+def toArchivePath(filePath):
+    # Archives are generated relative to 3rdParty folder, so strip everything before that in the path.
+    # zipfile is very particular in how paths to files within it are formatted.
+    slashesFixed = filePath.replace('\\', '/')
+    match = re.search("3rdParty/(.*)", slashesFixed)
+    if not match:
+        BuildThirdPartyUtils.printError("Could not find third party folder in file path {0}".format(filePath))
+    archivePath = match.group(1).strip("/").strip("\n")
+    return archivePath
+
+
+def prepFilesystemForFile(filePath):
+    directory = os.path.dirname(filePath)
+    if not os.path.exists(directory):
+        os.makedirs(directory)
+    if os.path.isfile(filePath):
+        os.remove(filePath)
+
+def createJSONString(dataToFormat):
+    return json.dumps(dataToFormat, sort_keys=True, indent=4, separators=(',', ': '))
+
+
+def generateSDKPackage(baseCloudfrontUrl,
+                       sdkName,
+                       sdkVersion,
+                       sdkPath,
+                       sdkPlatform,
+                       filesetHash,
+                       filePaths,
+                       archiveMaxSize):
+    zipFiles = zipPackage(sdkName, sdkVersion, sdkPath, sdkPlatform, filePaths, archiveMaxSize)
+    filelistPath = buildFilelistJSON(sdkPath, sdkPlatform, filesetHash, filePaths)
+    manifestPath = buildManifestJSON(baseCloudfrontUrl,
+                                     sdkName,
+                                     sdkPath,
+                                     sdkPlatform,
+                                     zipFiles,
+                                     filelistPath)
+    return manifestPath, filelistPath, zipFiles
+
+
+def getTempDir(sdkPath, sdkPlatform):
+    tempDir = os.path.join(tempfile.tempdir, "LY", "3rdPartySDKs", sdkPath, sdkPlatform)
+    return os.path.expandvars(tempDir)
+
+
+def zipPackage(sdkName, sdkVersion, sdkPath, sdkPlatform, filePaths, archiveMaxSize):
+    try:
+        import zlib
+        compression = zipfile.ZIP_DEFLATED
+    except:
+        compression = zipfile.ZIP_STORED
+
+    tempDir = getTempDir(sdkPath, sdkPlatform)
+    zipFiles = []
+    currentZipFile = None
+
+    fileIndex = 0
+    fileCount = len(filePaths)
+
+    for filePath in filePaths:
+
+        filePath = filePath.strip("\n")
+        archivePath = toArchivePath(filePath)
+        if currentZipFile is None or currentZipFile.file is None or currentZipFile.compressedSize > archiveMaxSize:
+            if currentZipFile and currentZipFile.file:
+                currentZipFile.file.close()
+            currentZipFile = SDKZipFile()
+            zipFiles.append(currentZipFile)
+            currentZipFile.filePath = os.path.join(tempDir, sdkName + "." + str(sdkPlatform) + "." + str(len(zipFiles)) + ".zip")
+            prepFilesystemForFile(currentZipFile.filePath)
+            currentZipFile.file = zipfile.ZipFile(currentZipFile.filePath, mode='w')
+        currentZipFile.file.write(filePath, compress_type=compression, arcname=archivePath)
+        fileInfo = currentZipFile.file.getinfo(archivePath)
+        currentZipFile.compressedSize += fileInfo.compress_size
+        currentZipFile.uncompressedSize += fileInfo.file_size
+        currentZipFile.contents.append(filePath)
+
+        fileIndex += 1
+        BuildThirdPartyUtils.reportIterationStatus(fileIndex, fileCount, 25, "files zipped")
+
+    if currentZipFile and currentZipFile.file:
+        currentZipFile.file.close()
+
+    for zipFile in zipFiles:
+        zipFile.compressedHash = LyChecksum.getChecksumForSingleFile(zipFile.filePath)
+
+    return zipFiles
+
+
+def buildFilelistJSON(sdkPath, sdkPlatform, filesetHash, filePaths):
+    jsonFormatFiles = []
+    for filePath in filePaths:
+        scrubbedFile = toArchivePath(filePath)
+        jsonFormatFiles.append(scrubbedFile)
+
+    filelistInfo = {
+        "filelist": {
+            "filelistVersion": getFilelistVersion(),
+            "checksum": filesetHash.hexdigest(),
+            "files": jsonFormatFiles,
+        }
+    }
+    filelistJSON = createJSONString(filelistInfo)
+    filelistName = getFilelistFileName(sdkPlatform)
+    filelistPath = getTempDir(sdkPath, sdkPlatform)
+    filelistFullPath = os.path.join(filelistPath, filelistName)
+
+    prepFilesystemForFile(filelistFullPath)
+
+    outputFile = open(filelistFullPath, 'w')
+    outputFile.write(filelistJSON)
+    outputFile.close()
+    return filelistFullPath
+
+
+def buildManifestJSON(baseCloudfrontUrl, sdkName, sdkPath, sdkPlatform, zipFiles, filelistPath):
+    uncompressedSize = 0
+    packageArchives = []
+    for zipFile in zipFiles:
+        uncompressedSize += zipFile.uncompressedSize
+        archiveUrl = ThirdPartySDKAWS.getProductionUrl(baseCloudfrontUrl, sdkPath, zipFile.filePath)
+        packageArchive = {
+            "archiveUrl": archiveUrl,
+            "archiveSize": str(zipFile.compressedSize),
+            "archiveChecksum": zipFile.compressedHash.hexdigest()
+        }
+        packageArchives.append(packageArchive)
+
+    filelistChecksum = LyChecksum.getChecksumForSingleFile(filelistPath)
+
+    filelistUrl = ThirdPartySDKAWS.getProductionUrl(baseCloudfrontUrl, sdkPath, filelistPath)
+    packageInfo = {
+        "package":
+        {
+            "manifestVersion": getManifestVersion(),
+            "identifier": sdkName,
+            "platform": sdkPlatform,
+            "uncompressedSize": str(uncompressedSize),
+            "filelistUrl": filelistUrl,
+            "filelistChecksum": filelistChecksum.hexdigest(),
+            "archives": packageArchives
+
+        }
+    }
+    manifestJSON = createJSONString(packageInfo)
+    manifestName = getManifestFileName(sdkPlatform)
+    manifestPath = getTempDir(sdkPath, sdkPlatform)
+    manifestFullPath = os.path.join(manifestPath, manifestName)
+
+    prepFilesystemForFile(manifestFullPath)
+
+    outputFile = open(manifestFullPath, 'w')
+    outputFile.write(manifestJSON)
+    outputFile.close()
+    return manifestFullPath
@@ -0,0 +1,76 @@
+"""
+
+ All or portions of this file Copyright (c) Amazon.com, Inc. or its affiliates or
+ its licensors.
+
+ For complete copyright and license terms please see the LICENSE at the root of this
+ distribution (the "License"). All use of this software is governed by the License,
+ or, if provided, by the license below or the license accompanying this file. Do not
+ remove or modify any license notices. This file is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+"""
+
+import os.path
+import sys
+import urlparse
+import SDKPackager
+import BuildThirdPartyUtils
+
+importDir = os.path.dirname(os.path.abspath(__file__))
+sys.path.append(os.path.join(importDir, "..")) #Required for AWS_PyTools
+from AWS_PyTools import LyCloudfrontOps
+
+def getS3StagingPath(stagingFolderPath, sdkPath):
+    # The staging path is used for the build machines to upload new builds of 3rd party packages.
+    # The Setup Assistant supports a global override so Lumberyard team members can pull from the staging location.
+    return urlparse.urljoin(stagingFolderPath, sdkPath.replace(' ', '_')) + '/'
+
+def getProductionUrl(cloudfrontUrl, sdkPath, filePath):
+    # The production path is the final path customers will download the SDK from.
+    # These links get baked into the manifest, which the Setup Assistant executable uses to acquire these SDKs.
+    # TODO : Generate an actual cloudfront production link.
+    #return cloudfrontUrl + getS3StagingPath(stagingFolderPath, sdkPath.replace(' ', '_')) + os.path.basename(filePath)
+    # we assume that the cloudfront url and the result of getS3stagingPath have a trailing '/'
+    return getS3StagingPath(cloudfrontUrl, sdkPath) + os.path.basename(filePath)
+
+def getSDKStagingStatus(bucket, baseBucketPath, sdkPath, sdkPlatform):
+
+    pathToSDK = getS3StagingPath(baseBucketPath, sdkPath)
+    pathToFilelist = pathToSDK + SDKPackager.getFilelistFileName(sdkPlatform)
+    pathToManifest = pathToSDK + SDKPackager.getManifestFileName(sdkPlatform)
+    manifestExists = False
+    filelistExists = False
+
+    objs = list(bucket.objects.filter(Prefix=pathToManifest))
+    if len(objs) > 0 and objs[0].key == pathToManifest:
+        manifestExists = True
+    else:
+        manifestExists = False
+
+    objs = list(bucket.objects.filter(Prefix=pathToFilelist))
+    if len(objs) > 0 and objs[0].key == pathToFilelist:
+        filelistExists = True
+    else:
+        filelistExists = False
+
+    return manifestExists, filelistExists
+
+def uploadSDKToStaging(bucket, baseBucketPath, sdkName, sdkVersion, sdkPath, sdkPlatform, manifestPath, filelistPath, zipFiles):
+    print "\tUploading SDK: {0} ({1},{2})".format(sdkName, sdkVersion, sdkPlatform)
+
+    manifestStagingPath = getS3StagingPath(baseBucketPath, sdkPath) + os.path.basename(manifestPath)
+    bucket.upload_file(manifestPath, manifestStagingPath)
+    print "\tUploaded manifest"
+
+    filelistStagingPath = getS3StagingPath(baseBucketPath, sdkPath) + os.path.basename(filelistPath)
+    bucket.upload_file(filelistPath, filelistStagingPath)
+    print "\tUploaded filelist"
+
+    filesUploaded = 0
+    totalFilesCount = len(zipFiles)
+    for zipFile in zipFiles:
+        fileLocalPath = zipFile.filePath
+        fileStagingPath = getS3StagingPath(baseBucketPath, sdkPath) + os.path.basename(zipFile.filePath)
+        bucket.upload_file(fileLocalPath, fileStagingPath)
+        filesUploaded += 1
+        BuildThirdPartyUtils.reportIterationStatus(filesUploaded, totalFilesCount, 5, "files uploaded")