SPEC-6276 Move ci_build and ci_build_metrics to scripts\build
This commit is contained in:
Esteban Papp
GitHub
Vendored
+1
-1
@@ -201,7 +201,7 @@ def CheckoutBootstrapScripts(String branchName) {
|
||||
sparseCheckoutPaths: [
|
||||
[ $class: "SparseCheckoutPath", path: "scripts/build/Jenkins/" ],
|
||||
[ $class: "SparseCheckoutPath", path: "scripts/build/bootstrap/" ],
|
||||
[ $class: "SparseCheckoutPath", path: "Tools/build/JenkinsScripts/build/Platform" ]
|
||||
[ $class: "SparseCheckoutPath", path: "scripts/build/Platform" ]
|
||||
]
|
||||
],
|
||||
[
|
||||
|
||||
@@ -1,12 +1,12 @@
|
||||
{
|
||||
"BUILD_ENTRY_POINT": "Tools/build/JenkinsScripts/build/ci_build.py",
|
||||
"BUILD_ENTRY_POINT": "scripts/build/ci_build.py",
|
||||
"PIPELINE_CONFIGS": [
|
||||
"Tools/build/JenkinsScripts/build/Platform/*/pipeline.json",
|
||||
"restricted/*/Tools/build/JenkinsScripts/build/pipeline.json"
|
||||
"scripts/build/Platform/*/pipeline.json",
|
||||
"restricted/*/scripts/build/pipeline.json"
|
||||
],
|
||||
"BUILD_CONFIGS": [
|
||||
"Tools/build/JenkinsScripts/build/Platform/*/build_config.json",
|
||||
"restricted/*/Tools/build/JenkinsScripts/build/build_config.json"
|
||||
"scripts/build/Platform/*/build_config.json",
|
||||
"restricted/*/scripts/build/build_config.json"
|
||||
],
|
||||
"PYTHON_DIR": "python"
|
||||
}
|
||||
|
||||
@@ -0,0 +1,45 @@
|
||||
@ECHO OFF
|
||||
REM
|
||||
REM All or portions of this file Copyright (c) Amazon.com, Inc. or its affiliates or
|
||||
REM its licensors.
|
||||
REM
|
||||
REM For complete copyright and license terms please see the LICENSE at the root of this
|
||||
REM distribution (the "License"). All use of this software is governed by the License,
|
||||
REM or, if provided, by the license below or the license accompanying this file. Do not
|
||||
REM remove or modify any license notices. This file is distributed on an "AS IS" BASIS,
|
||||
REM WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
REM
|
||||
|
||||
SETLOCAL EnableDelayedExpansion
|
||||
|
||||
CALL %~dp0gradle_windows.cmd
|
||||
IF NOT %ERRORLEVEL%==0 GOTO :error
|
||||
|
||||
IF NOT EXIST "%LY_3RDPARTY_PATH%" (
|
||||
ECHO [ci_build] LY_3RDPARTY_PATH is invalid or not set
|
||||
GOTO :error
|
||||
)
|
||||
|
||||
IF NOT EXIST "%LY_ANDROID_SDK%" (
|
||||
SET LY_ANDROID_SDK=!LY_3RDPARTY_PATH!/android-sdk/platform-29
|
||||
)
|
||||
IF NOT EXIST "%LY_ANDROID_SDK%" (
|
||||
ECHO [ci_build] FAIL: LY_ANDROID_SDK=!LY_ANDROID_SDK!
|
||||
GOTO :error
|
||||
)
|
||||
SET ANDROID_SDK_ROOT=%LY_ANDROID_SDK%
|
||||
ECHO "ANDROID_SDK_ROOT=!ANDROID_SDK_ROOT!"
|
||||
SET PYTHON=python\python.cmd
|
||||
ECHO [ci_build] %PYTHON% scripts\build\Platform\Android\run_test_on_android_simulator.py --android-sdk-path %LY_ANDROID_SDK% --build-path %OUTPUT_DIRECTORY% --build-config %CONFIGURATION%
|
||||
CALL %PYTHON% scripts\build\Platform\Android\run_test_on_android_simulator.py --android-sdk-path %LY_ANDROID_SDK% --build-path %OUTPUT_DIRECTORY% --build-config %CONFIGURATION%
|
||||
IF NOT %ERRORLEVEL%==0 GOTO :popd_error
|
||||
|
||||
EXIT /b 0
|
||||
|
||||
:popd_error
|
||||
POPD
|
||||
|
||||
:error
|
||||
ECHO ERROR
|
||||
|
||||
EXIT /b 1
|
||||
@@ -0,0 +1,167 @@
|
||||
{
|
||||
"clean": {
|
||||
"TAGS": [],
|
||||
"COMMAND": "../Windows/clean_windows.cmd",
|
||||
"PARAMETERS": {
|
||||
"OUTPUT_DIRECTORY": "build",
|
||||
"CMAKE_LY_PROJECTS": "AutomatedTesting"
|
||||
}
|
||||
},
|
||||
"profile_pipe": {
|
||||
"TAGS": [
|
||||
"default"
|
||||
],
|
||||
"steps": [
|
||||
"profile"
|
||||
]
|
||||
},
|
||||
"metrics": {
|
||||
"TAGS":[
|
||||
"weekly"
|
||||
],
|
||||
"COMMAND":"../Windows/python_windows.cmd",
|
||||
"PARAMETERS": {
|
||||
"SCRIPT_PATH":"scripts/build/ci_build_metrics.py",
|
||||
"SCRIPT_PARAMETERS":"--platform Android --jobname \"!JOB_NAME!\" --jobnumber \"!BUILD_NUMBER!\" --jobnode \"!NODE_NAME!\" --changelist \"!CHANGE_ID!\""
|
||||
}
|
||||
},
|
||||
"debug": {
|
||||
"TAGS":[
|
||||
"nightly",
|
||||
"weekly-build-metrics"
|
||||
],
|
||||
"COMMAND":"../Windows/build_ninja_windows.cmd",
|
||||
"PARAMETERS": {
|
||||
"CONFIGURATION":"debug",
|
||||
"OUTPUT_DIRECTORY":"build\\android",
|
||||
"CMAKE_OPTIONS":"-G \"Ninja Multi-Config\" -DCMAKE_TOOLCHAIN_FILE=cmake\\Platform\\Android\\Toolchain_android.cmake -DANDROID_ABI=arm64-v8a -DANDROID_ARM_MODE=arm -DANDROID_ARM_NEON=FALSE -DANDROID_NATIVE_API_LEVEL=21 -DLY_NDK_DIR=\"!LY_3RDPARTY_PATH!\\android-ndk\\r21d\" -DLY_UNITY_BUILD=TRUE",
|
||||
"CMAKE_LY_PROJECTS":"AutomatedTesting",
|
||||
"CMAKE_TARGET":"all",
|
||||
"CMAKE_BUILD_ARGS":"-j!NUMBER_OF_PROCESSORS!"
|
||||
}
|
||||
},
|
||||
"profile": {
|
||||
"TAGS":[
|
||||
"weekly-build-metrics",
|
||||
"daily-pipeline-metrics"
|
||||
],
|
||||
"COMMAND":"../Windows/build_ninja_windows.cmd",
|
||||
"PARAMETERS": {
|
||||
"CONFIGURATION":"profile",
|
||||
"OUTPUT_DIRECTORY":"build\\android",
|
||||
"CMAKE_OPTIONS":"-G \"Ninja Multi-Config\" -DCMAKE_TOOLCHAIN_FILE=cmake\\Platform\\Android\\Toolchain_android.cmake -DANDROID_ABI=arm64-v8a -DANDROID_ARM_MODE=arm -DANDROID_ARM_NEON=FALSE -DANDROID_NATIVE_API_LEVEL=21 -DLY_NDK_DIR=\"!LY_3RDPARTY_PATH!\\android-ndk\\r21d\" -DLY_UNITY_BUILD=TRUE",
|
||||
"CMAKE_LY_PROJECTS":"AutomatedTesting",
|
||||
"CMAKE_TARGET":"all",
|
||||
"CMAKE_BUILD_ARGS":"-j!NUMBER_OF_PROCESSORS!"
|
||||
}
|
||||
},
|
||||
"profile_nounity": {
|
||||
"TAGS":[
|
||||
"nightly",
|
||||
"weekly-build-metrics"
|
||||
],
|
||||
"COMMAND":"../Windows/build_ninja_windows.cmd",
|
||||
"PARAMETERS": {
|
||||
"CONFIGURATION":"profile",
|
||||
"OUTPUT_DIRECTORY":"build\\android",
|
||||
"CMAKE_OPTIONS":"-G \"Ninja Multi-Config\" -DCMAKE_TOOLCHAIN_FILE=cmake\\Platform\\Android\\Toolchain_android.cmake -DANDROID_ABI=arm64-v8a -DANDROID_ARM_MODE=arm -DANDROID_ARM_NEON=FALSE -DANDROID_NATIVE_API_LEVEL=21 -DLY_NDK_DIR=\"!LY_3RDPARTY_PATH!\\android-ndk\\r21d\" -DLY_UNITY_BUILD=FALSE",
|
||||
"CMAKE_LY_PROJECTS":"AutomatedTesting",
|
||||
"CMAKE_TARGET":"all",
|
||||
"CMAKE_BUILD_ARGS":"-j!NUMBER_OF_PROCESSORS!"
|
||||
}
|
||||
},
|
||||
"asset_profile": {
|
||||
"TAGS":[
|
||||
"default",
|
||||
"weekly-build-metrics"
|
||||
],
|
||||
"COMMAND":"../Windows/build_asset_windows.cmd",
|
||||
"PARAMETERS": {
|
||||
"CONFIGURATION":"profile",
|
||||
"OUTPUT_DIRECTORY":"build\\windows_vs2019",
|
||||
"CMAKE_OPTIONS":"-G \"Visual Studio 16 2019\" -DCMAKE_SYSTEM_VERSION=10.0 -DLY_UNITY_BUILD=TRUE",
|
||||
"CMAKE_LY_PROJECTS":"AutomatedTesting",
|
||||
"CMAKE_TARGET":"AssetProcessorBatch",
|
||||
"CMAKE_NATIVE_BUILD_ARGS": "/m /nologo",
|
||||
"ASSET_PROCESSOR_BINARY": "bin\\profile\\AssetProcessorBatch.exe",
|
||||
"ASSET_PROCESSOR_OPTIONS": "/zeroAnalysisMode --regset=\"/Amazon/AssetProcessor/Settings/Exclude Android/pattern=.*/DiffuseGlobalIllumination/.*precompiledshader\"",
|
||||
"ASSET_PROCESSOR_PLATFORMS":"es3"
|
||||
}
|
||||
},
|
||||
"asset_clean_profile": {
|
||||
"TAGS":[
|
||||
"nightly"
|
||||
],
|
||||
"PIPELINE_ENV": {
|
||||
"CLEAN_ASSETS": "1"
|
||||
},
|
||||
"steps": [
|
||||
"clean",
|
||||
"asset_profile"
|
||||
]
|
||||
},
|
||||
"release": {
|
||||
"TAGS":[
|
||||
"nightly",
|
||||
"weekly-build-metrics"
|
||||
],
|
||||
"COMMAND":"../Windows/build_ninja_windows.cmd",
|
||||
"PARAMETERS": {
|
||||
"CONFIGURATION":"release",
|
||||
"OUTPUT_DIRECTORY":"build\\android",
|
||||
"CMAKE_OPTIONS":"-G \"Ninja Multi-Config\" -DCMAKE_TOOLCHAIN_FILE=cmake\\Platform\\Android\\Toolchain_android.cmake -DANDROID_ABI=arm64-v8a -DANDROID_ARM_MODE=arm -DANDROID_ARM_NEON=FALSE -DANDROID_NATIVE_API_LEVEL=21 -DLY_NDK_DIR=\"!LY_3RDPARTY_PATH!\\android-ndk\\r21d\" -DLY_UNITY_BUILD=TRUE",
|
||||
"CMAKE_LY_PROJECTS":"AutomatedTesting",
|
||||
"CMAKE_TARGET":"all",
|
||||
"CMAKE_BUILD_ARGS":"-j!NUMBER_OF_PROCESSORS!"
|
||||
}
|
||||
},
|
||||
"monolithic_release": {
|
||||
"TAGS":[
|
||||
"nightly",
|
||||
"weekly-build-metrics"
|
||||
],
|
||||
"COMMAND":"../Windows/build_ninja_windows.cmd",
|
||||
"PARAMETERS": {
|
||||
"CONFIGURATION":"release",
|
||||
"OUTPUT_DIRECTORY":"build\\mono_android",
|
||||
"CMAKE_OPTIONS":"-G \"Ninja Multi-Config\" -DCMAKE_TOOLCHAIN_FILE=cmake\\Platform\\Android\\Toolchain_android.cmake -DANDROID_ABI=arm64-v8a -DANDROID_ARM_MODE=arm -DANDROID_ARM_NEON=FALSE -DANDROID_NATIVE_API_LEVEL=21 -DLY_NDK_DIR=\"!LY_3RDPARTY_PATH!\\android-ndk\\r21d\" -DLY_MONOLITHIC_GAME=TRUE -DLY_UNITY_BUILD=TRUE",
|
||||
"CMAKE_LY_PROJECTS":"AutomatedTesting",
|
||||
"CMAKE_TARGET":"all",
|
||||
"CMAKE_BUILD_ARGS":"-j!NUMBER_OF_PROCESSORS!"
|
||||
}
|
||||
},
|
||||
"gradle": {
|
||||
"TAGS":[
|
||||
"weekly-build-metrics"
|
||||
],
|
||||
"COMMAND":"gradle_windows.cmd",
|
||||
"PARAMETERS": {
|
||||
"CONFIGURATION":"profile",
|
||||
"OUTPUT_DIRECTORY":"build\\android_gradle",
|
||||
"GAME_PROJECT": "AutomatedTesting",
|
||||
"ANDROID_NDK_PLATFORM": "21",
|
||||
"ANDROID_SDK_PLATFORM": "29",
|
||||
"SIGN_APK": "false",
|
||||
"GRADLE_BUILD_CMD": "build",
|
||||
"ADDITIONAL_GENERATE_ARGS": ""
|
||||
}
|
||||
},
|
||||
"periodic_test_profile": {
|
||||
"TAGS":[
|
||||
"nightly",
|
||||
"weekly-build-metrics"
|
||||
],
|
||||
"COMMAND":"build_and_run_unit_tests.cmd",
|
||||
"PARAMETERS": {
|
||||
"CONFIGURATION":"profile",
|
||||
"OUTPUT_DIRECTORY":"build\\android_unittest",
|
||||
"GAME_PROJECT": "AutomatedTesting",
|
||||
"ANDROID_NDK_PLATFORM": "21",
|
||||
"ANDROID_SDK_PLATFORM": "29",
|
||||
"SIGN_APK": "true",
|
||||
"GRADLE_BUILD_CMD": "assemble",
|
||||
"ADDITIONAL_GENERATE_ARGS": "--unit-test"
|
||||
}
|
||||
}
|
||||
|
||||
}
|
||||
@@ -0,0 +1,213 @@
|
||||
@ECHO OFF
|
||||
REM
|
||||
REM All or portions of this file Copyright (c) Amazon.com, Inc. or its affiliates or
|
||||
REM its licensors.
|
||||
REM
|
||||
REM For complete copyright and license terms please see the LICENSE at the root of this
|
||||
REM distribution (the "License"). All use of this software is governed by the License,
|
||||
REM or, if provided, by the license below or the license accompanying this file. Do not
|
||||
REM remove or modify any license notices. This file is distributed on an "AS IS" BASIS,
|
||||
REM WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
REM
|
||||
|
||||
SETLOCAL EnableDelayedExpansion
|
||||
|
||||
IF NOT EXIST "%LY_3RDPARTY_PATH%" (
|
||||
ECHO [ci_build] LY_3RDPARTY_PATH is invalid or not set
|
||||
GOTO :error
|
||||
)
|
||||
|
||||
IF NOT EXIST "%GRADLE_HOME%" (
|
||||
REM This is the default for developers
|
||||
SET GRADLE_HOME=C:\Gradle\gradle-5.6.4
|
||||
)
|
||||
IF NOT EXIST "%GRADLE_HOME%" (
|
||||
ECHO [ci_build] FAIL: GRADLE_HOME=%GRADLE_HOME%
|
||||
GOTO :error
|
||||
)
|
||||
|
||||
IF NOT EXIST "%CMAKE_HOME%" (
|
||||
SET CMAKE_HOME=%LY_3RDPARTY_PATH%/CMake/3.19.1/Windows/
|
||||
)
|
||||
IF NOT EXIST "%CMAKE_HOME%" (
|
||||
ECHO [ci_build] FAIL: CMAKE_HOME=%CMAKE_HOME%
|
||||
GOTO :error
|
||||
)
|
||||
|
||||
IF NOT EXIST "%LY_NINJA_PATH%" (
|
||||
SET LY_NINJA_PATH=%LY_3RDPARTY_PATH%/ninja/1.10.1/Windows
|
||||
)
|
||||
IF NOT EXIST "%LY_NINJA_PATH%" (
|
||||
ECHO [ci_build] FAIL: LY_NINJA_PATH=%LY_NINJA_PATH%
|
||||
GOTO :error
|
||||
)
|
||||
|
||||
REM Make sure that Ninja in the Path variable
|
||||
ECHO Testing ninja on the current path variable
|
||||
call ninja --version
|
||||
IF %ERRORLEVEL%==0 GOTO ninja_on_path
|
||||
ECHO Ninja wasnt in the call path, add the value set by LY_NINJA_PATH
|
||||
SET PATH=%PATH%;%LY_NINJA_PATH%
|
||||
|
||||
:ninja_on_path
|
||||
IF NOT EXIST "%LY_ANDROID_SDK%" (
|
||||
SET LY_ANDROID_SDK=!LY_3RDPARTY_PATH!/android-sdk/platform-29
|
||||
)
|
||||
IF NOT EXIST "%LY_ANDROID_SDK%" (
|
||||
ECHO [ci_build] FAIL: LY_ANDROID_SDK=!LY_ANDROID_SDK!
|
||||
GOTO :error
|
||||
)
|
||||
|
||||
IF NOT EXIST "%LY_ANDROID_NDK%" (
|
||||
set LY_ANDROID_NDK=!LY_3RDPARTY_PATH!/android-ndk/r21d
|
||||
)
|
||||
IF NOT EXIST "%LY_ANDROID_NDK%" (
|
||||
ECHO [ci_build] LY_ANDROID_NDK=!LY_ANDROID_NDK!
|
||||
GOTO :error
|
||||
)
|
||||
|
||||
IF NOT EXIST %OUTPUT_DIRECTORY% (
|
||||
mkdir %OUTPUT_DIRECTORY%
|
||||
)
|
||||
|
||||
REM Jenkins reports MSB8029 when TMP/TEMP is not defined, define a dummy folder
|
||||
SET TMP=%cd%/temp
|
||||
SET TEMP=%cd%/temp
|
||||
IF NOT EXIST %TMP% (
|
||||
mkdir %TMP%
|
||||
)
|
||||
|
||||
REM Optionally sign the APK if we are generating an APK
|
||||
SET GENERATE_SIGNED_APK=false
|
||||
IF "%SIGN_APK%"=="true" (
|
||||
IF "%GRADLE_BUILD_CMD%"=="assemble" (
|
||||
SET GENERATE_SIGNED_APK=true
|
||||
)
|
||||
)
|
||||
|
||||
SET PYTHON=python\python.cmd
|
||||
|
||||
REM Regardless of whether or not we generate a signing key, apparently we must set variables outside of
|
||||
REM an IF clause otherwise it will not work.
|
||||
|
||||
REM First look for the JDK HOME in the environment variable
|
||||
IF EXIST "%JDK_HOME%" (
|
||||
ECHO JDK Home found in Environment: !JDK_HOME!
|
||||
GOTO JDK_FOUND
|
||||
)
|
||||
|
||||
REM Next, look in the registry
|
||||
FOR /F "skip=2 tokens=1,2*" %%A IN ('REG QUERY "HKEY_LOCAL_MACHINE\SOFTWARE\JavaSoft\Java Development Kit\1.8" /v "JavaHome" 2^>nul') DO (
|
||||
SET JDK_REG_VALUE=%%C
|
||||
)
|
||||
IF EXIST "%JDK_REG_VALUE%" (
|
||||
SET JDK_HOME=!JDK_REG_VALUE!
|
||||
ECHO JDK Home found in registry: !JDK_HOME!
|
||||
GOTO JDK_FOUND
|
||||
)
|
||||
|
||||
ECHO Unable to locate JDK_HOME
|
||||
GOTO error
|
||||
|
||||
:JDK_FOUND
|
||||
|
||||
SET JDK_BIN=%JDK_HOME%\bin
|
||||
IF NOT EXIST "%JDK_BIN%" (
|
||||
ECHO The environment variable JDK_HOME is not set to a valid JDK 1.8 folder %JDK_BIN%
|
||||
ECHO Make sure the variable is set to your local JDK 1.8 installation
|
||||
GOTO error
|
||||
)
|
||||
|
||||
SET KEYTOOL_PATH=%JDK_BIN%\keytool.exe
|
||||
IF NOT EXIST "%KEYTOOL_PATH%" (
|
||||
ECHO The environment variable JDK_HOME is not set to a valid JDK 1.8 folder. Cannot find keytool at %JDK_BIN%\keytool.exe
|
||||
ECHO Make sure the variable is set to your local JDK 1.8 installation
|
||||
GOTO error
|
||||
)
|
||||
|
||||
SET CI_ANDROID_KEYSTORE_FILE=ly-android-dev.keystore
|
||||
SET CI_ANDROID_KEYSTORE_ALIAS=ly-android
|
||||
SET CI_ANDROID_KEYSTORE_PASSWORD=lumberyard
|
||||
SET CI_ANDROID_KEYSTORE_DN=cn=LY Developer, ou=Lumberyard, o=Amazon, c=US
|
||||
SET CI_KEYSTORE_VALIDITY_DAYS=10000
|
||||
SET CI_KEYSTORE_CERT_DN=cn=LY Developer, ou=Lumberyard, o=Amazon, c=US
|
||||
|
||||
REM Clear out any existing keystore file since the password/alias may have changed
|
||||
SET CI_ANDROID_KEYSTORE_FILE_ABS=%cd%\%OUTPUT_DIRECTORY%\%CI_ANDROID_KEYSTORE_FILE%
|
||||
|
||||
|
||||
IF "%GENERATE_SIGNED_APK%"=="true" (
|
||||
|
||||
REM Prepare a temporary keystore just for this unit test session
|
||||
|
||||
REM Generate the keystore file if needed
|
||||
IF NOT EXIST "%CI_ANDROID_KEYSTORE_FILE_ABS%" (
|
||||
|
||||
ECHO [ci_build] Generating keystore file %CI_ANDROID_KEYSTORE_FILE%
|
||||
ECHO [ci_build] "%KEYTOOL_PATH%" -genkeypair -v -keystore %CI_ANDROID_KEYSTORE_FILE_ABS% -storepass %CI_ANDROID_KEYSTORE_PASSWORD% -alias %CI_ANDROID_KEYSTORE_ALIAS% -keypass %CI_ANDROID_KEYSTORE_PASSWORD% -keyalg RSA -keysize 2048 -validity %CI_KEYSTORE_VALIDITY_DAYS% -dname "%CI_KEYSTORE_CERT_DN%"
|
||||
CALL "%KEYTOOL_PATH%" -genkeypair -v -keystore %CI_ANDROID_KEYSTORE_FILE_ABS% -storepass %CI_ANDROID_KEYSTORE_PASSWORD% -alias %CI_ANDROID_KEYSTORE_ALIAS% -keypass %CI_ANDROID_KEYSTORE_PASSWORD% -keyalg RSA -keysize 2048 -validity %CI_KEYSTORE_VALIDITY_DAYS% -dname "%CI_KEYSTORE_CERT_DN%" 2> nul
|
||||
IF errorlevel 1 (
|
||||
ECHO Unable to generate keystore file "%CI_ANDROID_KEYSTORE_FILE_ABS%"
|
||||
GOTO error
|
||||
)
|
||||
) ELSE (
|
||||
ECHO Using keystore file at %CI_ANDROID_KEYSTORE_FILE_ABS%
|
||||
)
|
||||
|
||||
ECHO [ci_build] %PYTHON% cmake\Tools\Platform\Android\generate_android_project.py --engine-root=. --build-dir=%OUTPUT_DIRECTORY% -g %GAME_PROJECT% --gradle-install-path=%GRADLE_HOME% --cmake-install-path=%CMAKE_HOME% --ninja-install-path=%LY_NINJA_PATH% --third-party-path=%LY_3RDPARTY_PATH% --android-ndk-path=%LY_ANDROID_NDK% --android-sdk-path=%LY_ANDROID_SDK% --android-ndk-version=%ANDROID_NDK_PLATFORM% --android-sdk-version=%ANDROID_SDK_PLATFORM% --signconfig-store-file %CI_ANDROID_KEYSTORE_FILE_ABS% --signconfig-store-password %CI_ANDROID_KEYSTORE_PASSWORD% --signconfig-key-alias %CI_ANDROID_KEYSTORE_ALIAS% --signconfig-key-password %CI_ANDROID_KEYSTORE_PASSWORD% %OPTIONAL_TEST_FLAG% %ADDITIONAL_GENERATE_ARGS% --overwrite-existing
|
||||
CALL %PYTHON% cmake\Tools\Platform\Android\generate_android_project.py --engine-root=. --build-dir=%OUTPUT_DIRECTORY% -g %GAME_PROJECT% --gradle-install-path=%GRADLE_HOME% --cmake-install-path=%CMAKE_HOME% --ninja-install-path=%LY_NINJA_PATH% --third-party-path=%LY_3RDPARTY_PATH% --android-ndk-path=%LY_ANDROID_NDK% --android-sdk-path=%LY_ANDROID_SDK% --android-ndk-version=%ANDROID_NDK_PLATFORM% --android-sdk-version=%ANDROID_SDK_PLATFORM% --signconfig-store-file %CI_ANDROID_KEYSTORE_FILE_ABS% --signconfig-store-password %CI_ANDROID_KEYSTORE_PASSWORD% --signconfig-key-alias %CI_ANDROID_KEYSTORE_ALIAS% --signconfig-key-password %CI_ANDROID_KEYSTORE_PASSWORD% %ADDITIONAL_GENERATE_ARGS% --overwrite-existing
|
||||
) ELSE (
|
||||
ECHO [ci_build] %PYTHON% cmake\Tools\Platform\Android\generate_android_project.py --engine-root=. --build-dir=%OUTPUT_DIRECTORY% -g %GAME_PROJECT% --gradle-install-path=%GRADLE_HOME% --cmake-install-path=%CMAKE_HOME% --ninja-install-path=%LY_NINJA_PATH% --third-party-path=%LY_3RDPARTY_PATH% --android-ndk-path=%LY_ANDROID_NDK% --android-sdk-path=%LY_ANDROID_SDK% --android-ndk-version=%ANDROID_NDK_PLATFORM% --android-sdk-version=%ANDROID_SDK_PLATFORM% %ADDITIONAL_GENERATE_ARGS% --overwrite-existing
|
||||
CALL %PYTHON% cmake\Tools\Platform\Android\generate_android_project.py --engine-root=. --build-dir=%OUTPUT_DIRECTORY% -g %GAME_PROJECT% --gradle-install-path=%GRADLE_HOME% --cmake-install-path=%CMAKE_HOME% --ninja-install-path=%LY_NINJA_PATH% --third-party-path=%LY_3RDPARTY_PATH% --android-ndk-path=%LY_ANDROID_NDK% --android-sdk-path=%LY_ANDROID_SDK% --android-ndk-version=%ANDROID_NDK_PLATFORM% --android-sdk-version=%ANDROID_SDK_PLATFORM% %ADDITIONAL_GENERATE_ARGS% --overwrite-existing
|
||||
)
|
||||
|
||||
REM Validate the android project generation
|
||||
IF %ERRORLEVEL%==0 GOTO generate_project_success
|
||||
ECHO Error Generating Android Project
|
||||
goto error
|
||||
|
||||
:generate_project_success
|
||||
|
||||
REM Run the gradle build from the output directory
|
||||
PUSHD %OUTPUT_DIRECTORY%
|
||||
|
||||
REM Stop any running or orphaned gradle daemon
|
||||
ECHO [ci_build] gradlew --stop
|
||||
CALL gradlew --stop
|
||||
|
||||
ECHO [ci_build] gradlew --no-daemon %GRADLE_BUILD_CMD%%CONFIGURATION%
|
||||
CALL gradlew --no-daemon %GRADLE_BUILD_CMD%%CONFIGURATION%
|
||||
|
||||
IF %ERRORLEVEL%==0 GOTO gradle_build_success
|
||||
|
||||
REM Do another build with the debug flag to try to get the failure reasons
|
||||
|
||||
GOTO error
|
||||
|
||||
ECHO Error building gradle. Rebuilding with debug information
|
||||
ECHO [ci_build] gradlew --debug --full-stacktrace --no-daemon %GRADLE_BUILD_CMD%%CONFIGURATION%
|
||||
CALL gradlew --debug --full-stacktrace --no-daemon %GRADLE_BUILD_CMD%%CONFIGURATION%
|
||||
|
||||
ECHO [ci_build] gradlew --stop
|
||||
CALL gradlew --stop
|
||||
|
||||
POPD
|
||||
|
||||
GOTO error
|
||||
|
||||
|
||||
:gradle_build_success
|
||||
|
||||
ECHO [ci_build] gradlew --stop
|
||||
CALL gradlew --stop
|
||||
|
||||
POPD
|
||||
|
||||
EXIT /b 0
|
||||
|
||||
:popd_error
|
||||
POPD
|
||||
|
||||
:error
|
||||
|
||||
EXIT /b 1
|
||||
@@ -0,0 +1,18 @@
|
||||
{
|
||||
"ENV": {
|
||||
"GRADLE_HOME": "C:/Gradle/gradle-5.6.4",
|
||||
"NODE_LABEL": "windows-047e5cdf",
|
||||
"LY_3RDPARTY_PATH": "C:/ly/3rdParty",
|
||||
"TIMEOUT": 30,
|
||||
"WORKSPACE": "D:/workspace",
|
||||
"MOUNT_VOLUME": true
|
||||
},
|
||||
"PIPELINE_ENV_OVERRIDE": {
|
||||
"daily-pipeline-metrics": {
|
||||
"CLEAN_WORKSPACE": true
|
||||
},
|
||||
"packaging": {
|
||||
"CLEAN_WORKSPACE": true
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,544 @@
|
||||
#
|
||||
# All or portions of this file Copyright (c) Amazon.com, Inc. or its affiliates or
|
||||
# its licensors.
|
||||
#
|
||||
# For complete copyright and license terms please see the LICENSE at the root of this
|
||||
# distribution (the "License"). All use of this software is governed by the License,
|
||||
# or, if provided, by the license below or the license accompanying this file. Do not
|
||||
# remove or modify any license notices. This file is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
#
|
||||
|
||||
import argparse
|
||||
import os
|
||||
import pathlib
|
||||
import re
|
||||
import sys
|
||||
import subprocess
|
||||
import time
|
||||
import logging
|
||||
|
||||
CURRENT_PATH = pathlib.Path(os.path.dirname(__file__)).absolute()
|
||||
|
||||
ENGINE_ROOT = CURRENT_PATH.parent.parent.parent.parent.parent.parent
|
||||
|
||||
|
||||
class AndroidEmuError(Exception):
|
||||
pass
|
||||
|
||||
|
||||
def get_android_sdk_path():
|
||||
try:
|
||||
android_sdk_path = pathlib.Path(os.getenv('LY_ANDROID_SDK'))
|
||||
if not android_sdk_path:
|
||||
raise AndroidEmuError(f"LY_ANDROID_SDK environment variable is not set")
|
||||
if not android_sdk_path.is_dir():
|
||||
raise AndroidEmuError(f"Android SDK Path ('{android_sdk_path}') set with the LY_ANDROID_SDK variable is invalid")
|
||||
#TODO: Sanity check on necessary files
|
||||
return android_sdk_path
|
||||
except Exception as err:
|
||||
raise AndroidEmuError(f"Unable to determine android SDK path: {err}")
|
||||
|
||||
|
||||
class Command(object):
|
||||
|
||||
def __init__(self, tool_name, tool_path, run_as_shell=True):
|
||||
|
||||
if not tool_path.is_file():
|
||||
raise AndroidEmuError(f"Invalid path for {tool_name}. Cannot find ('{tool_path.absolute()}')")
|
||||
|
||||
self.tool_path = tool_path
|
||||
self.run_as_shell = run_as_shell
|
||||
|
||||
def run_return_output(self, cmd_args):
|
||||
|
||||
args = [str(self.tool_path)]
|
||||
if isinstance(cmd_args, str):
|
||||
args.append(cmd_args)
|
||||
elif isinstance(cmd_args, list):
|
||||
args.extend(cmd_args)
|
||||
else:
|
||||
assert False, "run_return_output argument must be a string or list of strings"
|
||||
|
||||
full_cmd = subprocess.list2cmdline(args)
|
||||
logging.debug(f"run_return_output: {full_cmd}")
|
||||
|
||||
run_result = subprocess.run(args,
|
||||
capture_output=True,
|
||||
encoding='UTF-8',
|
||||
errors='ignore',
|
||||
shell=self.run_as_shell)
|
||||
if run_result.returncode != 0:
|
||||
raise AndroidEmuError(f"Error executing command '{full_cmd}' (return code {run_result.returncode}): {run_result.stderr}")
|
||||
|
||||
return run_result.stdout
|
||||
|
||||
def run(self, cmd_args, cwd=None, suppress_output=False):
|
||||
args = [str(self.tool_path)]
|
||||
if isinstance(cmd_args, str):
|
||||
args.append(cmd_args)
|
||||
elif isinstance(cmd_args, list):
|
||||
args.extend(cmd_args)
|
||||
else:
|
||||
assert False, "run_return_output argument must be a string or list of strings"
|
||||
|
||||
full_cmd = subprocess.list2cmdline(args)
|
||||
logging.debug(f"run: {full_cmd}")
|
||||
|
||||
run_result = subprocess.run(args,
|
||||
#stdout=subprocess.DEVNULL if suppress_output else subprocess.STDOUT,
|
||||
capture_output=False,
|
||||
shell=self.run_as_shell,
|
||||
cwd=cwd)
|
||||
if run_result.returncode != 0:
|
||||
raise AndroidEmuError(f"Error executing command '{full_cmd}' (return code {run_result.returncode}): {run_result.stderr}")
|
||||
|
||||
def run_process(self, cmd_args):
|
||||
args = [str(self.tool_path)]
|
||||
if isinstance(cmd_args, str):
|
||||
args.append(cmd_args)
|
||||
elif isinstance(cmd_args, list):
|
||||
args.extend(cmd_args)
|
||||
else:
|
||||
assert False, "run_return_output argument must be a string or list of strings"
|
||||
|
||||
full_cmd = subprocess.list2cmdline(args)
|
||||
logging.debug(f"run_process: {full_cmd}")
|
||||
|
||||
process = subprocess.Popen(args,
|
||||
shell=True,
|
||||
stdout=subprocess.PIPE,
|
||||
creationflags=subprocess.CREATE_NEW_PROCESS_GROUP | subprocess.NORMAL_PRIORITY_CLASS |
|
||||
subprocess.CREATE_NO_WINDOW,
|
||||
encoding='UTF-8',
|
||||
errors='ignore')
|
||||
return process
|
||||
|
||||
|
||||
class AndroidEmulatorManager(object):
|
||||
|
||||
UNIT_TEST_AVD_NAME = "LY_UNITTEST_AVD"
|
||||
|
||||
UNIT_TEST_SYSTEM_IMAGE_PACKAGE = "android-30;google_apis;x86_64"
|
||||
|
||||
UNIT_TEST_DEVICE_TEMPLATE_NAME = "pixel_xl"
|
||||
|
||||
UNIT_TEST_DEVICE_SETTINGS_MAP = {
|
||||
"disk.dataPartition.size": "32G",
|
||||
"vm.heapSize": "1024",
|
||||
"hw.ramSize": "2048",
|
||||
"hw.sdCard": "no"
|
||||
}
|
||||
EMULATOR_STARTUP_TIMEOUT_SECS = 60*5 # Set the emulator startup timeout to 5 minutes
|
||||
|
||||
def __init__(self, base_android_sdk_path, hide_emulator_windows=True, force_avd_creation=False, emulator_startup_timeout=EMULATOR_STARTUP_TIMEOUT_SECS):
|
||||
|
||||
self.android_sdk_path = base_android_sdk_path
|
||||
|
||||
self.force_avd_creation = force_avd_creation
|
||||
|
||||
self.unit_test_avd_name = AndroidEmulatorManager.UNIT_TEST_AVD_NAME
|
||||
self.unit_test_device_template_name = AndroidEmulatorManager.UNIT_TEST_DEVICE_TEMPLATE_NAME
|
||||
self.unit_test_device_settings_map = AndroidEmulatorManager.UNIT_TEST_DEVICE_SETTINGS_MAP
|
||||
self.unit_test_avd_system_image = AndroidEmulatorManager.UNIT_TEST_SYSTEM_IMAGE_PACKAGE
|
||||
self.hide_emulator_windows = hide_emulator_windows
|
||||
|
||||
self.emulator_startup_timeout = emulator_startup_timeout
|
||||
self.emulator_cmd = Command("Emulator", self.android_sdk_path / 'emulator' / 'emulator.exe')
|
||||
|
||||
self.avd_manager_cmd = Command("AVD Manager", self.android_sdk_path / 'tools' / 'bin' / 'avdmanager.bat')
|
||||
|
||||
self.sdk_manager_cmd = Command("SDK Manager", self.android_sdk_path / 'tools' / 'bin' / 'sdkmanager.bat')
|
||||
|
||||
self.adb_cmd = Command("ADB", self.android_sdk_path / 'platform-tools' / 'adb.exe')
|
||||
|
||||
def collect_android_sdk_list(self):
|
||||
"""
|
||||
Use the SDK Manager to get the list of installed, available, and updateable packages
|
||||
:return: tuple of 3 lists: installed, available, and updateable packages
|
||||
"""
|
||||
|
||||
result_str = self.sdk_manager_cmd.run_return_output(['--list'])
|
||||
|
||||
# the result will be listed out in 3 sections: Installed packages, Available Packages, and Available updates
|
||||
# and each item is represented by 3 columns separated by a '|' character
|
||||
installed_packages = []
|
||||
available_packages = []
|
||||
available_updates = []
|
||||
|
||||
current_append_list = None
|
||||
for avd_item in result_str.split('\n'):
|
||||
avd_item_stripped = avd_item.strip()
|
||||
if not avd_item_stripped:
|
||||
continue
|
||||
if '|' not in avd_item_stripped:
|
||||
if avd_item_stripped.upper() == 'INSTALLED PACKAGES:':
|
||||
current_append_list = installed_packages
|
||||
elif avd_item_stripped.upper() == 'AVAILABLE PACKAGES:':
|
||||
current_append_list = available_packages
|
||||
elif avd_item_stripped.upper() == 'AVAILABLE UPDATES:':
|
||||
current_append_list = available_updates
|
||||
else:
|
||||
current_append_list = None
|
||||
continue
|
||||
item_parts = [split.strip() for split in avd_item_stripped.split('|')]
|
||||
if len(item_parts) < 3:
|
||||
continue
|
||||
elif item_parts[1].upper() in ('VERSION', 'INSTALLED', '-------'):
|
||||
continue
|
||||
elif current_append_list is None:
|
||||
continue
|
||||
|
||||
if current_append_list is not None:
|
||||
current_append_list.append(item_parts)
|
||||
|
||||
return installed_packages, available_packages, available_updates
|
||||
|
||||
def install_system_package_if_necessary(self):
|
||||
"""
|
||||
Make sure that we have the correct system image installed, and install if not
|
||||
"""
|
||||
|
||||
installed_packages, available_packages, _ = self.collect_android_sdk_list()
|
||||
|
||||
unit_test_sdk_package_name = f'system-images;{self.unit_test_avd_system_image}'
|
||||
detected_sdk_package_version = None
|
||||
for package_line_items in installed_packages:
|
||||
if package_line_items[0] == unit_test_sdk_package_name:
|
||||
detected_sdk_package_version = package_line_items[0]
|
||||
|
||||
if detected_sdk_package_version:
|
||||
# Already installed
|
||||
logging.info(f"Detected installed system image {self.unit_test_avd_system_image} version {detected_sdk_package_version}")
|
||||
return
|
||||
|
||||
# Make sure its an available image to install
|
||||
detected_available_sdk_package_version = None
|
||||
for package_line_items in available_packages:
|
||||
if package_line_items[0] == unit_test_sdk_package_name:
|
||||
detected_available_sdk_package_version = package_line_items[0]
|
||||
if not detected_available_sdk_package_version:
|
||||
raise AndroidEmuError(f"Unable to install required system image {self.unit_test_avd_system_image}, not found by the Android SDK Manager")
|
||||
|
||||
# Install the package
|
||||
logging.info(f"Installing system image {self.unit_test_avd_system_image}...")
|
||||
self.sdk_manager_cmd.run(['--install', unit_test_sdk_package_name])
|
||||
logging.info(f"Installed Completed")
|
||||
|
||||
def find_device_id_by_name(self, device_name):
|
||||
"""
|
||||
Find a device id (from AVD Manager) by the device name
|
||||
:param device_name: Name to lookup
|
||||
:return: The device id
|
||||
"""
|
||||
result_str = self.avd_manager_cmd.run_return_output(['list', 'device'])
|
||||
result_lines = [result_line.strip() for result_line in result_str.split('\n')]
|
||||
|
||||
result_line_count = len(result_lines)
|
||||
current_index = 0
|
||||
|
||||
device_to_id_map = {}
|
||||
|
||||
while current_index < result_line_count:
|
||||
|
||||
current_line = result_lines[current_index]
|
||||
current_index += 1
|
||||
|
||||
# This assumes the pattern "id: <id> or "<device name>"
|
||||
if current_line.startswith('id:') and 'or' in current_line:
|
||||
id_and_name_combo = current_line.split('or')
|
||||
id_and_value_combo = id_and_name_combo[0].split(' ')
|
||||
name = id_and_name_combo[1].replace('"', '').strip().upper()
|
||||
id = id_and_value_combo[1]
|
||||
device_to_id_map[name] = id
|
||||
|
||||
if current_line.startswith('Available Android targets:'):
|
||||
break
|
||||
|
||||
device_id = device_to_id_map.get(device_name.upper())
|
||||
if not device_id:
|
||||
raise AndroidEmuError(f"Unable to locate device id for '{device_name}'")
|
||||
return device_id
|
||||
|
||||
def query_installed_avds(self):
|
||||
"""
|
||||
Get maps of all valid and invalid AVDs installed on the current system
|
||||
:return: tuple of 2 maps (AVD Name -> Path): Valid and invalid
|
||||
"""
|
||||
result_str = self.avd_manager_cmd.run_return_output(['list', 'avd'])
|
||||
result_lines = [result_line.strip() for result_line in result_str.split('\n')]
|
||||
|
||||
line_count = len(result_lines)
|
||||
current_index = 0
|
||||
current_name = None
|
||||
current_path = None
|
||||
|
||||
valid_avd_to_path_map = {}
|
||||
invalid_avd_to_path_map = {}
|
||||
current_avd_to_path_map = valid_avd_to_path_map
|
||||
|
||||
while current_index < line_count:
|
||||
current_line = result_lines[current_index]
|
||||
current_index += 1
|
||||
|
||||
if current_line.startswith('Name:'):
|
||||
name = current_line[6:].strip()
|
||||
if current_name is not None:
|
||||
current_avd_to_path_map[current_name] = current_path
|
||||
current_path = None
|
||||
current_name = name
|
||||
|
||||
elif current_line.startswith('Path:'):
|
||||
current_path = current_line[6:].strip()
|
||||
elif current_line.startswith('Device:'):
|
||||
pass
|
||||
elif 'could not be loaded:' in current_line:
|
||||
if current_name is not None:
|
||||
current_avd_to_path_map[current_name] = current_path
|
||||
current_avd_to_path_map = invalid_avd_to_path_map
|
||||
current_path = None
|
||||
current_name = None
|
||||
|
||||
if current_name is not None:
|
||||
current_avd_to_path_map[current_name] = current_path
|
||||
|
||||
return valid_avd_to_path_map, invalid_avd_to_path_map
|
||||
|
||||
def create_unitest_avd(self):
|
||||
"""Create the unit test AVD"""
|
||||
|
||||
self.install_system_package_if_necessary()
|
||||
|
||||
device_id = self.find_device_id_by_name(self.unit_test_device_template_name)
|
||||
|
||||
self.avd_manager_cmd.run(['--silent',
|
||||
'create', 'avd',
|
||||
'--name', self.unit_test_avd_name,
|
||||
'--package', f'system-images;{self.unit_test_avd_system_image}',
|
||||
'--device', device_id])
|
||||
|
||||
valid_avd_map, _ = self.query_installed_avds()
|
||||
unit_test_avd_path = valid_avd_map.get(self.unit_test_avd_name)
|
||||
if not unit_test_avd_path:
|
||||
raise AndroidEmuError(f"Unable to create unit test AVD {self.unit_test_avd_name}")
|
||||
|
||||
unit_test_avd_config_path = pathlib.Path(unit_test_avd_path) / 'config.ini'
|
||||
if not unit_test_avd_config_path.is_file():
|
||||
raise AndroidEmuError(f"Unable to create unit test AVD {self.unit_test_avd_name}: The expected config file '{unit_test_avd_config_path}' does not exist.")
|
||||
|
||||
config_content_full = unit_test_avd_config_path.read_text(encoding='UTF-8', errors='ignore')
|
||||
for item, value in self.unit_test_device_settings_map.items():
|
||||
regex_friendly_str = item.replace('.', '\\.')
|
||||
repl_pattern = f"{regex_friendly_str}\\s*=\\s*[\\d]+"
|
||||
repl_value = f"{item}={value}"
|
||||
if re.search(repl_pattern, config_content_full):
|
||||
config_content_full = re.sub(repl_pattern, repl_value, config_content_full)
|
||||
else:
|
||||
if not config_content_full.endswith('\n'):
|
||||
config_content_full += '\n'
|
||||
config_content_full += f"{repl_value}\n"
|
||||
|
||||
unit_test_avd_config_path.write_text(config_content_full)
|
||||
|
||||
def query_emulator_device_id(self):
|
||||
result_str = self.adb_cmd.run_return_output(['devices', '-l'])
|
||||
emulators = []
|
||||
for result_line in result_str.split('\n'):
|
||||
if not result_line.startswith('emulator-'):
|
||||
continue
|
||||
emulator = result_line[:result_line.find(' ')].strip()
|
||||
emulators.append(emulator)
|
||||
|
||||
if len(emulators) > 1:
|
||||
logging.warning(f"Found multiple emulators connect ({','.join(emulators)}). Defaulting to {emulators[0]}")
|
||||
|
||||
return emulators[0] if len(emulators) > 0 else None
|
||||
|
||||
def install_unit_test_avd(self):
|
||||
"""
|
||||
Install the unit test AVD (Android Virtual Device)
|
||||
"""
|
||||
|
||||
valid_avd_map, invalid_avd_map = self.query_installed_avds()
|
||||
|
||||
if not self.unit_test_avd_name in valid_avd_map:
|
||||
create_avd = True
|
||||
elif self.force_avd_creation or self.unit_test_avd_name in invalid_avd_map:
|
||||
logging.info(f"Deleting AVD {self.unit_test_avd_name}..")
|
||||
self.avd_manager_cmd.run(['delete', 'avd', '--name', self.unit_test_avd_name])
|
||||
create_avd = True
|
||||
else:
|
||||
create_avd = False
|
||||
|
||||
if create_avd:
|
||||
self.create_unitest_avd()
|
||||
|
||||
def uninstall_unit_test_avd(self):
|
||||
"""
|
||||
Uninstall the unit test AVD
|
||||
"""
|
||||
logging.info(f"Uninstalling AVD {self.unit_test_avd_name}..")
|
||||
self.avd_manager_cmd.run(['delete', 'avd', '--name', self.unit_test_avd_name])
|
||||
|
||||
def launch_emulator_process(self):
|
||||
"""
|
||||
Launch the emulator process for the unit test avd and return the process handle and its device id
|
||||
:return: tuple of the process handle and the device id for the emulator
|
||||
"""
|
||||
emulator_device_id = None
|
||||
process = None
|
||||
try:
|
||||
# Launch the emulator process
|
||||
emulator_process_args = [
|
||||
"-avd",
|
||||
self.unit_test_avd_name
|
||||
]
|
||||
if self.hide_emulator_windows:
|
||||
emulator_process_args.append("-no-window")
|
||||
process = self.emulator_cmd.run_process(emulator_process_args)
|
||||
|
||||
# Wait for the emulator to signal that its bootup is complete
|
||||
boot_completed = False
|
||||
start_time = time.time()
|
||||
timeout_secs = 360
|
||||
while process.poll() is None:
|
||||
elapsed_time = time.time() - start_time
|
||||
if elapsed_time > timeout_secs > 0:
|
||||
break
|
||||
line = process.stdout.readline()
|
||||
print(line, end='')
|
||||
if "boot completed" in line:
|
||||
boot_completed = True
|
||||
break
|
||||
if not boot_completed:
|
||||
raise AndroidEmuError("Bootup of emulator timed out")
|
||||
|
||||
# query ADB to get the emulator ID
|
||||
emulator_device_id = self.query_emulator_device_id()
|
||||
|
||||
return process, emulator_device_id
|
||||
|
||||
except Exception:
|
||||
if process:
|
||||
if emulator_device_id:
|
||||
self.terminate_emulator_process(emulator_device_id)
|
||||
else:
|
||||
process.kill()
|
||||
raise
|
||||
|
||||
def terminate_emulator_process(self, device_id):
|
||||
# Terminate the emulator
|
||||
kill_emu_args = [
|
||||
'-s', device_id,
|
||||
'emu', 'kill'
|
||||
]
|
||||
self.adb_cmd.run(kill_emu_args)
|
||||
|
||||
def run_emulation_process(self, process_func):
|
||||
"""
|
||||
Execute a function that relies on the session based android simulator.
|
||||
|
||||
:param process_func: The process function to execute. Function requires one argument which will be the device id
|
||||
:return: The return value of the process function
|
||||
"""
|
||||
|
||||
emulator_device_id = None
|
||||
try:
|
||||
emulator_process, emulator_device_id = self.launch_emulator_process()
|
||||
|
||||
return process_func(emulator_device_id)
|
||||
|
||||
finally:
|
||||
if emulator_device_id is not None:
|
||||
self.terminate_emulator_process(emulator_device_id)
|
||||
|
||||
|
||||
def process_unit_test_on_simulator(base_android_sdk_path, build_path, build_config):
|
||||
"""
|
||||
Run the android unit tests on a sessioned simulator
|
||||
|
||||
:param base_android_sdk_path: The path to where the Android SDK exists
|
||||
:param build_path: The build path relative to the engine root where the android unit test project is configured and built
|
||||
:param build_config: The configuration of the build unit test APK to run
|
||||
"""
|
||||
|
||||
python_cmd = Command("Python", ENGINE_ROOT / 'python' / 'python.cmd')
|
||||
|
||||
android_script_root = ENGINE_ROOT / 'cmake' / 'Tools' / 'Platform' / 'Android'
|
||||
assert android_script_root.is_dir(), "Missing the android scripts path in the engine folder hierarchy"
|
||||
deploy_android_py_path = android_script_root / 'deploy_android.py'
|
||||
assert deploy_android_py_path.is_file(), "Missing the android deployment script in the engine folder hierarchy"
|
||||
launch_android_ptest_py_path = android_script_root / 'launch_android_test.py'
|
||||
assert launch_android_ptest_py_path.is_file(), "Missing the android unit test launcher script in the engine folder hierarchy"
|
||||
|
||||
def _install_and_run_unit_tests(emulator_id):
|
||||
|
||||
# install unit test on the emulator
|
||||
install_apk_args = [
|
||||
str(deploy_android_py_path),
|
||||
'-b', build_path,
|
||||
'-c', build_config,
|
||||
'--device-id-filter', emulator_id,
|
||||
'--clean'
|
||||
]
|
||||
python_cmd.run(cmd_args=install_apk_args,
|
||||
cwd=os.path.normpath(str(ENGINE_ROOT)))
|
||||
|
||||
try:
|
||||
# Launch the unit test on the emulator
|
||||
launch_apk_args = [
|
||||
str(launch_android_ptest_py_path),
|
||||
'-b', build_path,
|
||||
'-c', build_config,
|
||||
'--device-serial', emulator_id
|
||||
]
|
||||
python_cmd.run(cmd_args=launch_apk_args,
|
||||
cwd=os.path.normpath(str(ENGINE_ROOT)))
|
||||
return True
|
||||
except AndroidEmuError:
|
||||
print("\n\n")
|
||||
raise AndroidEmuError("Unit Tests Failed")
|
||||
|
||||
# Prepare the emulator manager
|
||||
manager = AndroidEmulatorManager(base_android_sdk_path=base_android_sdk_path,
|
||||
force_avd_creation=True)
|
||||
|
||||
# First Install or overwrite the unit test emulator
|
||||
manager.install_unit_test_avd()
|
||||
|
||||
# Run the emulator-dependent process based on the session AVD created by the manager
|
||||
manager.run_emulation_process(_install_and_run_unit_tests)
|
||||
|
||||
# Uninstall the AVD when done
|
||||
manager.uninstall_unit_test_avd()
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
|
||||
parser = argparse.ArgumentParser(description="Install and an android unit test APK on a android simulator.")
|
||||
|
||||
parser.add_argument('--android-sdk-path',
|
||||
help='Path to the Android SDK')
|
||||
parser.add_argument('--build-path',
|
||||
help='The build path (relative to the engine root) where the project was generated and the APK is built',
|
||||
required=True)
|
||||
parser.add_argument('--build-config',
|
||||
help='The build config of the built APK',
|
||||
required=True)
|
||||
parser.add_argument('--debug',
|
||||
help='Enable debug messages from this script',
|
||||
action="store_true")
|
||||
|
||||
parsed_args = parser.parse_args(sys.argv[1:])
|
||||
logging.basicConfig(format='%(levelname)s: %(message)s', level=logging.DEBUG if parsed_args.debug else logging.INFO)
|
||||
|
||||
try:
|
||||
base_android_sdk_path = pathlib.Path(parsed_args.android_sdk_path) if parsed_args.android_sdk_path else get_android_sdk_path()
|
||||
process_unit_test_on_simulator(base_android_sdk_path=base_android_sdk_path,
|
||||
build_path=parsed_args.build_path,
|
||||
build_config=parsed_args.build_config)
|
||||
exit(0)
|
||||
except AndroidEmuError as e:
|
||||
print(e)
|
||||
exit(1)
|
||||
|
||||
Executable
+32
@@ -0,0 +1,32 @@
|
||||
#!/usr/bin/env bash
|
||||
#
|
||||
# All or portions of this file Copyright (c) Amazon.com, Inc. or its affiliates or
|
||||
# its licensors.
|
||||
#
|
||||
# For complete copyright and license terms please see the LICENSE at the root of this
|
||||
# distribution (the "License"). All use of this software is governed by the License,
|
||||
# or, if provided, by the license below or the license accompanying this file. Do not
|
||||
# remove or modify any license notices. This file is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
#
|
||||
|
||||
set -o errexit # exit on the first failure encountered
|
||||
|
||||
if [[ ! -d $OUTPUT_DIRECTORY ]]; then
|
||||
echo [ci_build] Error: $OUTPUT_DIRECTORY was not found
|
||||
exit 1
|
||||
fi
|
||||
pushd $OUTPUT_DIRECTORY
|
||||
|
||||
if [[ ! -e $ASSET_PROCESSOR_BINARY ]]; then
|
||||
echo [ci_build] Error: $ASSET_PROCESSOR_BINARY was not found
|
||||
exit 1
|
||||
fi
|
||||
|
||||
for project in $(echo $CMAKE_LY_PROJECTS | sed "s/;/ /g")
|
||||
do
|
||||
echo [ci_build] ${ASSET_PROCESSOR_BINARY} $ASSET_PROCESSOR_OPTIONS --project-path=$project --platforms=$ASSET_PROCESSOR_PLATFORMS
|
||||
${ASSET_PROCESSOR_BINARY} $ASSET_PROCESSOR_OPTIONS --project-path=$project --platforms=$ASSET_PROCESSOR_PLATFORMS
|
||||
done
|
||||
|
||||
popd
|
||||
+18
@@ -0,0 +1,18 @@
|
||||
#!/usr/bin/env bash
|
||||
#
|
||||
# All or portions of this file Copyright (c) Amazon.com, Inc. or its affiliates or
|
||||
# its licensors.
|
||||
#
|
||||
# For complete copyright and license terms please see the LICENSE at the root of this
|
||||
# distribution (the "License"). All use of this software is governed by the License,
|
||||
# or, if provided, by the license below or the license accompanying this file. Do not
|
||||
# remove or modify any license notices. This file is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
#
|
||||
|
||||
set -o errexit # exit on the first failure encountered
|
||||
|
||||
BASEDIR=$(dirname "$0")
|
||||
source $BASEDIR/build_linux.sh
|
||||
|
||||
source $BASEDIR/asset_linux.sh
|
||||
@@ -0,0 +1,173 @@
|
||||
{
|
||||
"clean": {
|
||||
"TAGS": [],
|
||||
"COMMAND": "clean_linux.sh",
|
||||
"PARAMETERS": {
|
||||
"OUTPUT_DIRECTORY": "build",
|
||||
"CMAKE_LY_PROJECTS": "AutomatedTesting"
|
||||
}
|
||||
},
|
||||
"profile_pipe": {
|
||||
"TAGS": [
|
||||
"default"
|
||||
],
|
||||
"steps": [
|
||||
"profile_nounity",
|
||||
"asset_profile",
|
||||
"test_profile"
|
||||
]
|
||||
},
|
||||
"metrics": {
|
||||
"TAGS": [
|
||||
"weekly"
|
||||
],
|
||||
"COMMAND": "python_linux.sh",
|
||||
"PARAMETERS": {
|
||||
"SCRIPT_PATH": "scripts/build/ci_build_metrics.py",
|
||||
"SCRIPT_PARAMETERS": "--platform Linux --jobname '${JOB_NAME}' --jobnumber '${BUILD_NUMBER}' --jobnode '${NODE_NAME}' --changelist '${CHANGE_ID}'"
|
||||
}
|
||||
},
|
||||
"debug": {
|
||||
"TAGS": [
|
||||
"nightly",
|
||||
"weekly-build-metrics"
|
||||
],
|
||||
"COMMAND": "build_linux.sh",
|
||||
"PARAMETERS": {
|
||||
"CONFIGURATION": "debug",
|
||||
"OUTPUT_DIRECTORY": "build/linux",
|
||||
"CMAKE_OPTIONS": "-G 'Ninja Multi-Config' -DCMAKE_C_COMPILER=clang-6.0 -DCMAKE_CXX_COMPILER=clang++-6.0 -DLY_UNITY_BUILD=TRUE -DLY_PARALLEL_LINK_JOBS=4",
|
||||
"CMAKE_LY_PROJECTS": "AutomatedTesting",
|
||||
"CMAKE_TARGET": "all"
|
||||
}
|
||||
},
|
||||
"profile": {
|
||||
"TAGS": [
|
||||
"nightly",
|
||||
"daily-pipeline-metrics",
|
||||
"weekly-build-metrics"
|
||||
],
|
||||
"COMMAND": "build_linux.sh",
|
||||
"PARAMETERS": {
|
||||
"CONFIGURATION": "profile",
|
||||
"OUTPUT_DIRECTORY": "build/linux",
|
||||
"CMAKE_OPTIONS": "-G 'Ninja Multi-Config' -DCMAKE_C_COMPILER=clang-6.0 -DCMAKE_CXX_COMPILER=clang++-6.0 -DLY_UNITY_BUILD=TRUE -DLY_PARALLEL_LINK_JOBS=4",
|
||||
"CMAKE_LY_PROJECTS": "AutomatedTesting",
|
||||
"CMAKE_TARGET": "all"
|
||||
}
|
||||
},
|
||||
"profile_nounity": {
|
||||
"TAGS": [
|
||||
"weekly-build-metrics"
|
||||
],
|
||||
"COMMAND": "build_linux.sh",
|
||||
"PARAMETERS": {
|
||||
"CONFIGURATION": "profile",
|
||||
"OUTPUT_DIRECTORY": "build/linux",
|
||||
"CMAKE_OPTIONS": "-G 'Ninja Multi-Config' -DCMAKE_C_COMPILER=clang-6.0 -DCMAKE_CXX_COMPILER=clang++-6.0 -DLY_UNITY_BUILD=FALSE -DLY_PARALLEL_LINK_JOBS=4",
|
||||
"CMAKE_LY_PROJECTS": "AutomatedTesting",
|
||||
"CMAKE_TARGET": "all"
|
||||
}
|
||||
},
|
||||
"test_profile": {
|
||||
"TAGS": [
|
||||
"daily-pipeline-metrics",
|
||||
"weekly-build-metrics"
|
||||
],
|
||||
"COMMAND": "build_test_linux.sh",
|
||||
"PARAMETERS": {
|
||||
"CONFIGURATION": "profile",
|
||||
"OUTPUT_DIRECTORY": "build/linux",
|
||||
"CMAKE_OPTIONS": "-G 'Ninja Multi-Config' -DCMAKE_C_COMPILER=clang-6.0 -DCMAKE_CXX_COMPILER=clang++-6.0 -DLY_UNITY_BUILD=TRUE -DLY_PARALLEL_LINK_JOBS=4",
|
||||
"CMAKE_LY_PROJECTS": "AutomatedTesting",
|
||||
"CMAKE_TARGET": "all",
|
||||
"CTEST_OPTIONS": "-E (Gem::EMotionFX.Editor.Tests|Gem::AWSClientAuth.Tests|Gem::AWSCore.Editor.Tests) -L FRAMEWORK_googletest"
|
||||
}
|
||||
},
|
||||
"asset_profile": {
|
||||
"TAGS": [
|
||||
"weekly-build-metrics"
|
||||
],
|
||||
"COMMAND": "build_asset_linux.sh",
|
||||
"PARAMETERS": {
|
||||
"CONFIGURATION": "profile",
|
||||
"OUTPUT_DIRECTORY": "build/linux",
|
||||
"CMAKE_OPTIONS": "-G 'Ninja Multi-Config' -DCMAKE_C_COMPILER=clang-6.0 -DCMAKE_CXX_COMPILER=clang++-6.0 -DLY_UNITY_BUILD=TRUE -DLY_PARALLEL_LINK_JOBS=4",
|
||||
"CMAKE_LY_PROJECTS": "AutomatedTesting",
|
||||
"CMAKE_TARGET": "AssetProcessorBatch",
|
||||
"ASSET_PROCESSOR_BINARY": "bin/profile/AssetProcessorBatch",
|
||||
"ASSET_PROCESSOR_OPTIONS": "/zeroAnalysisMode",
|
||||
"ASSET_PROCESSOR_PLATFORMS": "pc,server"
|
||||
}
|
||||
},
|
||||
"asset_clean_profile": {
|
||||
"TAGS": [
|
||||
"nightly"
|
||||
],
|
||||
"PIPELINE_ENV": {
|
||||
"CLEAN_ASSETS": "1"
|
||||
},
|
||||
"steps": [
|
||||
"clean",
|
||||
"asset_profile"
|
||||
]
|
||||
},
|
||||
"periodic_test_profile": {
|
||||
"TAGS": [
|
||||
"nightly",
|
||||
"weekly-build-metrics"
|
||||
],
|
||||
"COMMAND": "build_test_linux.sh",
|
||||
"PARAMETERS": {
|
||||
"CONFIGURATION": "profile",
|
||||
"OUTPUT_DIRECTORY": "build/linux",
|
||||
"CMAKE_OPTIONS": "-G 'Ninja Multi-Config' -DCMAKE_C_COMPILER=clang-6.0 -DCMAKE_CXX_COMPILER=clang++-6.0 -DLY_UNITY_BUILD=TRUE -DLY_PARALLEL_LINK_JOBS=4",
|
||||
"CMAKE_LY_PROJECTS": "AutomatedTesting",
|
||||
"CMAKE_TARGET": "TEST_SUITE_periodic",
|
||||
"CTEST_OPTIONS": "-L \"(SUITE_periodic)\""
|
||||
}
|
||||
},
|
||||
"benchmark_test_profile": {
|
||||
"TAGS": [
|
||||
"nightly",
|
||||
"weekly-build-metrics"
|
||||
],
|
||||
"COMMAND": "build_test_linux.sh",
|
||||
"PARAMETERS": {
|
||||
"CONFIGURATION": "profile",
|
||||
"OUTPUT_DIRECTORY": "build/linux",
|
||||
"CMAKE_OPTIONS": "-G 'Ninja Multi-Config' -DCMAKE_C_COMPILER=clang-6.0 -DCMAKE_CXX_COMPILER=clang++-6.0 -DLY_UNITY_BUILD=TRUE -DLY_PARALLEL_LINK_JOBS=4",
|
||||
"CMAKE_LY_PROJECTS": "AutomatedTesting",
|
||||
"CMAKE_TARGET": "TEST_SUITE_benchmark",
|
||||
"CTEST_OPTIONS": "-L \"(SUITE_benchmark)\""
|
||||
}
|
||||
},
|
||||
"release": {
|
||||
"TAGS": [
|
||||
"nightly",
|
||||
"weekly-build-metrics"
|
||||
],
|
||||
"COMMAND": "build_linux.sh",
|
||||
"PARAMETERS": {
|
||||
"CONFIGURATION": "release",
|
||||
"OUTPUT_DIRECTORY": "build/linux",
|
||||
"CMAKE_OPTIONS": "-G 'Ninja Multi-Config' -DCMAKE_C_COMPILER=clang-6.0 -DCMAKE_CXX_COMPILER=clang++-6.0 -DLY_UNITY_BUILD=TRUE -DLY_PARALLEL_LINK_JOBS=4",
|
||||
"CMAKE_LY_PROJECTS": "AutomatedTesting",
|
||||
"CMAKE_TARGET": "all"
|
||||
}
|
||||
},
|
||||
"monolithic_release": {
|
||||
"TAGS": [
|
||||
"nightly",
|
||||
"weekly-build-metrics"
|
||||
],
|
||||
"COMMAND": "build_linux.sh",
|
||||
"PARAMETERS": {
|
||||
"CONFIGURATION": "release",
|
||||
"OUTPUT_DIRECTORY": "build/mono_linux",
|
||||
"CMAKE_OPTIONS": "-G 'Ninja Multi-Config' -DCMAKE_C_COMPILER=clang-6.0 -DCMAKE_CXX_COMPILER=clang++-6.0 -DLY_MONOLITHIC_GAME=TRUE -DLY_UNITY_BUILD=TRUE -DLY_PARALLEL_LINK_JOBS=4",
|
||||
"CMAKE_LY_PROJECTS": "AutomatedTesting",
|
||||
"CMAKE_TARGET": "all"
|
||||
}
|
||||
}
|
||||
}
|
||||
Executable
+49
@@ -0,0 +1,49 @@
|
||||
#!/usr/bin/env bash
|
||||
#
|
||||
# All or portions of this file Copyright (c) Amazon.com, Inc. or its affiliates or
|
||||
# its licensors.
|
||||
#
|
||||
# For complete copyright and license terms please see the LICENSE at the root of this
|
||||
# distribution (the "License"). All use of this software is governed by the License,
|
||||
# or, if provided, by the license below or the license accompanying this file. Do not
|
||||
# remove or modify any license notices. This file is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
#
|
||||
|
||||
set -o errexit # exit on the first failure encountered
|
||||
|
||||
BASEDIR=$(dirname "$0")
|
||||
source $BASEDIR/env_linux.sh
|
||||
|
||||
mkdir -p ${OUTPUT_DIRECTORY}
|
||||
SOURCE_DIRECTORY=${PWD}
|
||||
pushd $OUTPUT_DIRECTORY
|
||||
|
||||
LAST_CONFIGURE_CMD_FILE=ci_last_configure_cmd.txt
|
||||
CONFIGURE_CMD="cmake ${SOURCE_DIRECTORY} ${CMAKE_OPTIONS} ${EXTRA_CMAKE_OPTIONS} -DLY_3RDPARTY_PATH=${LY_3RDPARTY_PATH} -DLY_PROJECTS='${CMAKE_LY_PROJECTS}'"
|
||||
if [[ ! -e "CMakeCache.txt" ]]; then
|
||||
echo [ci_build] First run, generating
|
||||
RUN_CONFIGURE=1
|
||||
elif [[ ! -e ${LAST_CONFIGURE_CMD_FILE} ]]; then
|
||||
echo [ci_build] Last run command not found, generating
|
||||
RUN_CONFIGURE=1
|
||||
else
|
||||
# Detect if the input has changed
|
||||
LAST_CMD=$(<${LAST_CONFIGURE_CMD_FILE})
|
||||
if [[ "${LAST_CMD}" != "${CONFIGURE_CMD}" ]]; then
|
||||
echo [ci_build] Last run command different, generating
|
||||
RUN_CONFIGURE=1
|
||||
fi
|
||||
fi
|
||||
if [[ ! -z "$RUN_CONFIGURE" ]]; then
|
||||
# have to use eval since $CMAKE_OPTIONS (${EXTRA_CMAKE_OPTIONS}) contains quotes that need to be processed
|
||||
echo [ci_build] ${CONFIGURE_CMD}
|
||||
eval ${CONFIGURE_CMD}
|
||||
# Save the run only if success
|
||||
echo "${CONFIGURE_CMD}" > ${LAST_CONFIGURE_CMD_FILE}
|
||||
fi
|
||||
|
||||
echo [ci_build] cmake --build . --target ${CMAKE_TARGET} --config ${CONFIGURATION} -j $(grep -c processor /proc/cpuinfo) -- ${CMAKE_NATIVE_BUILD_ARGS}
|
||||
cmake --build . --target ${CMAKE_TARGET} --config ${CONFIGURATION} -j $(grep -c processor /proc/cpuinfo) -- ${CMAKE_NATIVE_BUILD_ARGS}
|
||||
|
||||
popd
|
||||
+18
@@ -0,0 +1,18 @@
|
||||
#!/usr/bin/env bash
|
||||
#
|
||||
# All or portions of this file Copyright (c) Amazon.com, Inc. or its affiliates or
|
||||
# its licensors.
|
||||
#
|
||||
# For complete copyright and license terms please see the LICENSE at the root of this
|
||||
# distribution (the "License"). All use of this software is governed by the License,
|
||||
# or, if provided, by the license below or the license accompanying this file. Do not
|
||||
# remove or modify any license notices. This file is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
#
|
||||
|
||||
set -o errexit # exit on the first failure encountered
|
||||
|
||||
BASEDIR=$(dirname "$0")
|
||||
source $BASEDIR/build_linux.sh
|
||||
|
||||
source $BASEDIR/test_linux.sh
|
||||
Executable
+32
@@ -0,0 +1,32 @@
|
||||
#!/usr/bin/env bash
|
||||
#
|
||||
# All or portions of this file Copyright (c) Amazon.com, Inc. or its affiliates or
|
||||
# its licensors.
|
||||
#
|
||||
# For complete copyright and license terms please see the LICENSE at the root of this
|
||||
# distribution (the "License"). All use of this software is governed by the License,
|
||||
# or, if provided, by the license below or the license accompanying this file. Do not
|
||||
# remove or modify any license notices. This file is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
#
|
||||
|
||||
set -o errexit # exit on the first failure encountered
|
||||
|
||||
if [[ -n "$CLEAN_ASSETS" ]]; then
|
||||
echo "[ci_build] CLEAN_ASSETS option set"
|
||||
for project in $(echo $CMAKE_LY_PROJECTS | sed "s/;/ /g")
|
||||
do
|
||||
if [[ -d "$project/Cache" ]]; then
|
||||
echo "[ci_build] Deleting \"$project/Cache\""
|
||||
rm -rf $project/Cache
|
||||
fi
|
||||
done
|
||||
fi
|
||||
|
||||
if [[ -n "$CLEAN_OUTPUT_DIRECTORY" ]]; then
|
||||
echo "[ci_build] CLEAN_OUTPUT_DIRECTORY option set"
|
||||
if [[ -d $OUTPUT_DIRECTORY ]]; then
|
||||
echo "[ci_build] Deleting \"${OUTPUT_DIRECTORY}\""
|
||||
rm -rf ${OUTPUT_DIRECTORY}
|
||||
fi
|
||||
fi
|
||||
Executable
+39
@@ -0,0 +1,39 @@
|
||||
#!/usr/bin/env bash
|
||||
#
|
||||
# All or portions of this file Copyright (c) Amazon.com, Inc. or its affiliates or
|
||||
# its licensors.
|
||||
#
|
||||
# For complete copyright and license terms please see the LICENSE at the root of this
|
||||
# distribution (the "License"). All use of this software is governed by the License,
|
||||
# or, if provided, by the license below or the license accompanying this file. Do not
|
||||
# remove or modify any license notices. This file is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
#
|
||||
|
||||
set -o errexit # exit on the first failure encountered
|
||||
|
||||
if ! command -v cmake &> /dev/null; then
|
||||
if [[ -z $LY_CMAKE_PATH ]]; then LY_CMAKE_PATH=${LY_3RDPARTY_PATH}/CMake/3.19.1/Linux/bin; fi
|
||||
if [[ ! -d $LY_CMAKE_PATH ]]; then
|
||||
echo "[ci_build] CMake path not found"
|
||||
exit 1
|
||||
fi
|
||||
PATH=${LY_CMAKE_PATH}:${PATH}
|
||||
if ! command -v cmake &> /dev/null; then
|
||||
echo "[ci_build] CMake not found"
|
||||
exit 1
|
||||
fi
|
||||
fi
|
||||
|
||||
if ! command -v ninja &> /dev/null; then
|
||||
if [[ -z $LY_NINJA_PATH ]]; then LY_NINJA_PATH=${LY_3RDPARTY_PATH}/ninja/1.10.1/Linux; fi
|
||||
if [[ ! -d $LY_NINJA_PATH ]]; then
|
||||
echo "[ci_build] Ninja path not found"
|
||||
exit 1
|
||||
fi
|
||||
PATH=${LY_NINJA_PATH}:${PATH}
|
||||
if ! command -v ninja &> /dev/null; then
|
||||
echo "[ci_build] Ninja not found"
|
||||
exit 1
|
||||
fi
|
||||
fi
|
||||
@@ -0,0 +1,17 @@
|
||||
{
|
||||
"ENV": {
|
||||
"NODE_LABEL": "linux",
|
||||
"LY_3RDPARTY_PATH": "/home/lybuilder/ly/workspace/3rdParty",
|
||||
"TIMEOUT": 30,
|
||||
"WORKSPACE": "/data/workspace",
|
||||
"MOUNT_VOLUME": true
|
||||
},
|
||||
"PIPELINE_ENV_OVERRIDE": {
|
||||
"daily-pipeline-metrics": {
|
||||
"CLEAN_WORKSPACE": true
|
||||
},
|
||||
"packaging": {
|
||||
"CLEAN_WORKSPACE": true
|
||||
}
|
||||
}
|
||||
}
|
||||
Executable
+16
@@ -0,0 +1,16 @@
|
||||
#!/usr/bin/env bash
|
||||
#
|
||||
# All or portions of this file Copyright (c) Amazon.com, Inc. or its affiliates or
|
||||
# its licensors.
|
||||
#
|
||||
# For complete copyright and license terms please see the LICENSE at the root of this
|
||||
# distribution (the "License"). All use of this software is governed by the License,
|
||||
# or, if provided, by the license below or the license accompanying this file. Do not
|
||||
# remove or modify any license notices. This file is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
#
|
||||
|
||||
set -o errexit # exit on the first failure encountered
|
||||
|
||||
echo [ci_build] python/python.sh -u ${SCRIPT_PATH} ${SCRIPT_PARAMETERS}
|
||||
python/python.sh -u ${SCRIPT_PATH} ${SCRIPT_PARAMETERS}
|
||||
Executable
+32
@@ -0,0 +1,32 @@
|
||||
#!/usr/bin/env bash
|
||||
#
|
||||
# All or portions of this file Copyright (c) Amazon.com, Inc. or its affiliates or
|
||||
# its licensors.
|
||||
#
|
||||
# For complete copyright and license terms please see the LICENSE at the root of this
|
||||
# distribution (the "License"). All use of this software is governed by the License,
|
||||
# or, if provided, by the license below or the license accompanying this file. Do not
|
||||
# remove or modify any license notices. This file is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
#
|
||||
|
||||
set -o errexit # exit on the first failure encountered
|
||||
|
||||
BASEDIR=$(dirname "$0")
|
||||
source $BASEDIR/env_linux.sh
|
||||
|
||||
if [[ ! -d $OUTPUT_DIRECTORY ]]; then
|
||||
echo [ci_build] Error: $OUTPUT_DIRECTORY was not found
|
||||
exit 1
|
||||
fi
|
||||
pushd $OUTPUT_DIRECTORY
|
||||
|
||||
# Find the CTEST_RUN_FLAGS from cmake's cache variables, then replace the $<CONFIG> with the current configuration
|
||||
IFS='=' read -ra CTEST_RUN_FLAGS <<< $(cmake -N -LA . | grep "CTEST_RUN_FLAGS:STRING")
|
||||
CTEST_RUN_FLAGS=${CTEST_RUN_FLAGS[1]/$<CONFIG>/${CONFIGURATION}}
|
||||
|
||||
# Run ctest
|
||||
echo [ci_build] ctest ${CTEST_RUN_FLAGS} ${CTEST_OPTIONS}
|
||||
ctest ${CTEST_RUN_FLAGS} ${CTEST_OPTIONS}
|
||||
|
||||
popd
|
||||
Executable
+32
@@ -0,0 +1,32 @@
|
||||
#!/usr/bin/env bash
|
||||
#
|
||||
# All or portions of this file Copyright (c) Amazon.com, Inc. or its affiliates or
|
||||
# its licensors.
|
||||
#
|
||||
# For complete copyright and license terms please see the LICENSE at the root of this
|
||||
# distribution (the "License"). All use of this software is governed by the License,
|
||||
# or, if provided, by the license below or the license accompanying this file. Do not
|
||||
# remove or modify any license notices. This file is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
#
|
||||
|
||||
set -o errexit # exit on the first failure encountered
|
||||
|
||||
if [[ ! -d $OUTPUT_DIRECTORY ]]; then
|
||||
echo [ci_build] Error: $OUTPUT_DIRECTORY was not found
|
||||
exit 1
|
||||
fi
|
||||
pushd $OUTPUT_DIRECTORY
|
||||
|
||||
if [[ ! -e $ASSET_PROCESSOR_BINARY ]]; then
|
||||
echo [ci_build] Error: $ASSET_PROCESSOR_BINARY was not found
|
||||
exit 1
|
||||
fi
|
||||
|
||||
for project in $(echo $CMAKE_LY_PROJECTS | sed "s/;/ /g")
|
||||
do
|
||||
echo [ci_build] ${ASSET_PROCESSOR_BINARY} $ASSET_PROCESSOR_OPTIONS --project-path=$project --platforms=$ASSET_PROCESSOR_PLATFORMS
|
||||
${ASSET_PROCESSOR_BINARY} $ASSET_PROCESSOR_OPTIONS --project-path=$project --platforms=$ASSET_PROCESSOR_PLATFORMS
|
||||
done
|
||||
|
||||
popd
|
||||
+18
@@ -0,0 +1,18 @@
|
||||
#!/usr/bin/env bash
|
||||
#
|
||||
# All or portions of this file Copyright (c) Amazon.com, Inc. or its affiliates or
|
||||
# its licensors.
|
||||
#
|
||||
# For complete copyright and license terms please see the LICENSE at the root of this
|
||||
# distribution (the "License"). All use of this software is governed by the License,
|
||||
# or, if provided, by the license below or the license accompanying this file. Do not
|
||||
# remove or modify any license notices. This file is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
#
|
||||
|
||||
set -o errexit # exit on the first failure encountered
|
||||
|
||||
BASEDIR=$(dirname "$0")
|
||||
source $BASEDIR/build_mac.sh
|
||||
|
||||
source $BASEDIR/asset_mac.sh
|
||||
@@ -0,0 +1,167 @@
|
||||
{
|
||||
"clean": {
|
||||
"TAGS": [],
|
||||
"COMMAND": "clean_mac.sh",
|
||||
"PARAMETERS": {
|
||||
"OUTPUT_DIRECTORY": "build",
|
||||
"CMAKE_LY_PROJECTS": "AutomatedTesting"
|
||||
}
|
||||
},
|
||||
"profile_pipe": {
|
||||
"TAGS": [
|
||||
"nightly"
|
||||
],
|
||||
"steps": [
|
||||
"profile",
|
||||
"asset_profile"
|
||||
]
|
||||
},
|
||||
"metrics": {
|
||||
"TAGS": [
|
||||
"weekly"
|
||||
],
|
||||
"COMMAND": "python_mac.sh",
|
||||
"PARAMETERS": {
|
||||
"SCRIPT_PATH": "scripts/build/ci_build_metrics.py",
|
||||
"SCRIPT_PARAMETERS": "--platform Mac --jobname '${JOB_NAME}' --jobnumber '${BUILD_NUMBER}' --jobnode '${NODE_NAME}' --changelist '${CHANGE_ID}'"
|
||||
}
|
||||
},
|
||||
"debug": {
|
||||
"TAGS": [
|
||||
"nightly",
|
||||
"weekly-build-metrics"
|
||||
],
|
||||
"COMMAND": "build_mac.sh",
|
||||
"PARAMETERS": {
|
||||
"CONFIGURATION": "debug",
|
||||
"OUTPUT_DIRECTORY": "build/mac",
|
||||
"CMAKE_OPTIONS": "-G Xcode -DLY_UNITY_BUILD=TRUE",
|
||||
"CMAKE_LY_PROJECTS": "AutomatedTesting",
|
||||
"CMAKE_TARGET": "ALL_BUILD"
|
||||
}
|
||||
},
|
||||
"profile": {
|
||||
"TAGS": [
|
||||
"daily-pipeline-metrics",
|
||||
"weekly-build-metrics"
|
||||
],
|
||||
"COMMAND": "build_mac.sh",
|
||||
"PARAMETERS": {
|
||||
"CONFIGURATION": "profile",
|
||||
"OUTPUT_DIRECTORY": "build/mac",
|
||||
"CMAKE_OPTIONS": "-G Xcode -DLY_UNITY_BUILD=TRUE",
|
||||
"CMAKE_LY_PROJECTS": "AutomatedTesting",
|
||||
"CMAKE_TARGET": "ALL_BUILD"
|
||||
}
|
||||
},
|
||||
"profile_nounity": {
|
||||
"TAGS": [
|
||||
"nightly",
|
||||
"weekly-build-metrics"
|
||||
],
|
||||
"COMMAND": "build_mac.sh",
|
||||
"PARAMETERS": {
|
||||
"CONFIGURATION": "profile",
|
||||
"OUTPUT_DIRECTORY": "build/mac",
|
||||
"CMAKE_OPTIONS": "-G Xcode -DLY_UNITY_BUILD=FALSE",
|
||||
"CMAKE_LY_PROJECTS": "AutomatedTesting",
|
||||
"CMAKE_TARGET": "ALL_BUILD"
|
||||
}
|
||||
},
|
||||
"asset_profile": {
|
||||
"TAGS": [
|
||||
"weekly-build-metrics"
|
||||
],
|
||||
"COMMAND": "build_asset_mac.sh",
|
||||
"PARAMETERS": {
|
||||
"CONFIGURATION": "profile",
|
||||
"OUTPUT_DIRECTORY": "build/mac",
|
||||
"CMAKE_OPTIONS": "-G Xcode -DLY_UNITY_BUILD=TRUE",
|
||||
"CMAKE_LY_PROJECTS": "AutomatedTesting",
|
||||
"CMAKE_TARGET": "AssetProcessorBatch",
|
||||
"ASSET_PROCESSOR_BINARY": "bin/profile/AssetProcessorBatch",
|
||||
"ASSET_PROCESSOR_OPTIONS": "/zeroAnalysisMode",
|
||||
"ASSET_PROCESSOR_PLATFORMS": "osx_gl"
|
||||
}
|
||||
},
|
||||
"asset_clean_profile": {
|
||||
"TAGS": [
|
||||
"nightly"
|
||||
],
|
||||
"PIPELINE_ENV": {
|
||||
"CLEAN_ASSETS": "1"
|
||||
},
|
||||
"steps": [
|
||||
"clean",
|
||||
"asset_profile"
|
||||
]
|
||||
},
|
||||
"periodic_test_profile": {
|
||||
"TAGS": [
|
||||
"nightly",
|
||||
"weekly-build-metrics"
|
||||
],
|
||||
"COMMAND": "build_test_mac.sh",
|
||||
"PARAMETERS": {
|
||||
"CONFIGURATION": "profile",
|
||||
"OUTPUT_DIRECTORY": "build/mac",
|
||||
"CMAKE_OPTIONS": "-G Xcode -DLY_UNITY_BUILD=TRUE",
|
||||
"CMAKE_LY_PROJECTS": "AutomatedTesting",
|
||||
"CMAKE_TARGET": "TEST_SUITE_periodic",
|
||||
"CTEST_OPTIONS": "-L \"(SUITE_periodic)\""
|
||||
}
|
||||
},
|
||||
"benchmark_test_profile": {
|
||||
"TAGS": [
|
||||
"nightly",
|
||||
"weekly-build-metrics"
|
||||
],
|
||||
"COMMAND": "build_test_mac.sh",
|
||||
"PARAMETERS": {
|
||||
"CONFIGURATION": "profile",
|
||||
"OUTPUT_DIRECTORY": "build/mac",
|
||||
"CMAKE_OPTIONS": "-G Xcode -DLY_UNITY_BUILD=TRUE",
|
||||
"CMAKE_LY_PROJECTS": "AutomatedTesting",
|
||||
"CMAKE_TARGET": "TEST_SUITE_benchmark",
|
||||
"CTEST_OPTIONS": "-L \"(SUITE_benchmark)\""
|
||||
}
|
||||
},
|
||||
"release": {
|
||||
"TAGS": [
|
||||
"nightly",
|
||||
"weekly-build-metrics"
|
||||
],
|
||||
"COMMAND": "build_mac.sh",
|
||||
"PARAMETERS": {
|
||||
"CONFIGURATION": "release",
|
||||
"OUTPUT_DIRECTORY": "build/mac",
|
||||
"CMAKE_OPTIONS": "-G Xcode -DLY_UNITY_BUILD=TRUE",
|
||||
"CMAKE_LY_PROJECTS": "AutomatedTesting",
|
||||
"CMAKE_TARGET": "ALL_BUILD"
|
||||
}
|
||||
},
|
||||
"monolithic_release": {
|
||||
"TAGS": [
|
||||
"nightly",
|
||||
"weekly-build-metrics"
|
||||
],
|
||||
"COMMAND": "build_mac.sh",
|
||||
"PARAMETERS": {
|
||||
"CONFIGURATION": "release",
|
||||
"OUTPUT_DIRECTORY": "build/mono_mac",
|
||||
"CMAKE_OPTIONS": "-G Xcode -DLY_MONOLITHIC_GAME=TRUE -DLY_UNITY_BUILD=TRUE",
|
||||
"CMAKE_LY_PROJECTS": "AutomatedTesting",
|
||||
"CMAKE_TARGET": "ALL_BUILD"
|
||||
}
|
||||
},
|
||||
"mac_packaging_all": {
|
||||
"TAGS": [
|
||||
"packaging"
|
||||
],
|
||||
"COMMAND": "python_mac.sh",
|
||||
"PARAMETERS": {
|
||||
"SCRIPT_PATH": "scripts/build/package/package.py",
|
||||
"SCRIPT_PARAMETERS": "--platform Mac --type all"
|
||||
}
|
||||
}
|
||||
}
|
||||
Executable
+49
@@ -0,0 +1,49 @@
|
||||
#!/usr/bin/env bash
|
||||
#
|
||||
# All or portions of this file Copyright (c) Amazon.com, Inc. or its affiliates or
|
||||
# its licensors.
|
||||
#
|
||||
# For complete copyright and license terms please see the LICENSE at the root of this
|
||||
# distribution (the "License"). All use of this software is governed by the License,
|
||||
# or, if provided, by the license below or the license accompanying this file. Do not
|
||||
# remove or modify any license notices. This file is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
#
|
||||
|
||||
set -o errexit # exit on the first failure encountered
|
||||
|
||||
BASEDIR=$(dirname "$0")
|
||||
source $BASEDIR/env_mac.sh
|
||||
|
||||
mkdir -p ${OUTPUT_DIRECTORY}
|
||||
SOURCE_DIRECTORY=${PWD}
|
||||
pushd $OUTPUT_DIRECTORY
|
||||
|
||||
LAST_CONFIGURE_CMD_FILE=ci_last_configure_cmd.txt
|
||||
CONFIGURE_CMD="cmake ${SOURCE_DIRECTORY} ${CMAKE_OPTIONS} ${EXTRA_CMAKE_OPTIONS} -DLY_3RDPARTY_PATH=${LY_3RDPARTY_PATH} -DLY_PROJECTS='${CMAKE_LY_PROJECTS}'"
|
||||
if [[ ! -e "CMakeCache.txt" ]]; then
|
||||
echo [ci_build] First run, generating
|
||||
RUN_CONFIGURE=1
|
||||
elif [[ ! -e ${LAST_CONFIGURE_CMD_FILE} ]]; then
|
||||
echo [ci_build] Last run command not found, generating
|
||||
RUN_CONFIGURE=1
|
||||
else
|
||||
# Detect if the input has changed
|
||||
LAST_CMD=$(<${LAST_CONFIGURE_CMD_FILE})
|
||||
if [[ "${LAST_CMD}" != "${CONFIGURE_CMD}" ]]; then
|
||||
echo [ci_build] Last run command different, generating
|
||||
RUN_CONFIGURE=1
|
||||
fi
|
||||
fi
|
||||
if [[ ! -z "$RUN_CONFIGURE" ]]; then
|
||||
# have to use eval since $CMAKE_OPTIONS (${EXTRA_CMAKE_OPTIONS}) contains quotes that need to be processed
|
||||
echo [ci_build] ${CONFIGURE_CMD}
|
||||
eval ${CONFIGURE_CMD}
|
||||
# Save the run only if success
|
||||
echo "${CONFIGURE_CMD}" > ${LAST_CONFIGURE_CMD_FILE}
|
||||
fi
|
||||
|
||||
echo [ci_build] cmake --build . --target ${CMAKE_TARGET} --config ${CONFIGURATION} -j $(sysctl -n hw.ncpu) -- ${CMAKE_NATIVE_BUILD_ARGS}
|
||||
cmake --build . --target ${CMAKE_TARGET} --config ${CONFIGURATION} -j $(sysctl -n hw.ncpu) -- ${CMAKE_NATIVE_BUILD_ARGS}
|
||||
|
||||
popd
|
||||
Executable
+18
@@ -0,0 +1,18 @@
|
||||
#!/usr/bin/env bash
|
||||
#
|
||||
# All or portions of this file Copyright (c) Amazon.com, Inc. or its affiliates or
|
||||
# its licensors.
|
||||
#
|
||||
# For complete copyright and license terms please see the LICENSE at the root of this
|
||||
# distribution (the "License"). All use of this software is governed by the License,
|
||||
# or, if provided, by the license below or the license accompanying this file. Do not
|
||||
# remove or modify any license notices. This file is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
#
|
||||
|
||||
set -o errexit # exit on the first failure encountered
|
||||
|
||||
BASEDIR=$(dirname "$0")
|
||||
source $BASEDIR/build_mac.sh
|
||||
|
||||
source $BASEDIR/test_mac.sh
|
||||
Executable
+32
@@ -0,0 +1,32 @@
|
||||
#!/usr/bin/env bash
|
||||
#
|
||||
# All or portions of this file Copyright (c) Amazon.com, Inc. or its affiliates or
|
||||
# its licensors.
|
||||
#
|
||||
# For complete copyright and license terms please see the LICENSE at the root of this
|
||||
# distribution (the "License"). All use of this software is governed by the License,
|
||||
# or, if provided, by the license below or the license accompanying this file. Do not
|
||||
# remove or modify any license notices. This file is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
#
|
||||
|
||||
set -o errexit # exit on the first failure encountered
|
||||
|
||||
if [[ -n "$CLEAN_ASSETS" ]]; then
|
||||
echo "[ci_build] CLEAN_ASSETS option set"
|
||||
for project in $(echo $CMAKE_LY_PROJECTS | sed "s/;/ /g")
|
||||
do
|
||||
if [[ -d "$project/Cache" ]]; then
|
||||
echo "[ci_build] Deleting \"$project/Cache\""
|
||||
rm -rf $project/Cache
|
||||
fi
|
||||
done
|
||||
fi
|
||||
|
||||
if [[ -n "$CLEAN_OUTPUT_DIRECTORY" ]]; then
|
||||
echo "[ci_build] CLEAN_OUTPUT_DIRECTORY option set"
|
||||
if [[ -d $OUTPUT_DIRECTORY ]]; then
|
||||
echo "[ci_build] Deleting \"${OUTPUT_DIRECTORY}\""
|
||||
rm -rf ${OUTPUT_DIRECTORY}
|
||||
fi
|
||||
fi
|
||||
Executable
+26
@@ -0,0 +1,26 @@
|
||||
#!/usr/bin/env bash
|
||||
#
|
||||
# All or portions of this file Copyright (c) Amazon.com, Inc. or its affiliates or
|
||||
# its licensors.
|
||||
#
|
||||
# For complete copyright and license terms please see the LICENSE at the root of this
|
||||
# distribution (the "License"). All use of this software is governed by the License,
|
||||
# or, if provided, by the license below or the license accompanying this file. Do not
|
||||
# remove or modify any license notices. This file is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
#
|
||||
|
||||
set -o errexit # exit on the first failure encountered
|
||||
|
||||
if ! command -v cmake &> /dev/null; then
|
||||
if [[ -z $LY_CMAKE_PATH ]]; then LY_CMAKE_PATH=${LY_3RDPARTY_PATH}/CMake/3.19.1/Mac/CMake.app/Contents/bin; fi
|
||||
if [[ ! -d $LY_CMAKE_PATH ]]; then
|
||||
echo "[ci_build] CMake path not found"
|
||||
exit 1
|
||||
fi
|
||||
PATH=${LY_CMAKE_PATH}:${PATH}
|
||||
if ! command -v cmake &> /dev/null; then
|
||||
echo "[ci_build] CMake not found"
|
||||
exit 1
|
||||
fi
|
||||
fi
|
||||
@@ -0,0 +1,17 @@
|
||||
{
|
||||
"ENV": {
|
||||
"NODE_LABEL": "mac",
|
||||
"LY_3RDPARTY_PATH": "/Users/lybuilder/3rdParty",
|
||||
"TIMEOUT": 30,
|
||||
"WORKSPACE": "/Users/lybuilder/workspace",
|
||||
"MOUNT_VOLUME": false
|
||||
},
|
||||
"PIPELINE_ENV_OVERRIDE": {
|
||||
"daily-pipeline-metrics": {
|
||||
"CLEAN_WORKSPACE": true
|
||||
},
|
||||
"packaging": {
|
||||
"CLEAN_WORKSPACE": true
|
||||
}
|
||||
}
|
||||
}
|
||||
Executable
+16
@@ -0,0 +1,16 @@
|
||||
#!/usr/bin/env bash
|
||||
#
|
||||
# All or portions of this file Copyright (c) Amazon.com, Inc. or its affiliates or
|
||||
# its licensors.
|
||||
#
|
||||
# For complete copyright and license terms please see the LICENSE at the root of this
|
||||
# distribution (the "License"). All use of this software is governed by the License,
|
||||
# or, if provided, by the license below or the license accompanying this file. Do not
|
||||
# remove or modify any license notices. This file is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
#
|
||||
|
||||
set -o errexit # exit on the first failure encountered
|
||||
|
||||
echo [ci_build] python/python.sh -u ${SCRIPT_PATH} ${SCRIPT_PARAMETERS}
|
||||
python/python.sh -u ${SCRIPT_PATH} ${SCRIPT_PARAMETERS}
|
||||
Executable
+32
@@ -0,0 +1,32 @@
|
||||
#!/usr/bin/env bash
|
||||
#
|
||||
# All or portions of this file Copyright (c) Amazon.com, Inc. or its affiliates or
|
||||
# its licensors.
|
||||
#
|
||||
# For complete copyright and license terms please see the LICENSE at the root of this
|
||||
# distribution (the "License"). All use of this software is governed by the License,
|
||||
# or, if provided, by the license below or the license accompanying this file. Do not
|
||||
# remove or modify any license notices. This file is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
#
|
||||
|
||||
set -o errexit # exit on the first failure encountered
|
||||
|
||||
BASEDIR=$(dirname "$0")
|
||||
source $BASEDIR/env_mac.sh
|
||||
|
||||
if [[ ! -d $OUTPUT_DIRECTORY ]]; then
|
||||
echo [ci_build] Error: $OUTPUT_DIRECTORY was not found
|
||||
exit 1
|
||||
fi
|
||||
pushd $OUTPUT_DIRECTORY
|
||||
|
||||
# Find the CTEST_RUN_FLAGS from the CMakeCache.txt file, then replace the $<CONFIG> with the current configuration
|
||||
IFS='=' read -ra CTEST_RUN_FLAGS <<< $(cmake -N -LA . | grep "CTEST_RUN_FLAGS:STRING")
|
||||
CTEST_RUN_FLAGS=${CTEST_RUN_FLAGS[1]/$<CONFIG>/${CONFIGURATION}}
|
||||
|
||||
# Run ctest
|
||||
echo [ci_build] ctest ${CTEST_RUN_FLAGS} ${CTEST_OPTIONS}
|
||||
ctest ${CTEST_RUN_FLAGS} ${CTEST_OPTIONS}
|
||||
|
||||
popd
|
||||
@@ -0,0 +1,39 @@
|
||||
@ECHO OFF
|
||||
REM
|
||||
REM All or portions of this file Copyright (c) Amazon.com, Inc. or its affiliates or
|
||||
REM its licensors.
|
||||
REM
|
||||
REM For complete copyright and license terms please see the LICENSE at the root of this
|
||||
REM distribution (the "License"). All use of this software is governed by the License,
|
||||
REM or, if provided, by the license below or the license accompanying this file. Do not
|
||||
REM remove or modify any license notices. This file is distributed on an "AS IS" BASIS,
|
||||
REM WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
REM
|
||||
|
||||
SETLOCAL EnableDelayedExpansion
|
||||
|
||||
IF NOT EXIST %OUTPUT_DIRECTORY% (
|
||||
ECHO [ci_build] Error: %OUTPUT_DIRECTORY% was not found
|
||||
GOTO :error
|
||||
)
|
||||
PUSHD %OUTPUT_DIRECTORY%
|
||||
|
||||
IF NOT EXIST %ASSET_PROCESSOR_BINARY% (
|
||||
ECHO [ci_build] Error: %ASSET_PROCESSOR_BINARY% was not found
|
||||
GOTO :error
|
||||
)
|
||||
|
||||
FOR %%P in (%CMAKE_LY_PROJECTS%) do (
|
||||
ECHO [ci_build] %ASSET_PROCESSOR_BINARY% %ASSET_PROCESSOR_OPTIONS% --project-path=%%P --platforms=%ASSET_PROCESSOR_PLATFORMS%
|
||||
%ASSET_PROCESSOR_BINARY% %ASSET_PROCESSOR_OPTIONS% --project-path=%%P --platforms=%ASSET_PROCESSOR_PLATFORMS%
|
||||
IF NOT !ERRORLEVEL!==0 GOTO :popd_error
|
||||
)
|
||||
|
||||
POPD
|
||||
EXIT /b 0
|
||||
|
||||
:popd_error
|
||||
POPD
|
||||
|
||||
:error
|
||||
EXIT /b 1
|
||||
@@ -0,0 +1,22 @@
|
||||
@ECHO OFF
|
||||
REM
|
||||
REM All or portions of this file Copyright (c) Amazon.com, Inc. or its affiliates or
|
||||
REM its licensors.
|
||||
REM
|
||||
REM For complete copyright and license terms please see the LICENSE at the root of this
|
||||
REM distribution (the "License"). All use of this software is governed by the License,
|
||||
REM or, if provided, by the license below or the license accompanying this file. Do not
|
||||
REM remove or modify any license notices. This file is distributed on an "AS IS" BASIS,
|
||||
REM WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
REM
|
||||
|
||||
CALL "%~dp0build_windows.cmd"
|
||||
IF NOT %ERRORLEVEL%==0 GOTO :error
|
||||
|
||||
CALL "%~dp0asset_windows.cmd"
|
||||
IF NOT %ERRORLEVEL%==0 GOTO :error
|
||||
|
||||
EXIT /b 0
|
||||
|
||||
:error
|
||||
EXIT /b 1
|
||||
@@ -0,0 +1,285 @@
|
||||
{
|
||||
"clean": {
|
||||
"TAGS": [],
|
||||
"COMMAND": "clean_windows.cmd",
|
||||
"PARAMETERS": {
|
||||
"OUTPUT_DIRECTORY": "build",
|
||||
"CMAKE_LY_PROJECTS": "AutomatedTesting"
|
||||
}
|
||||
},
|
||||
"validation_pipe": {
|
||||
"TAGS": [
|
||||
"default"
|
||||
],
|
||||
"steps": [
|
||||
"validation"
|
||||
]
|
||||
},
|
||||
"debug_vs2019_pipe": {
|
||||
"TAGS": [
|
||||
"nightly"
|
||||
],
|
||||
"steps": [
|
||||
"debug_vs2019",
|
||||
"test_debug_vs2019"
|
||||
]
|
||||
},
|
||||
"profile_vs2019_pipe": {
|
||||
"TAGS": [
|
||||
"default"
|
||||
],
|
||||
"steps": [
|
||||
"profile_vs2019",
|
||||
"asset_profile_vs2019",
|
||||
"test_cpu_profile_vs2019"
|
||||
]
|
||||
},
|
||||
"scrubbing": {
|
||||
"TAGS": [],
|
||||
"COMMAND": "python_windows.cmd",
|
||||
"PARAMETERS": {
|
||||
"SCRIPT_PATH": "scripts/build/scrubbing_job.py"
|
||||
}
|
||||
},
|
||||
"validation": {
|
||||
"TAGS": [],
|
||||
"COMMAND": "python_windows.cmd",
|
||||
"PARAMETERS": {
|
||||
"SCRIPT_PATH": "scripts/commit_validation/validate_file_or_folder.py"
|
||||
}
|
||||
},
|
||||
"metrics": {
|
||||
"TAGS": [
|
||||
"weekly"
|
||||
],
|
||||
"COMMAND": "python_windows.cmd",
|
||||
"PARAMETERS": {
|
||||
"SCRIPT_PATH": "scripts/build/ci_build_metrics.py",
|
||||
"SCRIPT_PARAMETERS": "--platform Windows --jobname \"!JOB_NAME!\" --jobnumber \"!BUILD_NUMBER!\" --jobnode \"!NODE_NAME!\" --changelist \"!CHANGE_ID!\""
|
||||
}
|
||||
},
|
||||
"windows_packaging_all": {
|
||||
"TAGS": [
|
||||
"packaging"
|
||||
],
|
||||
"COMMAND": "python_windows.cmd",
|
||||
"PARAMETERS": {
|
||||
"SCRIPT_PATH": "scripts/build/package/package.py",
|
||||
"SCRIPT_PARAMETERS": "--platform Windows --type all"
|
||||
}
|
||||
},
|
||||
"3rdParty_all": {
|
||||
"TAGS": [
|
||||
"packaging"
|
||||
],
|
||||
"COMMAND": "python_windows.cmd",
|
||||
"PARAMETERS": {
|
||||
"SCRIPT_PATH": "scripts/build/package/package.py",
|
||||
"SCRIPT_PARAMETERS": "--platform 3rdParty --type 3rdParty_all"
|
||||
}
|
||||
},
|
||||
"debug_vs2019": {
|
||||
"TAGS": [
|
||||
"weekly-build-metrics"
|
||||
],
|
||||
"COMMAND": "build_windows.cmd",
|
||||
"PARAMETERS": {
|
||||
"CONFIGURATION": "debug",
|
||||
"OUTPUT_DIRECTORY": "build\\windows_vs2019",
|
||||
"CMAKE_OPTIONS": "-G \"Visual Studio 16 2019\" -DCMAKE_SYSTEM_VERSION=10.0 -DLY_UNITY_BUILD=TRUE -DLY_BUILD_WITH_INCREMENTAL_LINKING_DEBUG=FALSE",
|
||||
"CMAKE_LY_PROJECTS": "AutomatedTesting",
|
||||
"CMAKE_TARGET": "ALL_BUILD",
|
||||
"CMAKE_NATIVE_BUILD_ARGS": "/m /nologo"
|
||||
}
|
||||
},
|
||||
"test_debug_vs2019": {
|
||||
"TAGS": [
|
||||
"weekly-build-metrics"
|
||||
],
|
||||
"COMMAND": "build_test_windows.cmd",
|
||||
"PARAMETERS": {
|
||||
"CONFIGURATION": "debug",
|
||||
"OUTPUT_DIRECTORY": "build\\windows_vs2019",
|
||||
"CMAKE_OPTIONS": "-G \"Visual Studio 16 2019\" -DCMAKE_SYSTEM_VERSION=10.0 -DLY_UNITY_BUILD=TRUE -DLY_BUILD_WITH_INCREMENTAL_LINKING_DEBUG=FALSE",
|
||||
"CMAKE_LY_PROJECTS": "AutomatedTesting",
|
||||
"CMAKE_TARGET": "TEST_SUITE_smoke TEST_SUITE_main",
|
||||
"CMAKE_NATIVE_BUILD_ARGS": "/m /nologo",
|
||||
"CTEST_OPTIONS": "-L \"(SUITE_smoke|SUITE_main)\" -LE \"(REQUIRES_gpu)\" -T Test"
|
||||
}
|
||||
},
|
||||
"profile_vs2019": {
|
||||
"TAGS": [
|
||||
"project",
|
||||
"daily-pipeline-metrics",
|
||||
"weekly-build-metrics"
|
||||
],
|
||||
"COMMAND": "build_windows.cmd",
|
||||
"PARAMETERS": {
|
||||
"CONFIGURATION": "profile",
|
||||
"OUTPUT_DIRECTORY": "build\\windows_vs2019",
|
||||
"CMAKE_OPTIONS": "-G \"Visual Studio 16 2019\" -DCMAKE_SYSTEM_VERSION=10.0 -DLY_UNITY_BUILD=TRUE",
|
||||
"CMAKE_LY_PROJECTS": "AutomatedTesting",
|
||||
"CMAKE_TARGET": "ALL_BUILD",
|
||||
"CMAKE_NATIVE_BUILD_ARGS": "/m /nologo"
|
||||
}
|
||||
},
|
||||
"profile_vs2019_nounity": {
|
||||
"TAGS": [
|
||||
"nightly",
|
||||
"weekly-build-metrics"
|
||||
],
|
||||
"COMMAND": "build_windows.cmd",
|
||||
"PARAMETERS": {
|
||||
"CONFIGURATION": "profile",
|
||||
"OUTPUT_DIRECTORY": "build\\windows_vs2019",
|
||||
"CMAKE_OPTIONS": "-G \"Visual Studio 16 2019\" -DCMAKE_SYSTEM_VERSION=10.0 -DLY_UNITY_BUILD=FALSE",
|
||||
"CMAKE_LY_PROJECTS": "AutomatedTesting",
|
||||
"CMAKE_TARGET": "ALL_BUILD",
|
||||
"CMAKE_NATIVE_BUILD_ARGS": "/m /nologo"
|
||||
}
|
||||
},
|
||||
"test_cpu_profile_vs2019": {
|
||||
"TAGS": [
|
||||
"daily-pipeline-metrics",
|
||||
"weekly-build-metrics"
|
||||
],
|
||||
"COMMAND": "build_test_windows.cmd",
|
||||
"PARAMETERS": {
|
||||
"CONFIGURATION": "profile",
|
||||
"OUTPUT_DIRECTORY": "build\\windows_vs2019",
|
||||
"CMAKE_OPTIONS": "-G \"Visual Studio 16 2019\" -DCMAKE_SYSTEM_VERSION=10.0 -DLY_UNITY_BUILD=TRUE",
|
||||
"CMAKE_LY_PROJECTS": "AutomatedTesting",
|
||||
"CMAKE_TARGET": "TEST_SUITE_smoke TEST_SUITE_main",
|
||||
"CMAKE_NATIVE_BUILD_ARGS": "/m /nologo",
|
||||
"CTEST_OPTIONS": "-L \"(SUITE_smoke|SUITE_main)\" -LE \"(REQUIRES_gpu)\" -T Test"
|
||||
}
|
||||
},
|
||||
"test_gpu_profile_vs2019": {
|
||||
"TAGS":[
|
||||
"nightly"
|
||||
],
|
||||
"PIPELINE_ENV":{
|
||||
"NODE_LABEL":"windows-gpu"
|
||||
},
|
||||
"COMMAND": "build_test_windows.cmd",
|
||||
"PARAMETERS": {
|
||||
"CONFIGURATION": "profile",
|
||||
"OUTPUT_DIRECTORY": "build\\windows_vs2019",
|
||||
"CMAKE_OPTIONS": "-G \"Visual Studio 16 2019\" -DCMAKE_SYSTEM_VERSION=10.0 -DLY_UNITY_BUILD=TRUE",
|
||||
"CMAKE_LY_PROJECTS": "AutomatedTesting",
|
||||
"CMAKE_TARGET": "TEST_SUITE_smoke TEST_SUITE_main",
|
||||
"CMAKE_NATIVE_BUILD_ARGS": "/m /nologo",
|
||||
"CTEST_OPTIONS": "-L \"(SUITE_smoke_REQUIRES_gpu|SUITE_main_REQUIRES_gpu)\" -T Test"
|
||||
}
|
||||
},
|
||||
"asset_profile_vs2019": {
|
||||
"TAGS": [
|
||||
"weekly-build-metrics"
|
||||
],
|
||||
"COMMAND": "build_asset_windows.cmd",
|
||||
"PARAMETERS": {
|
||||
"CONFIGURATION": "profile",
|
||||
"OUTPUT_DIRECTORY": "build\\windows_vs2019",
|
||||
"CMAKE_OPTIONS": "-G \"Visual Studio 16 2019\" -DCMAKE_SYSTEM_VERSION=10.0 -DLY_UNITY_BUILD=TRUE",
|
||||
"CMAKE_LY_PROJECTS": "AutomatedTesting",
|
||||
"CMAKE_TARGET": "AssetProcessorBatch",
|
||||
"CMAKE_NATIVE_BUILD_ARGS": "/m /nologo",
|
||||
"ASSET_PROCESSOR_BINARY": "bin\\profile\\AssetProcessorBatch.exe",
|
||||
"ASSET_PROCESSOR_OPTIONS": "/zeroAnalysisMode",
|
||||
"ASSET_PROCESSOR_PLATFORMS": "pc,server"
|
||||
}
|
||||
},
|
||||
"asset_clean_profile_vs2019": {
|
||||
"TAGS": [
|
||||
"nightly"
|
||||
],
|
||||
"PIPELINE_ENV": {
|
||||
"CLEAN_ASSETS": "1"
|
||||
},
|
||||
"steps": [
|
||||
"clean",
|
||||
"asset_profile_vs2019"
|
||||
]
|
||||
},
|
||||
"periodic_test_profile_vs2019": {
|
||||
"TAGS": [
|
||||
"nightly",
|
||||
"weekly-build-metrics"
|
||||
],
|
||||
"COMMAND": "build_test_windows.cmd",
|
||||
"PARAMETERS": {
|
||||
"CONFIGURATION": "profile",
|
||||
"OUTPUT_DIRECTORY": "build\\windows_vs2019",
|
||||
"CMAKE_OPTIONS": "-G \"Visual Studio 16 2019\" -DCMAKE_SYSTEM_VERSION=10.0 -DLY_UNITY_BUILD=TRUE",
|
||||
"CMAKE_LY_PROJECTS": "AutomatedTesting",
|
||||
"CMAKE_TARGET": "TEST_SUITE_periodic",
|
||||
"CMAKE_NATIVE_BUILD_ARGS": "/m /nologo",
|
||||
"CTEST_OPTIONS": "-L \"(SUITE_periodic)\" -T Test"
|
||||
}
|
||||
},
|
||||
"sandbox_test_profile_vs2019": {
|
||||
"TAGS": [
|
||||
"nightly",
|
||||
"weekly-build-metrics"
|
||||
],
|
||||
"PIPELINE_ENV": {
|
||||
"ON_FAILURE_MARK": "UNSTABLE"
|
||||
},
|
||||
"COMMAND": "build_test_windows.cmd",
|
||||
"PARAMETERS": {
|
||||
"CONFIGURATION": "profile",
|
||||
"OUTPUT_DIRECTORY": "build\\windows_vs2019",
|
||||
"CMAKE_OPTIONS": "-G \"Visual Studio 16 2019\" -DCMAKE_SYSTEM_VERSION=10.0 -DLY_UNITY_BUILD=TRUE",
|
||||
"CMAKE_LY_PROJECTS": "AutomatedTesting",
|
||||
"CMAKE_TARGET": "TEST_SUITE_sandbox",
|
||||
"CMAKE_NATIVE_BUILD_ARGS": "/m /nologo",
|
||||
"CTEST_OPTIONS": "-L \"(SUITE_sandbox)\" -T Test"
|
||||
}
|
||||
},
|
||||
"benchmark_test_profile_vs2019": {
|
||||
"TAGS": [
|
||||
"nightly",
|
||||
"weekly-build-metrics"
|
||||
],
|
||||
"COMMAND": "build_test_windows.cmd",
|
||||
"PARAMETERS": {
|
||||
"CONFIGURATION": "profile",
|
||||
"OUTPUT_DIRECTORY": "build\\windows_vs2019",
|
||||
"CMAKE_OPTIONS": "-G \"Visual Studio 16 2019\" -DCMAKE_SYSTEM_VERSION=10.0 -DLY_UNITY_BUILD=TRUE",
|
||||
"CMAKE_LY_PROJECTS": "AutomatedTesting",
|
||||
"CMAKE_TARGET": "TEST_SUITE_benchmark",
|
||||
"CMAKE_NATIVE_BUILD_ARGS": "/m /nologo",
|
||||
"CTEST_OPTIONS": "-L \"(SUITE_benchmark)\" -T Test"
|
||||
}
|
||||
},
|
||||
"release_vs2019": {
|
||||
"TAGS": [
|
||||
"nightly",
|
||||
"weekly-build-metrics"
|
||||
],
|
||||
"COMMAND": "build_windows.cmd",
|
||||
"PARAMETERS": {
|
||||
"CONFIGURATION": "release",
|
||||
"OUTPUT_DIRECTORY": "build\\windows_vs2019",
|
||||
"CMAKE_OPTIONS": "-G \"Visual Studio 16 2019\" -DCMAKE_SYSTEM_VERSION=10.0 -DLY_UNITY_BUILD=TRUE",
|
||||
"CMAKE_LY_PROJECTS": "AutomatedTesting",
|
||||
"CMAKE_TARGET": "ALL_BUILD",
|
||||
"CMAKE_NATIVE_BUILD_ARGS": "/m /nologo"
|
||||
}
|
||||
},
|
||||
"monolithic_release_vs2019": {
|
||||
"TAGS": [
|
||||
"nightly",
|
||||
"weekly-build-metrics"
|
||||
],
|
||||
"COMMAND": "build_windows.cmd",
|
||||
"PARAMETERS": {
|
||||
"CONFIGURATION": "release",
|
||||
"OUTPUT_DIRECTORY": "build\\mono_windows_vs2019",
|
||||
"CMAKE_OPTIONS": "-G \"Visual Studio 16 2019\" -DCMAKE_SYSTEM_VERSION=10.0 -DLY_MONOLITHIC_GAME=TRUE -DLY_UNITY_BUILD=TRUE",
|
||||
"CMAKE_LY_PROJECTS": "AutomatedTesting",
|
||||
"CMAKE_TARGET": "ALL_BUILD",
|
||||
"CMAKE_NATIVE_BUILD_ARGS": "/m /nologo"
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,30 @@
|
||||
@ECHO OFF
|
||||
REM
|
||||
REM All or portions of this file Copyright (c) Amazon.com, Inc. or its affiliates or
|
||||
REM its licensors.
|
||||
REM
|
||||
REM For complete copyright and license terms please see the LICENSE at the root of this
|
||||
REM distribution (the "License"). All use of this software is governed by the License,
|
||||
REM or, if provided, by the license below or the license accompanying this file. Do not
|
||||
REM remove or modify any license notices. This file is distributed on an "AS IS" BASIS,
|
||||
REM WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
REM
|
||||
|
||||
SETLOCAL EnableDelayedExpansion
|
||||
|
||||
IF NOT EXIST "%LY_NINJA_PATH%" (
|
||||
SET LY_NINJA_PATH=%LY_3RDPARTY_PATH%/ninja/1.10.1/Windows
|
||||
)
|
||||
IF NOT EXIST "%LY_NINJA_PATH%" (
|
||||
ECHO [ci_build] FAIL: LY_NINJA_PATH=%LY_NINJA_PATH%
|
||||
GOTO :error
|
||||
)
|
||||
PATH %LY_NINJA_PATH%;%PATH%
|
||||
|
||||
CALL "%~dp0build_windows.cmd"
|
||||
IF NOT %ERRORLEVEL%==0 GOTO :error
|
||||
|
||||
EXIT /b 0
|
||||
|
||||
:error
|
||||
EXIT /b 1
|
||||
@@ -0,0 +1,22 @@
|
||||
@ECHO OFF
|
||||
REM
|
||||
REM All or portions of this file Copyright (c) Amazon.com, Inc. or its affiliates or
|
||||
REM its licensors.
|
||||
REM
|
||||
REM For complete copyright and license terms please see the LICENSE at the root of this
|
||||
REM distribution (the "License"). All use of this software is governed by the License,
|
||||
REM or, if provided, by the license below or the license accompanying this file. Do not
|
||||
REM remove or modify any license notices. This file is distributed on an "AS IS" BASIS,
|
||||
REM WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
REM
|
||||
|
||||
CALL "%~dp0build_windows.cmd"
|
||||
IF NOT %ERRORLEVEL%==0 GOTO :error
|
||||
|
||||
CALL "%~dp0test_windows.cmd"
|
||||
IF NOT %ERRORLEVEL%==0 GOTO :error
|
||||
|
||||
EXIT /b 0
|
||||
|
||||
:error
|
||||
EXIT /b 1
|
||||
@@ -0,0 +1,73 @@
|
||||
@ECHO OFF
|
||||
REM
|
||||
REM All or portions of this file Copyright (c) Amazon.com, Inc. or its affiliates or
|
||||
REM its licensors.
|
||||
REM
|
||||
REM For complete copyright and license terms please see the LICENSE at the root of this
|
||||
REM distribution (the "License"). All use of this software is governed by the License,
|
||||
REM or, if provided, by the license below or the license accompanying this file. Do not
|
||||
REM remove or modify any license notices. This file is distributed on an "AS IS" BASIS,
|
||||
REM WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
REM
|
||||
|
||||
SETLOCAL EnableDelayedExpansion
|
||||
|
||||
CALL %~dp0env_windows.cmd
|
||||
|
||||
IF NOT EXIST "%OUTPUT_DIRECTORY%" (
|
||||
MKDIR %OUTPUT_DIRECTORY%.
|
||||
)
|
||||
|
||||
SET SOURCE_DIRECTORY=%CD%
|
||||
PUSHD %OUTPUT_DIRECTORY%
|
||||
|
||||
ECHO [ci_build] cmake --version
|
||||
cmake --version
|
||||
IF ERRORLEVEL 1 (
|
||||
ECHO [ci_build] CMAKE not found!
|
||||
exit /b 1
|
||||
)
|
||||
|
||||
REM Jenkins reports MSB8029 when TMP/TEMP is not defined, define a dummy folder
|
||||
SET TMP=%cd%/temp
|
||||
SET TEMP=%cd%/temp
|
||||
IF NOT EXIST %TMP% (
|
||||
MKDIR temp
|
||||
)
|
||||
|
||||
REM Compute half the amount of processors so some jobs can run
|
||||
SET /a HALF_PROCESSORS = NUMBER_OF_PROCESSORS / 2
|
||||
|
||||
SET LAST_CONFIGURE_CMD_FILE=ci_last_configure_cmd.txt
|
||||
SET CONFIGURE_CMD=cmake %SOURCE_DIRECTORY% %CMAKE_OPTIONS% %EXTRA_CMAKE_OPTIONS% -DLY_3RDPARTY_PATH="%LY_3RDPARTY_PATH%" -DLY_PROJECTS=%CMAKE_LY_PROJECTS%
|
||||
IF NOT EXIST CMakeCache.txt (
|
||||
ECHO [ci_build] First run, generating
|
||||
SET RUN_CONFIGURE=1
|
||||
) ELSE IF NOT EXIST %LAST_CONFIGURE_CMD_FILE% (
|
||||
ECHO [ci_build] Last run command not found, generating
|
||||
SET RUN_CONFIGURE=1
|
||||
) ELSE (
|
||||
REM Detect if the input has changed
|
||||
FOR /F "delims=" %%x in (%LAST_CONFIGURE_CMD_FILE%) DO SET LAST_CMD=%%x
|
||||
IF !LAST_CMD! NEQ !CONFIGURE_CMD! (
|
||||
ECHO [ci_build] Last run command different, generating
|
||||
SET RUN_CONFIGURE=1
|
||||
)
|
||||
)
|
||||
IF DEFINED RUN_CONFIGURE (
|
||||
ECHO [ci_build] %CONFIGURE_CMD%
|
||||
%CONFIGURE_CMD%
|
||||
IF NOT !ERRORLEVEL!==0 GOTO :error
|
||||
ECHO !CONFIGURE_CMD!> %LAST_CONFIGURE_CMD_FILE%
|
||||
)
|
||||
|
||||
ECHO [ci_build] cmake --build . --target %CMAKE_TARGET% --config %CONFIGURATION% %CMAKE_BUILD_ARGS% -- %CMAKE_NATIVE_BUILD_ARGS%
|
||||
cmake --build . --target %CMAKE_TARGET% --config %CONFIGURATION% %CMAKE_BUILD_ARGS% -- %CMAKE_NATIVE_BUILD_ARGS%
|
||||
IF NOT %ERRORLEVEL%==0 GOTO :error
|
||||
|
||||
POPD
|
||||
EXIT /b 0
|
||||
|
||||
:error
|
||||
POPD
|
||||
EXIT /b 1
|
||||
@@ -0,0 +1,31 @@
|
||||
@ECHO OFF
|
||||
REM
|
||||
REM All or portions of this file Copyright (c) Amazon.com, Inc. or its affiliates or
|
||||
REM its licensors.
|
||||
REM
|
||||
REM For complete copyright and license terms please see the LICENSE at the root of this
|
||||
REM distribution (the "License"). All use of this software is governed by the License,
|
||||
REM or, if provided, by the license below or the license accompanying this file. Do not
|
||||
REM remove or modify any license notices. This file is distributed on an "AS IS" BASIS,
|
||||
REM WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
REM
|
||||
|
||||
SETLOCAL EnableDelayedExpansion
|
||||
|
||||
IF DEFINED CLEAN_ASSETS (
|
||||
ECHO [ci_build] CLEAN_ASSETS option set
|
||||
FOR %%P in (%CMAKE_LY_PROJECTS%) do (
|
||||
IF EXIST %%P\Cache (
|
||||
ECHO [ci_build] Deleting "%%P\Cache"
|
||||
DEL /s /q /f %%P\Cache 1>nul
|
||||
)
|
||||
)
|
||||
)
|
||||
|
||||
IF DEFINED CLEAN_OUTPUT_DIRECTORY (
|
||||
ECHO [ci_build] CLEAN_OUTPUT_DIRECTORY option set
|
||||
IF EXIST %OUTPUT_DIRECTORY% (
|
||||
ECHO [ci_build] Deleting "%OUTPUT_DIRECTORY%"
|
||||
DEL /s /q /f %OUTPUT_DIRECTORY% 1>nul
|
||||
)
|
||||
)
|
||||
@@ -0,0 +1,31 @@
|
||||
@ECHO OFF
|
||||
REM
|
||||
REM All or portions of this file Copyright (c) Amazon.com, Inc. or its affiliates or
|
||||
REM its licensors.
|
||||
REM
|
||||
REM For complete copyright and license terms please see the LICENSE at the root of this
|
||||
REM distribution (the "License"). All use of this software is governed by the License,
|
||||
REM or, if provided, by the license below or the license accompanying this file. Do not
|
||||
REM remove or modify any license notices. This file is distributed on an "AS IS" BASIS,
|
||||
REM WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
REM
|
||||
|
||||
where /Q cmake
|
||||
IF NOT %ERRORLEVEL%==0 (
|
||||
IF "%LY_CMAKE_PATH%"=="" (SET LY_CMAKE_PATH=%LY_3RDPARTY_PATH%/CMake/3.19.1/Windows/bin)
|
||||
IF NOT EXIST !LY_CMAKE_PATH! (
|
||||
ECHO [ci_build] CMake path not found
|
||||
GOTO :error
|
||||
)
|
||||
PATH !LY_CMAKE_PATH!;!PATH!
|
||||
where /Q cmake
|
||||
IF NOT !ERRORLEVEL!==0 (
|
||||
ECHO [ci_build] CMake not found
|
||||
GOTO :error
|
||||
)
|
||||
)
|
||||
|
||||
EXIT /b 0
|
||||
|
||||
:error
|
||||
EXIT /b 1
|
||||
@@ -0,0 +1,24 @@
|
||||
{
|
||||
"profile_vs2017_atom": {
|
||||
"COMMAND": "build_windows.cmd",
|
||||
"PARAMETERS": {
|
||||
"CONFIGURATION": "profile",
|
||||
"OUTPUT_DIRECTORY": "windows_vs2017",
|
||||
"CMAKE_OPTIONS": "-G \"Visual Studio 15 2017\" -A x64 -T host=x64 -DCMAKE_SYSTEM_VERSION=10.0 -DLY_UNITY_BUILD=TRUE",
|
||||
"CMAKE_LY_PROJECTS": "AtomTest;AtomSampleViewer",
|
||||
"CMAKE_TARGET": "ALL_BUILD",
|
||||
"CMAKE_NATIVE_BUILD_ARGS": "/m:4 /p:CL_MPCount=!HALF_PROCESSORS! /nologo"
|
||||
}
|
||||
},
|
||||
"profile_vs2019_atom": {
|
||||
"COMMAND":"build_windows.cmd",
|
||||
"PARAMETERS": {
|
||||
"CONFIGURATION":"profile",
|
||||
"OUTPUT_DIRECTORY":"windows_vs2019",
|
||||
"CMAKE_OPTIONS":"-G \"Visual Studio 16 2019\" -DCMAKE_SYSTEM_VERSION=10.0 -DLY_UNITY_BUILD=TRUE",
|
||||
"CMAKE_LY_PROJECTS":"AtomTest;AtomSampleViewer",
|
||||
"CMAKE_TARGET":"ALL_BUILD",
|
||||
"CMAKE_NATIVE_BUILD_ARGS": "/m /nologo"
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,17 @@
|
||||
{
|
||||
"ENV": {
|
||||
"NODE_LABEL": "windows-047e5cdf",
|
||||
"LY_3RDPARTY_PATH": "C:/ly/3rdParty",
|
||||
"TIMEOUT": 30,
|
||||
"WORKSPACE": "D:/workspace",
|
||||
"MOUNT_VOLUME": true
|
||||
},
|
||||
"PIPELINE_ENV_OVERRIDE": {
|
||||
"daily-pipeline-metrics": {
|
||||
"CLEAN_WORKSPACE": true
|
||||
},
|
||||
"packaging": {
|
||||
"CLEAN_WORKSPACE": true
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,18 @@
|
||||
@ECHO OFF
|
||||
REM
|
||||
REM All or portions of this file Copyright (c) Amazon.com, Inc. or its affiliates or
|
||||
REM its licensors.
|
||||
REM
|
||||
REM For complete copyright and license terms please see the LICENSE at the root of this
|
||||
REM distribution (the "License"). All use of this software is governed by the License,
|
||||
REM or, if provided, by the license below or the license accompanying this file. Do not
|
||||
REM remove or modify any license notices. This file is distributed on an "AS IS" BASIS,
|
||||
REM WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
REM
|
||||
|
||||
SETLOCAL EnableDelayedExpansion
|
||||
|
||||
ECHO [ci_build] python/python.cmd -u %SCRIPT_PATH% %SCRIPT_PARAMETERS%
|
||||
CALL python/python.cmd -u %SCRIPT_PATH% %SCRIPT_PARAMETERS%
|
||||
|
||||
EXIT /b %ERRORLEVEL%
|
||||
@@ -0,0 +1,41 @@
|
||||
@ECHO OFF
|
||||
REM
|
||||
REM All or portions of this file Copyright (c) Amazon.com, Inc. or its affiliates or
|
||||
REM its licensors.
|
||||
REM
|
||||
REM For complete copyright and license terms please see the LICENSE at the root of this
|
||||
REM distribution (the "License"). All use of this software is governed by the License,
|
||||
REM or, if provided, by the license below or the license accompanying this file. Do not
|
||||
REM remove or modify any license notices. This file is distributed on an "AS IS" BASIS,
|
||||
REM WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
REM
|
||||
|
||||
SETLOCAL EnableDelayedExpansion
|
||||
|
||||
CALL %~dp0env_windows.cmd
|
||||
|
||||
IF NOT EXIST %OUTPUT_DIRECTORY% (
|
||||
ECHO [ci_build] Error: $OUTPUT_DIRECTORY was not found
|
||||
GOTO :error
|
||||
)
|
||||
PUSHD %OUTPUT_DIRECTORY%
|
||||
|
||||
REM Find the CTEST_RUN_FLAGS from cmake's cache variables, then replace the $<CONFIG> with the current configuration
|
||||
FOR /F "tokens=2 delims==" %%V IN ('cmake -N -LA . ^| findstr /I CTEST_RUN_FLAGS:STRING') do (
|
||||
SET CTEST_RUN_FLAGS=%%V
|
||||
)
|
||||
SET CTEST_RUN_FLAGS=%CTEST_RUN_FLAGS:$<CONFIG>=!CONFIGURATION!%
|
||||
|
||||
REM Run ctest
|
||||
ECHO [ci_build] ctest !CTEST_RUN_FLAGS! !CTEST_OPTIONS!
|
||||
ctest !CTEST_RUN_FLAGS! !CTEST_OPTIONS!
|
||||
IF NOT %ERRORLEVEL%==0 GOTO :popd_error
|
||||
|
||||
POPD
|
||||
EXIT /b 0
|
||||
|
||||
:popd_error
|
||||
POPD
|
||||
|
||||
:error
|
||||
EXIT /b 1
|
||||
@@ -0,0 +1,110 @@
|
||||
{
|
||||
"clean": {
|
||||
"TAGS": [],
|
||||
"COMMAND": "../Mac/clean_mac.sh",
|
||||
"PARAMETERS": {
|
||||
"OUTPUT_DIRECTORY": "build",
|
||||
"CMAKE_LY_PROJECTS": "AutomatedTesting"
|
||||
}
|
||||
},
|
||||
"metrics": {
|
||||
"TAGS": [
|
||||
"weekly"
|
||||
],
|
||||
"COMMAND": "../Mac/python_mac.sh",
|
||||
"PARAMETERS": {
|
||||
"SCRIPT_PATH": "scripts/build/ci_build_metrics.py",
|
||||
"SCRIPT_PARAMETERS": "--platform iOS --jobname '${JOB_NAME}' --jobnumber '${BUILD_NUMBER}' --jobnode '${NODE_NAME}' --changelist '${CHANGE_ID}'"
|
||||
}
|
||||
},
|
||||
"debug": {
|
||||
"TAGS": [
|
||||
"nightly",
|
||||
"weekly-build-metrics"
|
||||
],
|
||||
"COMMAND": "../Mac/build_mac.sh",
|
||||
"PARAMETERS": {
|
||||
"CONFIGURATION": "debug",
|
||||
"OUTPUT_DIRECTORY": "build/ios",
|
||||
"CMAKE_OPTIONS": "-G Xcode -DCMAKE_TOOLCHAIN_FILE=cmake/Platform/iOS/Toolchain_ios.cmake -DLY_MONOLITHIC_GAME=TRUE -DCMAKE_XCODE_ATTRIBUTE_CODE_SIGNING_REQUIRED=FALSE -DLY_IOS_CODE_SIGNING_IDENTITY=\"\" -DCMAKE_XCODE_ATTRIBUTE_CODE_SIGN_ENTITLEMENTS=\"\" -DCMAKE_XCODE_ATTRIBUTE_CODE_SIGNING_ALLOWED=FALSE -DLY_UNITY_BUILD=TRUE",
|
||||
"CMAKE_LY_PROJECTS": "AutomatedTesting",
|
||||
"CMAKE_TARGET": "ALL_BUILD",
|
||||
"CMAKE_NATIVE_BUILD_ARGS": "-destination generic/platform=iOS"
|
||||
}
|
||||
},
|
||||
"profile": {
|
||||
"TAGS": [
|
||||
"nightly",
|
||||
"daily-pipeline-metrics",
|
||||
"weekly-build-metrics"
|
||||
],
|
||||
"COMMAND": "../Mac/build_mac.sh",
|
||||
"PARAMETERS": {
|
||||
"CONFIGURATION": "profile",
|
||||
"OUTPUT_DIRECTORY": "build/ios",
|
||||
"CMAKE_OPTIONS": "-G Xcode -DCMAKE_TOOLCHAIN_FILE=cmake/Platform/iOS/Toolchain_ios.cmake -DLY_MONOLITHIC_GAME=TRUE -DCMAKE_XCODE_ATTRIBUTE_CODE_SIGNING_REQUIRED=FALSE -DLY_IOS_CODE_SIGNING_IDENTITY=\"\" -DCMAKE_XCODE_ATTRIBUTE_CODE_SIGN_ENTITLEMENTS=\"\" -DCMAKE_XCODE_ATTRIBUTE_CODE_SIGNING_ALLOWED=FALSE -DLY_UNITY_BUILD=TRUE",
|
||||
"CMAKE_LY_PROJECTS": "AutomatedTesting",
|
||||
"CMAKE_TARGET": "ALL_BUILD",
|
||||
"CMAKE_NATIVE_BUILD_ARGS": "-destination generic/platform=iOS"
|
||||
}
|
||||
},
|
||||
"profile_nounity": {
|
||||
"TAGS": [
|
||||
"nightly",
|
||||
"weekly-build-metrics"
|
||||
],
|
||||
"COMMAND": "../Mac/build_mac.sh",
|
||||
"PARAMETERS": {
|
||||
"CONFIGURATION": "profile",
|
||||
"OUTPUT_DIRECTORY": "build/ios",
|
||||
"CMAKE_OPTIONS": "-G Xcode -DCMAKE_TOOLCHAIN_FILE=cmake/Platform/iOS/Toolchain_ios.cmake -DLY_MONOLITHIC_GAME=TRUE -DCMAKE_XCODE_ATTRIBUTE_CODE_SIGNING_REQUIRED=FALSE -DLY_IOS_CODE_SIGNING_IDENTITY=\"\" -DCMAKE_XCODE_ATTRIBUTE_CODE_SIGN_ENTITLEMENTS=\"\" -DCMAKE_XCODE_ATTRIBUTE_CODE_SIGNING_ALLOWED=FALSE -DLY_UNITY_BUILD=FALSE",
|
||||
"CMAKE_LY_PROJECTS": "AutomatedTesting",
|
||||
"CMAKE_TARGET": "ALL_BUILD",
|
||||
"CMAKE_NATIVE_BUILD_ARGS": "-destination generic/platform=iOS"
|
||||
}
|
||||
},
|
||||
"asset_profile": {
|
||||
"TAGS": [
|
||||
"nightly",
|
||||
"weekly-build-metrics"
|
||||
],
|
||||
"COMMAND": "../Mac/build_asset_mac.sh",
|
||||
"PARAMETERS": {
|
||||
"CONFIGURATION": "profile",
|
||||
"OUTPUT_DIRECTORY": "build/mac",
|
||||
"CMAKE_OPTIONS": "-G Xcode -DLY_UNITY_BUILD=TRUE",
|
||||
"CMAKE_LY_PROJECTS": "AutomatedTesting",
|
||||
"CMAKE_TARGET": "AssetProcessorBatch",
|
||||
"ASSET_PROCESSOR_BINARY": "bin/profile/AssetProcessorBatch",
|
||||
"ASSET_PROCESSOR_OPTIONS": "/zeroAnalysisMode",
|
||||
"ASSET_PROCESSOR_PLATFORMS": "ios"
|
||||
}
|
||||
},
|
||||
"asset_clean_profile": {
|
||||
"TAGS": [
|
||||
"nightly"
|
||||
],
|
||||
"PIPELINE_ENV": {
|
||||
"CLEAN_ASSETS": "true"
|
||||
},
|
||||
"steps": [
|
||||
"clean",
|
||||
"asset_profile"
|
||||
]
|
||||
},
|
||||
"release": {
|
||||
"TAGS": [
|
||||
"nightly",
|
||||
"weekly-build-metrics"
|
||||
],
|
||||
"COMMAND": "../Mac/build_mac.sh",
|
||||
"PARAMETERS": {
|
||||
"CONFIGURATION": "release",
|
||||
"OUTPUT_DIRECTORY": "build/ios",
|
||||
"CMAKE_OPTIONS": "-G Xcode -DCMAKE_TOOLCHAIN_FILE=cmake/Platform/iOS/Toolchain_ios.cmake -DLY_MONOLITHIC_GAME=TRUE -DCMAKE_XCODE_ATTRIBUTE_CODE_SIGNING_REQUIRED=FALSE -DLY_IOS_CODE_SIGNING_IDENTITY=\"\" -DCMAKE_XCODE_ATTRIBUTE_CODE_SIGN_ENTITLEMENTS=\"\" -DCMAKE_XCODE_ATTRIBUTE_CODE_SIGNING_ALLOWED=FALSE -DLY_UNITY_BUILD=TRUE",
|
||||
"CMAKE_LY_PROJECTS": "AutomatedTesting",
|
||||
"CMAKE_TARGET": "ALL_BUILD",
|
||||
"CMAKE_NATIVE_BUILD_ARGS": "-destination generic/platform=iOS"
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,17 @@
|
||||
{
|
||||
"ENV": {
|
||||
"NODE_LABEL": "mac",
|
||||
"LY_3RDPARTY_PATH": "/Users/lybuilder/3rdParty",
|
||||
"TIMEOUT": 30,
|
||||
"WORKSPACE": "/Users/lybuilder/workspace",
|
||||
"MOUNT_VOLUME": false
|
||||
},
|
||||
"PIPELINE_ENV_OVERRIDE": {
|
||||
"daily-pipeline-metrics": {
|
||||
"CLEAN_WORKSPACE": true
|
||||
},
|
||||
"packaging": {
|
||||
"CLEAN_WORKSPACE": true
|
||||
}
|
||||
}
|
||||
}
|
||||
Executable
+104
@@ -0,0 +1,104 @@
|
||||
#
|
||||
# All or portions of this file Copyright (c) Amazon.com, Inc. or its affiliates or
|
||||
# its licensors.
|
||||
#
|
||||
# For complete copyright and license terms please see the LICENSE at the root of this
|
||||
# distribution (the "License"). All use of this software is governed by the License,
|
||||
# or, if provided, by the license below or the license accompanying this file. Do not
|
||||
# remove or modify any license notices. This file is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
#
|
||||
|
||||
import argparse
|
||||
import json
|
||||
import os
|
||||
import sys
|
||||
import subprocess
|
||||
|
||||
def parse_args():
|
||||
cur_dir = os.path.dirname(os.path.abspath(__file__))
|
||||
parser = argparse.ArgumentParser()
|
||||
parser.add_argument('-p', '--platform', dest="build_platform", help="Platform to build")
|
||||
parser.add_argument('-t', '--type', dest="build_type", help="Target config type to build")
|
||||
parser.add_argument('-c', '--config', dest="build_config_filename",
|
||||
default="build_config.json",
|
||||
help="JSON filename in Platform/<platform> that defines build configurations for the platform")
|
||||
args = parser.parse_args()
|
||||
|
||||
# Input validation
|
||||
if args.build_platform is None:
|
||||
print('[ci_build] No platform specified')
|
||||
sys.exit(-1)
|
||||
|
||||
if args.build_type is None:
|
||||
print('[ci_build] No type specified')
|
||||
sys.exit(-1)
|
||||
|
||||
return args
|
||||
|
||||
def build(build_config_filename, build_platform, build_type):
|
||||
# Read build_config and locate build_type
|
||||
current_dir = os.path.dirname(os.path.abspath(__file__))
|
||||
cwd_dir = os.path.abspath(os.path.join(current_dir, '../..')) # engine's root
|
||||
config_dir = os.path.abspath(os.path.join(current_dir, 'Platform', build_platform))
|
||||
build_config_abspath = os.path.join(config_dir, build_config_filename)
|
||||
if not os.path.exists(build_config_abspath):
|
||||
config_dir = os.path.abspath(os.path.join(cwd_dir, 'restricted', build_platform, os.path.relpath(current_dir, cwd_dir)))
|
||||
build_config_abspath = os.path.join(config_dir, build_config_filename)
|
||||
if not os.path.exists(build_config_abspath):
|
||||
print('[ci_build] File: {} not found'.format(build_config_abspath))
|
||||
return -1
|
||||
|
||||
with open(build_config_abspath) as f:
|
||||
build_config_json = json.load(f)
|
||||
|
||||
build_type_config = build_config_json[build_type]
|
||||
if build_type_config is None:
|
||||
print('[ci_build] Build type {} was not found in {}'.format(build_type, build_config_abspath))
|
||||
|
||||
# Load the command to execute
|
||||
build_cmd = build_type_config['COMMAND']
|
||||
if build_cmd is None:
|
||||
print('[ci_build] Build type {} in {} is missing required COMMAND entry'.format(build_type, build_config_abspath))
|
||||
return -1
|
||||
|
||||
build_params = build_type_config['PARAMETERS']
|
||||
# Parameters are optional, so we could have none
|
||||
|
||||
# build_cmd is relative to the folder where this file is
|
||||
build_cmd_path = os.path.join(current_dir, 'Platform/{}/{}'.format(build_platform, build_cmd))
|
||||
if not os.path.exists(build_cmd_path):
|
||||
config_dir = os.path.abspath(os.path.join(cwd_dir, 'restricted', build_platform, os.path.relpath(current_dir, cwd_dir)))
|
||||
build_cmd_path = os.path.join(config_dir, build_cmd)
|
||||
if not os.path.exists(build_cmd_path):
|
||||
print('[ci_build] File: {} not found'.format(build_cmd_path))
|
||||
return -1
|
||||
|
||||
print('[ci_build] Executing \"{}\"'.format(build_cmd_path))
|
||||
print(' cwd = {}'.format(cwd_dir))
|
||||
print(' paramaters:')
|
||||
env_params = os.environ.copy()
|
||||
for v in build_params:
|
||||
if v[:6] == "FORCE:":
|
||||
env_params[v[6:]] = build_params[v]
|
||||
print(' {} = {} (forced)'.format(v[6:], env_params[v[6:]]))
|
||||
else:
|
||||
existing_param = env_params.get(v)
|
||||
if not existing_param:
|
||||
env_params[v] = build_params[v]
|
||||
print(' {} = {} {}'.format(v, env_params[v], '(environment override)' if existing_param else ''))
|
||||
print('--------------------------------------------------------------------------------', flush=True)
|
||||
process_return = subprocess.run(build_cmd_path, cwd=cwd_dir, env=env_params)
|
||||
print('--------------------------------------------------------------------------------')
|
||||
if process_return.returncode != 0:
|
||||
print('[ci_build] FAIL: Command {} returned {}'.format(build_cmd_path, process_return.returncode), flush=True)
|
||||
return process_return.returncode
|
||||
else:
|
||||
print('[ci_build] OK', flush=True)
|
||||
|
||||
return 0
|
||||
|
||||
if __name__ == "__main__":
|
||||
args = parse_args()
|
||||
ret = build(args.build_config_filename, args.build_platform, args.build_type)
|
||||
sys.exit(ret)
|
||||
Executable
+299
@@ -0,0 +1,299 @@
|
||||
#
|
||||
# All or portions of this file Copyright (c) Amazon.com, Inc. or its affiliates or
|
||||
# its licensors.
|
||||
#
|
||||
# For complete copyright and license terms please see the LICENSE at the root of this
|
||||
# distribution (the "License"). All use of this software is governed by the License,
|
||||
# or, if provided, by the license below or the license accompanying this file. Do not
|
||||
# remove or modify any license notices. This file is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
#
|
||||
|
||||
import argparse
|
||||
import datetime
|
||||
import json
|
||||
import math
|
||||
import os
|
||||
import platform
|
||||
import psutil
|
||||
import shutil
|
||||
import stat
|
||||
import subprocess
|
||||
import sys
|
||||
import time
|
||||
import ci_build
|
||||
from pathlib import Path
|
||||
import submit_metrics
|
||||
|
||||
if platform.system() == 'Windows':
|
||||
EXE_EXTENSION = '.exe'
|
||||
else:
|
||||
EXE_EXTENSION = ''
|
||||
|
||||
DATE_FORMAT = "%Y-%m-%dT%H:%M:%S"
|
||||
|
||||
METRICS_TEST_MODE = os.environ.get('METRICS_TEST_MODE')
|
||||
|
||||
def parse_args():
|
||||
cur_dir = os.path.dirname(os.path.abspath(__file__))
|
||||
parser = argparse.ArgumentParser()
|
||||
parser.add_argument('-p', '--platform', dest="platform", help="Platform to gather metrics for")
|
||||
parser.add_argument('-a', '--jobname', dest="jobname", default="unknown", help="Name/tag of the job in the CI system (used to track where the report comes from, constant through multiple runs)")
|
||||
parser.add_argument('-u', '--jobnumber', dest="jobnumber", default=-1, help="Number of run in the CI system (used to track where the report comes from, variable through runs)")
|
||||
parser.add_argument('-o', '--jobnode', dest="jobnode", default="unknown", help="Build node name (used to track where the build happened in CI systems where the same jobs run in different hosts)")
|
||||
parser.add_argument('-l', '--changelist', dest="changelist", default=-1, help="Last changelist in this workspace")
|
||||
parser.add_argument('-c', '--config', dest="build_config_filename", default="build_config.json",
|
||||
help="JSON filename in Platform/<platform> that defines build configurations for the platform")
|
||||
args = parser.parse_args()
|
||||
|
||||
# Input validation
|
||||
if args.platform is None:
|
||||
print('[ci_build_metrics] No platform specified')
|
||||
sys.exit(-1)
|
||||
|
||||
return args
|
||||
|
||||
def shutdown_processes():
|
||||
process_list = {f'AssetBuilder{EXE_EXTENSION}', f'AssetProcessor{EXE_EXTENSION}', f'RC{EXE_EXTENSION}', f'Editor{EXE_EXTENSION}'}
|
||||
|
||||
for process in psutil.process_iter():
|
||||
try:
|
||||
if process.name() in process_list:
|
||||
process.kill()
|
||||
except Exception as e:
|
||||
print(f'Exception while trying to kill process: {e}')
|
||||
|
||||
def on_rmtree_error(func, path, exc_info):
|
||||
if os.path.exists(path):
|
||||
try:
|
||||
os.chmod(path, stat.S_IWRITE)
|
||||
os.unlink(path)
|
||||
except Exception as e:
|
||||
print(f'Exception while trying to remove {path}: {e}')
|
||||
|
||||
def clean_folder(folder):
|
||||
if os.path.exists(folder):
|
||||
print(f'[ci_build_metrics] Cleaning {folder}...', flush=True)
|
||||
if not METRICS_TEST_MODE:
|
||||
shutil.rmtree(folder, onerror=on_rmtree_error)
|
||||
print(f'[ci_build_metrics] Cleaned {folder}', flush=True)
|
||||
|
||||
def compute_folder_size(folder):
|
||||
total = 0
|
||||
if os.path.exists(folder):
|
||||
folder_path = Path(folder)
|
||||
print(f'[ci_build_metrics] Computing size of {folder_path}...', flush=True)
|
||||
if not METRICS_TEST_MODE:
|
||||
total += sum(f.stat().st_size for f in folder_path.glob('**/*') if f.is_file())
|
||||
print(f'[ci_build_metrics] Computed size of {folder_path}', flush=True)
|
||||
return total
|
||||
|
||||
def build(metrics, folders_of_interest, build_config_filename, platform, build_type, output_directory, time_delta = 0):
|
||||
build_start = time.time()
|
||||
if not METRICS_TEST_MODE:
|
||||
metrics['result'] = ci_build.build(build_config_filename, platform, build_type)
|
||||
else:
|
||||
metrics['result'] = -1 # mark as failure so the data is not used in elastic
|
||||
build_end = time.time()
|
||||
|
||||
# truncate the duration (expressed in seconds), we dont need more precision
|
||||
metrics['duration'] = math.trunc(build_end - build_start - time_delta)
|
||||
|
||||
metrics['output_sizes'] = []
|
||||
output_sizes = metrics['output_sizes']
|
||||
for folder in folders_of_interest:
|
||||
output_size = dict()
|
||||
if folder != output_directory:
|
||||
output_size['folder'] = folder
|
||||
else:
|
||||
output_size['folder'] = 'OUTPUT_DIRECTORY' # make it homogenous to facilitate search
|
||||
output_size['output_size'] = compute_folder_size(os.path.join(engine_dir, folder))
|
||||
output_sizes.append(output_size)
|
||||
|
||||
def gather_build_metrics(current_dir, build_config_filename, platform):
|
||||
config_dir = os.path.abspath(os.path.join(current_dir, 'Platform', platform))
|
||||
build_config_abspath = os.path.join(config_dir, build_config_filename)
|
||||
if not os.path.exists(build_config_abspath):
|
||||
cwd_dir = os.path.abspath(os.path.join(current_dir, '../..')) # engine's root
|
||||
config_dir = os.path.abspath(os.path.join(cwd_dir, 'restricted', platform, os.path.relpath(current_dir, cwd_dir)))
|
||||
build_config_abspath = os.path.join(config_dir, build_config_filename)
|
||||
|
||||
if not os.path.exists(build_config_abspath):
|
||||
print(f'[ci_build_metrics] File: {build_config_abspath} not found', flush=True)
|
||||
sys.exit(-1)
|
||||
|
||||
with open(build_config_abspath) as f:
|
||||
build_config_json = json.load(f)
|
||||
|
||||
all_builds_metrics = []
|
||||
|
||||
for build_type in build_config_json:
|
||||
|
||||
build_config = build_config_json[build_type]
|
||||
if not 'weekly-build-metrics' in build_config['TAGS']:
|
||||
# skip build configs that are not tagged with 'weekly-build-metrics'
|
||||
continue
|
||||
|
||||
print(f'[ci_build_metrics] Starting {build_type}', flush=True)
|
||||
metrics = dict()
|
||||
metrics['build_type'] = build_type
|
||||
|
||||
build_parameters = build_config['PARAMETERS']
|
||||
if not build_parameters:
|
||||
metrics['result'] = -1
|
||||
reason = f'PARAMETERS entry {build_type} in {build_config_abspath} is missing.'
|
||||
metrics['reason'] = reason
|
||||
print(f'[ci_build_metrics] {reason}', flush=True)
|
||||
continue
|
||||
|
||||
# Clean the build output
|
||||
output_directory = build_parameters['OUTPUT_DIRECTORY'] if 'OUTPUT_DIRECTORY' in build_parameters else None
|
||||
if not output_directory:
|
||||
metrics['result'] = -1
|
||||
reason = f'OUTPUT_DIRECTORY entry in {build_config_abspath} is missing.'
|
||||
metrics['reason'] = reason
|
||||
print(f'[ci_build_metrics] {reason}', flush=True)
|
||||
continue
|
||||
folders_of_interest = [output_directory]
|
||||
|
||||
# Clean the AP output
|
||||
cmake_ly_projects = build_parameters['CMAKE_LY_PROJECTS'] if 'CMAKE_LY_PROJECTS' in build_parameters else None
|
||||
if cmake_ly_projects:
|
||||
projects = cmake_ly_projects.split(';')
|
||||
for project in projects:
|
||||
folders_of_interest.append(os.path.join(project, 'user', 'AssetProcessorTemp'))
|
||||
folders_of_interest.append(os.path.join(project, 'Cache'))
|
||||
|
||||
metrics['build_metrics'] = []
|
||||
build_metrics = metrics['build_metrics']
|
||||
|
||||
# Do the clean build
|
||||
shutdown_processes()
|
||||
for folder in folders_of_interest:
|
||||
clean_folder(os.path.join(engine_dir, folder))
|
||||
|
||||
build_metric_clean = dict()
|
||||
build_metric_clean['build_metric'] = 'clean'
|
||||
build(build_metric_clean, folders_of_interest, build_config_filename, platform, build_type, output_directory)
|
||||
build_metrics.append(build_metric_clean)
|
||||
|
||||
# Do the incremental "zero" build
|
||||
build_metric_zero = dict()
|
||||
build_metric_zero['build_metric'] = 'zero'
|
||||
build(build_metric_zero, folders_of_interest, build_config_filename, platform, build_type, output_directory)
|
||||
build_metrics.append(build_metric_zero)
|
||||
|
||||
# Do a reconfigure
|
||||
# To measure a reconfigure, we will delete the "ci_last_configure_cmd.txt" file from the output and trigger a
|
||||
# zero build, then we will substract the time from the zero_build above
|
||||
last_configure_file = os.path.join(output_directory, 'ci_last_configure_cmd.txt')
|
||||
if os.path.exists(last_configure_file):
|
||||
os.remove(last_configure_file)
|
||||
build_metric_generation = dict()
|
||||
build_metric_generation['build_metric'] = 'generation'
|
||||
build(build_metric_generation, folders_of_interest, build_config_filename, platform, build_type, output_directory, build_metric_zero['duration'])
|
||||
build_metrics.append(build_metric_generation)
|
||||
|
||||
# Clean the otuput before ending to reduce the size of these workspaces
|
||||
shutdown_processes()
|
||||
for folder in folders_of_interest:
|
||||
clean_folder(os.path.join(engine_dir, folder))
|
||||
|
||||
metrics['result'] = 0
|
||||
metrics['reason'] = 'OK'
|
||||
|
||||
all_builds_metrics.append(metrics)
|
||||
|
||||
return all_builds_metrics
|
||||
|
||||
def prepare_metrics(args, build_metrics):
|
||||
return {
|
||||
'changelist': args.changelist,
|
||||
'job': {'name': args.jobname, 'number': args.jobnumber, 'node': args.jobnode},
|
||||
'platform': args.platform,
|
||||
'build_types': build_metrics,
|
||||
'timestamp': timestamp.strftime("%Y-%m-%dT%H:%M:%S")
|
||||
}
|
||||
|
||||
def upload_to_s3(upload_script_path, base_dir, bucket, key_prefix):
|
||||
try:
|
||||
subprocess.run([sys.executable, upload_script_path,
|
||||
'--base_dir', base_dir,
|
||||
'--file_regex', '.*',
|
||||
'--bucket', bucket,
|
||||
'--key_prefix', key_prefix],
|
||||
check=True)
|
||||
except subprocess.CalledProcessError as err:
|
||||
print(f'[ci_build_metrics] {upload_script_path} failed with error {err}')
|
||||
sys.exit(1)
|
||||
|
||||
def submit_report_document(report_file):
|
||||
print(f'[ci_build_metrics] Submitting {report_file}')
|
||||
with open(report_file) as json_file:
|
||||
report_json = json.load(json_file)
|
||||
|
||||
ret = True
|
||||
for build_type in report_json['build_types']:
|
||||
for build_metric in build_type['build_metrics']:
|
||||
newjson = {
|
||||
'timestamp': report_json['timestamp'],
|
||||
'changelist': report_json['changelist'],
|
||||
'job': report_json['job'],
|
||||
'platform': report_json['platform'],
|
||||
'type': build_type['build_type'],
|
||||
'result': int(build_type['result']) or int(build_metric['result']),
|
||||
'reason': build_type['reason'],
|
||||
'metric': build_metric['build_metric'],
|
||||
'duration': build_metric['duration'],
|
||||
'output_sizes': build_metric['output_sizes']
|
||||
}
|
||||
|
||||
index = "pappeste.build_metrics." + datetime.datetime.strptime(report_json['timestamp'], DATE_FORMAT).strftime("%Y.%m")
|
||||
ret &= submit_metrics.submit(index, newjson)
|
||||
|
||||
if ret:
|
||||
print(f'[ci_build_metrics] {report_file} submitted')
|
||||
else:
|
||||
print(f'[ci_build_metrics] {report_file} failed to submit')
|
||||
return ret
|
||||
|
||||
if __name__ == "__main__":
|
||||
args = parse_args()
|
||||
print(f"[ci_build_metrics] Generatic build metrics for:"
|
||||
f"\n\tPlatform: {args.platform}"
|
||||
f"\n\tJob Name: {args.jobname}"
|
||||
f"\n\tJob Number: {args.jobnumber}"
|
||||
f"\n\tJob Node: {args.jobnode}"
|
||||
f"\n\tChangelist: {args.changelist}")
|
||||
|
||||
# Read build_config
|
||||
current_dir = os.path.dirname(os.path.abspath(__file__))
|
||||
engine_dir = os.path.abspath(os.path.join(current_dir, '../..')) # engine's root
|
||||
timestamp = datetime.datetime.now()
|
||||
|
||||
build_metrics = gather_build_metrics(current_dir, args.build_config_filename, args.platform)
|
||||
metrics = prepare_metrics(args, build_metrics)
|
||||
|
||||
# Temporarly just printing the metrics until we get an API to uplaod it to CloudWatch
|
||||
# SPEC-1810 will then upload these metrics
|
||||
print("[ci_build_metrics] metrics:")
|
||||
print(json.dumps(metrics, sort_keys=True, indent=4))
|
||||
|
||||
metric_file_path = os.path.join(engine_dir, 'build_metrics')
|
||||
if os.path.exists(metric_file_path):
|
||||
shutil.rmtree(metric_file_path)
|
||||
os.makedirs(metric_file_path)
|
||||
|
||||
metric_file_path = os.path.join(metric_file_path, timestamp.strftime("%Y%m%d_%H%M%S.json"))
|
||||
|
||||
with open(metric_file_path, 'w') as metric_file:
|
||||
json.dump(metrics, metric_file, sort_keys=True, indent=4)
|
||||
|
||||
# transfer
|
||||
upload_script = os.path.join(current_dir, 'tools', 'upload_to_s3.py')
|
||||
upload_to_s3(upload_script, os.path.join(engine_dir, 'build_metrics'), 'ly-jenkins-cmake-metrics', args.jobname)
|
||||
|
||||
# submit
|
||||
submit_report_document(metric_file_path)
|
||||
|
||||
# Dont cleanup, next build will remove the file, but leaving it helps to do some post-build forensics
|
||||
@@ -16,7 +16,7 @@ import progressbar
|
||||
from optparse import OptionParser
|
||||
from PackageEnv import PackageEnv
|
||||
cur_dir = cur_dir = os.path.dirname(os.path.abspath(__file__))
|
||||
sys.path.insert(0, f'{cur_dir}/../../../Tools/build/JenkinsScripts/build')
|
||||
sys.path.insert(0, f'{cur_dir}/..')
|
||||
from ci_build import build
|
||||
from util import *
|
||||
from glob3 import glob
|
||||
|
||||
Executable
+92
@@ -0,0 +1,92 @@
|
||||
#
|
||||
# All or portions of this file Copyright (c) Amazon.com, Inc. or its affiliates or
|
||||
# its licensors.
|
||||
#
|
||||
# For complete copyright and license terms please see the LICENSE at the root of this
|
||||
# distribution (the "License"). All use of this software is governed by the License,
|
||||
# or, if provided, by the license below or the license accompanying this file. Do not
|
||||
# remove or modify any license notices. This file is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
#
|
||||
|
||||
import argparse
|
||||
import logging
|
||||
import json
|
||||
import socket
|
||||
from datetime import datetime
|
||||
|
||||
SOCKET_TIMEOUT = 60
|
||||
DATE_FORMAT = "%Y-%m-%dT%H:%M:%S"
|
||||
FILEBEAT_PIPELINE = "filebeat"
|
||||
FILEBEAT_DEFAULT_IP = "127.0.0.1"
|
||||
FILEBEAT_DEFAULT_PORT = 9000
|
||||
|
||||
def parse_args():
|
||||
parser = argparse.ArgumentParser(
|
||||
prog="submit_metrics.py",
|
||||
description="Pushes a JSON document via Filebeat.",
|
||||
add_help=False
|
||||
)
|
||||
|
||||
def file_arg(arg):
|
||||
try:
|
||||
with open(arg) as json_file:
|
||||
return json.load(json_file)
|
||||
except ValueError:
|
||||
raise argparse.ArgumentTypeError("Invalid json file '%s'" % arg)
|
||||
|
||||
parser.add_argument("-f", "--file", default=None, type=file_arg, help="File containing JSON data to upload.")
|
||||
parser.add_argument("-i", "--index", default=None, help="Index to use when sending the data")
|
||||
parser.add_argument("-ip", "--filebeat_ip", default=FILEBEAT_DEFAULT_IP, help="IP address where filebeat service is listening")
|
||||
parser.add_argument("-port", "--filebeat_port", default=FILEBEAT_DEFAULT_PORT, help="Port where filebeat service is listening")
|
||||
return parser.parse_args()
|
||||
|
||||
def submit(index, payload, filebeat_ip = FILEBEAT_DEFAULT_IP, filebeat_port = FILEBEAT_DEFAULT_PORT):
|
||||
try:
|
||||
filebeat_address = filebeat_ip, filebeat_port
|
||||
|
||||
logging.debug(f"Connecting to Filebeat on '{filebeat_address[0]}:{filebeat_address[1]}'")
|
||||
fb_socket = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
|
||||
fb_socket.settimeout(SOCKET_TIMEOUT)
|
||||
fb_socket.connect(filebeat_address)
|
||||
|
||||
event = {
|
||||
"index": index,
|
||||
"timestamp": datetime.strptime(payload['timestamp'], DATE_FORMAT).strftime(DATE_FORMAT),
|
||||
"pipeline": FILEBEAT_PIPELINE,
|
||||
"payload": json.dumps(payload),
|
||||
}
|
||||
|
||||
# Serialise event, add new line and encode as UTF-8 before sending to Filebeat.
|
||||
data = json.dumps(event) + "\n"
|
||||
data = data.encode()
|
||||
|
||||
total_sent = 0
|
||||
|
||||
logging.debug(f"Sending JSON data")
|
||||
while total_sent < len(data):
|
||||
try:
|
||||
sent = fb_socket.send(data[total_sent:])
|
||||
except BrokenPipeError:
|
||||
print("An exception occurred while sending data")
|
||||
fb_socket.close()
|
||||
total_sent = 0
|
||||
else:
|
||||
total_sent = total_sent + sent
|
||||
logging.debug("JSON data sent")
|
||||
fb_socket.close()
|
||||
logging.debug(f"Disconnected from Filebeat on '{filebeat_address[0]}:{filebeat_address[1]}'")
|
||||
except (ConnectionError, socket.timeout):
|
||||
logging.error("Failed to connect to Filebeat")
|
||||
return False
|
||||
return True
|
||||
|
||||
if __name__ == "__main__":
|
||||
# Parse CLI arguments.
|
||||
args = parse_args()
|
||||
if not args.index:
|
||||
logging.error(f"Index not specified")
|
||||
exit(1)
|
||||
|
||||
if not submit(args.index, json.dumps(args.file), args.filebeat_ip, args.filebeat_port):
|
||||
exit(1)
|
||||
@@ -0,0 +1 @@
|
||||
Lorem ipsum dolor sit amet, consectetur adipiscing elit, sed do eiusmod tempor incididunt ut labore et dolore magna aliqua. Ut enim ad minim veniam, quis nostrud exercitation ullamco laboris nisi ut aliquip ex ea commodo consequat. Duis aute irure dolor in reprehenderit in voluptate velit esse cillum dolore eu fugiat nulla pariatur. Excepteur sint occaecat cupidatat non proident, sunt in culpa qui officia deserunt mollit anim id est laborum.
|
||||
Executable
+32
@@ -0,0 +1,32 @@
|
||||
#
|
||||
# All or portions of this file Copyright (c) Amazon.com, Inc. or its affiliates or
|
||||
# its licensors.
|
||||
#
|
||||
# For complete copyright and license terms please see the LICENSE at the root of this
|
||||
# distribution (the "License"). All use of this software is governed by the License,
|
||||
# or, if provided, by the license below or the license accompanying this file. Do not
|
||||
# remove or modify any license notices. This file is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
#
|
||||
|
||||
import os
|
||||
import sys
|
||||
cur_dir = cur_dir = os.path.dirname(os.path.abspath(__file__))
|
||||
sys.path.insert(0, os.path.abspath(f'{cur_dir}/../build/package'))
|
||||
import util
|
||||
|
||||
# Run validator
|
||||
success = True
|
||||
validator_path = os.path.join(os.path.dirname(os.path.realpath(__file__)), 'validator.py')
|
||||
engine_root = os.path.dirname(os.path.dirname(os.path.dirname(os.path.realpath(__file__))))
|
||||
if sys.platform == 'win32':
|
||||
python = os.path.join(engine_root, 'python', 'python.cmd')
|
||||
else:
|
||||
python = os.path.join(engine_root, 'python', 'python.sh')
|
||||
args = [python, validator_path, '--package_platform', 'Windows', '--package_type', 'all', engine_root]
|
||||
return_code = util.safe_execute_system_call(args)
|
||||
if return_code != 0:
|
||||
success = False
|
||||
if not success:
|
||||
util.error('Restricted file validator failed.')
|
||||
print('Restricted file validator completed successfully.')
|
||||
Executable
+463
@@ -0,0 +1,463 @@
|
||||
#!/usr/bin/env python
|
||||
"""Python script to validate that code does not contain an console platform specific
|
||||
references or code that should not be published. Can be used to scan any directory."""
|
||||
|
||||
#
|
||||
# All or portions of this file Copyright (c) Amazon.com, Inc. or its affiliates or
|
||||
# its licensors.
|
||||
#
|
||||
# For complete copyright and license terms please see the LICENSE at the root of this
|
||||
# distribution (the "License"). All use of this software is governed by the License,
|
||||
# or, if provided, by the license below or the license accompanying this file. Do not
|
||||
# remove or modify any license notices. This file is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
#
|
||||
|
||||
# pylint: disable-msg=C0301
|
||||
|
||||
from __future__ import absolute_import
|
||||
from __future__ import print_function
|
||||
import six
|
||||
import logging
|
||||
import os
|
||||
import re
|
||||
import sys
|
||||
import traceback
|
||||
import json
|
||||
from pathlib import Path
|
||||
from optparse import OptionParser
|
||||
if six.PY2:
|
||||
from cStringIO import StringIO
|
||||
else:
|
||||
from io import StringIO
|
||||
|
||||
import validator_data_LEGAL_REVIEW_REQUIRED # pull in the data we need to configure this tool
|
||||
sys.path.append(os.path.join(os.path.dirname(os.path.realpath(__file__)), '..', 'build', 'package'))
|
||||
from glob_to_regex import generate_include_exclude_regexes
|
||||
|
||||
class Validator(object):
|
||||
"""Class to contain the validator program"""
|
||||
# Set of all acceptable_use patterns actually used during the run
|
||||
pattern_used = set([])
|
||||
|
||||
def validate_line(self, line, filepath, fileline, failed, errors, info):
|
||||
"""Check that a line of text does not have any pattern that leaks IP."""
|
||||
# The acceptable_use_patterns must be kept up to date as the code base evolves, and it is desirable
|
||||
# that this list be short. The -u command line option will run the validator in a mode
|
||||
# where it will report acceptable use patterns that did not get used in the check.
|
||||
# Those patterns are candidates for elimination. Ultimately we want the acceptable use
|
||||
# pattern set to be empty, or as close to empty as possible.
|
||||
#
|
||||
# This is a 3 stage check. First we do a rapid check against fixed text
|
||||
# strings (self.prefilter). These strings must capture a superset of potential
|
||||
# problematic texts. The test is run by lowercasing all characters, so that
|
||||
# case specific strings are not required. For example "abc" tests for the
|
||||
# presence of any combination of the letters in any case.
|
||||
# This first stage is a speed optimization, because regular expression matching
|
||||
# in python is really horribly slow (orders of magnitude slower than unix
|
||||
# grep in many cases!).
|
||||
lower = line.lower()
|
||||
if any(ext in lower for ext in self.prefilter):
|
||||
line = line.rstrip(os.linesep)
|
||||
|
||||
# Before we engage the accepted use machinery, make sure that the line contains a bad pattern that would
|
||||
# otherwise fail validation
|
||||
if self.compiled_bad_pattern.search(line) != None:
|
||||
|
||||
# The second stage checks against a more specific regular expression. (self.compiled_acceptable_pattern)
|
||||
# if the regular expression is matched, then the line contains information
|
||||
# that may be acceptable use or a false positive. We check if the line is covered by an established
|
||||
# exception by the presence of matching pattern in the self.acceptable_use_patterns regular expression.
|
||||
# Make a copy of the line and replace any matching accepted uses with EXCEPTION. If something
|
||||
# changes, then we have a possible accepted use which we'll have to vet.
|
||||
origline = line
|
||||
line = self.compiled_acceptable_pattern.sub('EXCEPTION', line)
|
||||
accepting_patterns = []
|
||||
if line != origline:
|
||||
|
||||
# Convert the path to forward slashes, then loop through all instances of an accepted use match
|
||||
# on this line
|
||||
line = origline
|
||||
canonical_filepath = os.path.abspath(filepath).replace('\\', '/')
|
||||
while True:
|
||||
m = self.compiled_acceptable_pattern.search(line)
|
||||
if not m:
|
||||
break
|
||||
|
||||
# Find the specific matching pattern
|
||||
match = line[m.start():m.end()]
|
||||
found = False
|
||||
for pattern,compiledp,fileset in self.acceptable_use_patterns:
|
||||
if compiledp.search(match) != None:
|
||||
|
||||
# If file testing isn't enabled, assume the match is good. Otherwise, search the file
|
||||
# patterns to see if this file is allowed
|
||||
if self.options.ignore_file_paths:
|
||||
found = True
|
||||
else:
|
||||
for fp in fileset:
|
||||
if fp.search(canonical_filepath) != None:
|
||||
found = True
|
||||
break
|
||||
if not found:
|
||||
errors.append("File rejected by pattern '{}': {}: line {}: {}".format(
|
||||
pattern, filepath, fileline, origline))
|
||||
|
||||
# First pattern match is good enough
|
||||
if found:
|
||||
if self.options.check_unused_patterns:
|
||||
self.pattern_used.add(pattern)
|
||||
accepting_patterns.append(pattern)
|
||||
break
|
||||
|
||||
# If no match was vetted by the filename, stop processing the line.
|
||||
if not found:
|
||||
break
|
||||
|
||||
# We remove the accepted use from the line and replace it with "EXCEPTION" and retest the line.
|
||||
line = line[:m.start()] + 'EXCEPTION' + line[m.end():]
|
||||
|
||||
# Once any possible accepted uses have been replaced, we check the resultant line to see if any bad
|
||||
# patterns remain in the line. If so, then we fail the line.
|
||||
if self.compiled_bad_pattern.search(line) != None:
|
||||
if self.options.list:
|
||||
self.output_unique_filepath(filepath)
|
||||
else:
|
||||
errors.append('validation failure in {}: line {}: {}'.format(filepath, fileline, origline))
|
||||
failed = 1
|
||||
|
||||
# Otherwise, spit out the details of each match
|
||||
else:
|
||||
for a in accepting_patterns:
|
||||
info.append("Allowed by '{}' pattern: {}: line {}: {}".format(a, filepath, fileline, origline))
|
||||
if self.options.exception_file:
|
||||
self.exceptions_output.write("Allowed by '{}' pattern: {}: line {}: {}\n".format(a, filepath, fileline, origline))
|
||||
return failed
|
||||
|
||||
def output_unique_filepath(self, filepath):
|
||||
"""Output the name of a file exactly once in a run."""
|
||||
# Dictionary to use to ensure that we know which files we have already talked about
|
||||
global printed_filepath
|
||||
try:
|
||||
if not filepath in printed_filepath:
|
||||
print(filepath)
|
||||
printed_filepath[filepath] = 1
|
||||
except:
|
||||
# Global dict does not exist yet. Make it.
|
||||
printed_filepath = {}
|
||||
printed_filepath[filepath] = 1
|
||||
# Just print the filepath if we have not already printed it
|
||||
print(filepath)
|
||||
|
||||
def validate_file(self, filepath):
|
||||
"""Validate the content of a file 'filepath'.
|
||||
Return 0 if no issues are found, and 1 if an issue was noted."""
|
||||
failed = 0
|
||||
|
||||
# Otherwise read the file off disk. Check the filename itself to make sure no naughty
|
||||
# bits are there.
|
||||
errors = []
|
||||
info = []
|
||||
failed = self.validate_line(filepath, 'filename', 0, failed, errors, info)
|
||||
for e in errors:
|
||||
logging.error(e)
|
||||
for i in info:
|
||||
logging.info(i)
|
||||
|
||||
# Check if this file is a binary file, or an extension we always skip
|
||||
# These extensions are here because they sometimes look like text files,
|
||||
# but are not really text files in practice.
|
||||
if validator_data_LEGAL_REVIEW_REQUIRED.skip_file(filepath):
|
||||
logging.debug('Skipping %s', filepath)
|
||||
return
|
||||
|
||||
# Python3 requires specific encoding but the repo is a mix of UTF-8, UTF-16, and latin-1
|
||||
# Just try except the possibilities until it works
|
||||
encodings = ["utf8", "utf-16-le", "utf-16-be", "latin-1"]
|
||||
for encoding_format in encodings:
|
||||
try:
|
||||
with open(filepath, encoding=encoding_format) as f:
|
||||
logging.debug('Validating %s', filepath)
|
||||
|
||||
# Take care to deal with files that have unreasonably large lines.
|
||||
# if we don't do this, then the validator can segfault as it tries
|
||||
# to read a line that is insanely large. A try/except will not prevent this.
|
||||
# This can happen, for example, in XML file or obj files (the 3d format, not
|
||||
# the compiler output kind).
|
||||
# Quesion: if we encounter such a file, should we call it "binary" and quit?
|
||||
# TODO: There is a small "leak" here, in that we don't deal with crossing
|
||||
# a non line boundary clealy. In particular, a validation pattern
|
||||
# that occurs in the boundary crossing will not be properly searched.
|
||||
# The easiest thing to do is probably to retain the prior 128 bytes or
|
||||
# so from the prior portion of the line as a prefix to the tail of the rest of the line.
|
||||
# This will make the logic below much more complex.
|
||||
fileline = 0
|
||||
line = f.readline(10000)
|
||||
while line != '':
|
||||
fileline += 1
|
||||
errors = []
|
||||
info = []
|
||||
failed = self.validate_line(line, filepath, fileline, failed, errors, info)
|
||||
for e in errors:
|
||||
logging.error(e)
|
||||
for i in info:
|
||||
logging.info(i)
|
||||
line = f.readline(10000)
|
||||
return failed
|
||||
except UnicodeDecodeError:
|
||||
continue
|
||||
|
||||
raise UnicodeError("Could not decode {0} due to an unexpected file encoding".format(filepath))
|
||||
|
||||
# Walk directory tree and find all file paths, and run the search for bad code on each file.
|
||||
# We explicitly skip "SDKs" directories, "BinTemp" and "Python" directories and various others.
|
||||
# The first two are never part of the package, and Python is a special case SDK that we ship as is.\
|
||||
# TODO: Perhaps the directories to skip should become a parameter so we can use the validator
|
||||
# on non-Lumberyard trees.
|
||||
def validate_directory_tree(self, root, platform):
|
||||
prohibited_platforms = validator_data_LEGAL_REVIEW_REQUIRED.get_prohibited_platforms_for_package(self.options.package_platform)
|
||||
(includes, excludes) = generate_include_exclude_regexes(self.options.package_platform, self.options.package_type, root, prohibited_platforms)
|
||||
|
||||
"""Walk from root to find all files to validate and call the validator on each file.
|
||||
Return 0 if no problems where found, and 1 if any validation failures occured."""
|
||||
counter = 0
|
||||
platform_failed = 0
|
||||
scanned = 0
|
||||
validations = 0
|
||||
bypassed_directories = validator_data_LEGAL_REVIEW_REQUIRED.get_bypassed_directories(self.options.all)
|
||||
|
||||
for dirname, dirnames, filenames in os.walk(os.path.normpath(root)):
|
||||
# First deal with the files in the current directory
|
||||
for filename in filenames:
|
||||
filepath = os.path.join(dirname, filename)
|
||||
include_match = includes.match(filepath)
|
||||
exclude_match = excludes.match(filepath)
|
||||
allowed = include_match and not exclude_match
|
||||
|
||||
if self.options.all or allowed:
|
||||
scanned += 1
|
||||
file_failed = self.validate_file(os.path.normpath(filepath))
|
||||
if file_failed:
|
||||
platform_failed = file_failed
|
||||
else:
|
||||
validations += 1
|
||||
counter += 1
|
||||
|
||||
# Trim out whitelisted subdirectories in the current directory if allowed
|
||||
for name in bypassed_directories:
|
||||
if name in dirnames:
|
||||
dirnames.remove(name)
|
||||
if counter == 0 or scanned == 0:
|
||||
logging.error('No files scanned at target search directory: %s', root)
|
||||
platform_failed = 1
|
||||
else:
|
||||
print('validated {} of {} package files ({} non-package files skipped)'.format(validations, scanned, counter - scanned))
|
||||
return platform_failed
|
||||
|
||||
|
||||
def compile_filter_patterns(self, platform):
|
||||
"""Join together patterns listed in data file into single patterns and compile for speed."""
|
||||
if not platform in validator_data_LEGAL_REVIEW_REQUIRED.restricted_platforms:
|
||||
logging.error('platform data for platform %s not provided in validator_data_LEGAL_REVIEW_REQUIRED.py.', platform)
|
||||
sys.exit(1)
|
||||
restricted_platform = validator_data_LEGAL_REVIEW_REQUIRED.restricted_platforms[platform]
|
||||
if not 'prefilter' in restricted_platform:
|
||||
logging.error('prefilter list not found for platform %s in validator_data_LEGAL_REVIEW_REQUIRED.py', platform)
|
||||
sys.exit(1)
|
||||
self.prefilter = restricted_platform['prefilter']
|
||||
if not 'patterns' in restricted_platform:
|
||||
logging.error('patterns list not found for platform %s in validator_data_LEGAL_REVIEW_REQUIRED.py', platform)
|
||||
sys.exit(1)
|
||||
self.bad_patterns = restricted_platform['patterns']
|
||||
if not 'acceptable_use' in restricted_platform:
|
||||
logging.error('acceptable_use list not found for platform %s in validator_data_LEGAL_REVIEW_REQUIRED.py', platform)
|
||||
sys.exit(1)
|
||||
self.acceptable_use_patterns = []
|
||||
for p,fileset in restricted_platform['acceptable_use']:
|
||||
try:
|
||||
self.acceptable_use_patterns.append((p, re.compile(p), [re.compile(f) for f in fileset]))
|
||||
except:
|
||||
logging.error("Couldn't compile pattern {}...".format(p))
|
||||
traceback.print_exc()
|
||||
sys.exit(1)
|
||||
|
||||
try:
|
||||
# Compile the search patterns for speed
|
||||
bad_pattern = '|'.join(self.bad_patterns)
|
||||
acceptable_pattern = '|'.join([p for p,compiledp,fileset in self.acceptable_use_patterns])
|
||||
self.compiled_bad_pattern = re.compile(bad_pattern)
|
||||
self.compiled_acceptable_pattern = re.compile(acceptable_pattern)
|
||||
except:
|
||||
logging.error('Could not compile patterns for validation. Check patterns in validator_data_LEGAL_REVIEW_REQUIRED.py for correctness.')
|
||||
traceback.print_exc()
|
||||
sys.exit(1)
|
||||
|
||||
def test_prefilter_covers_bad_patterns(self):
|
||||
double_pattern = re.compile(r'\[(.)\1\]')
|
||||
for bad in self.bad_patterns:
|
||||
reduced = re.sub(double_pattern, r'\1', bad.lower())
|
||||
found = False
|
||||
for p in self.prefilter:
|
||||
if p in reduced:
|
||||
found = True
|
||||
break
|
||||
if not found:
|
||||
logging.error('Could not find a prefilter for {}.'.format(bad))
|
||||
return False
|
||||
return True
|
||||
|
||||
def test_all_bad_patterns_active(self, platform):
|
||||
|
||||
# Open the canary file relative to the validator script, that way we don't have to worry about temporary files and whatnot
|
||||
# Once we split the repos we will have to worry about multiple root points etc. but that is a problem for future us.
|
||||
# All the packaging safelist stuff goes away once repo is split for platforms
|
||||
this_path = Path(__file__).resolve()
|
||||
root_folder = this_path.parents[5]
|
||||
relative_folder = os.path.relpath(this_path.parent, root_folder)
|
||||
canary_file = os.path.join(root_folder, 'restricted', platform, relative_folder, platform.lower() + '_canary.txt')
|
||||
try:
|
||||
with open(canary_file) as canary:
|
||||
bad_patterns = self.bad_patterns
|
||||
canary.seek(0, 0)
|
||||
fileline = 0
|
||||
for line in canary:
|
||||
fileline += 1
|
||||
errors = []
|
||||
info = []
|
||||
|
||||
# Each validation failure needs to be tracked back to the patterns that detect it. Once we find one, eliminate it
|
||||
# from the search list since we've found an instance where it would trigger
|
||||
if self.validate_line(line, canary_file, fileline, 0, errors, info) == 1:
|
||||
found = []
|
||||
for bad in bad_patterns:
|
||||
if re.search(bad, line):
|
||||
found.append(bad)
|
||||
for bad in found:
|
||||
bad_patterns.remove(bad)
|
||||
|
||||
# If the search list isn't empty, then whatever remains needs a canary
|
||||
if len(bad_patterns) > 0:
|
||||
for bad in bad_patterns:
|
||||
logging.error("Could not find a canary for '{}'.".format(bad))
|
||||
return False
|
||||
except:
|
||||
logging.error('Could not open canary file {}.'.format(canary_file))
|
||||
return False
|
||||
return True
|
||||
|
||||
def test(self, platform):
|
||||
if not self.test_prefilter_covers_bad_patterns():
|
||||
return False
|
||||
if not self.test_all_bad_patterns_active(platform):
|
||||
return False
|
||||
return True
|
||||
|
||||
def validate(self, platform):
|
||||
self.compile_filter_patterns(platform)
|
||||
if not self.test(platform):
|
||||
logging.error('Validation could not pass {} self tests! Results cannot be trusted!'.format(platform))
|
||||
sys.exit(1)
|
||||
else:
|
||||
print('{} self tests SUCCEEDED'.format(platform))
|
||||
|
||||
# Add the source code / SDK paths to check
|
||||
platform_failed = self.validate_directory_tree(os.path.abspath(self.args[0]), platform)
|
||||
|
||||
# If the user asked, output any acceptable_use patterns that didn't get used
|
||||
if self.options.check_unused_patterns:
|
||||
for pattern,compiledp,fileset in self.acceptable_use_patterns:
|
||||
if not pattern in self.pattern_used:
|
||||
print("UNUSED ACCEPTABLE_USE PATTERN: '{}'".format(pattern))
|
||||
|
||||
return platform_failed
|
||||
|
||||
def __init__(self, options, args):
|
||||
self.options = options
|
||||
self.args = args
|
||||
self.prefilter = None
|
||||
self.compiled_bad_pattern = None
|
||||
self.compiled_acceptable_pattern = None
|
||||
self.bad_patterns = None
|
||||
self.acceptable_use_patterns = None
|
||||
|
||||
def parse_options():
|
||||
"""Set up the options parser, and parse the options the user gave to validator."""
|
||||
usage = 'usage: %prog [options] scandir'
|
||||
parser = OptionParser(usage)
|
||||
platform_choices = list(validator_data_LEGAL_REVIEW_REQUIRED.restricted_platforms_for_package.keys())
|
||||
platform_choices.sort()
|
||||
parser.add_option('--package_platform', action='store', type='choice',
|
||||
choices=platform_choices,
|
||||
dest='package_platform',
|
||||
help='Package platform to validate. Must be one of {}.'.format(platform_choices))
|
||||
parser.add_option('--package_type', action='store', type='string', default='all', dest='package_type',
|
||||
help='Package type to validate.')
|
||||
parser.add_option('-s', '--store-exceptions', action='store', type='string', default='',
|
||||
dest='exception_file',
|
||||
help='Store list of lines that the validator gave exceptions to by matching accepted use patterns. These can be diffed with prior runs to see what is changing.')
|
||||
parser.add_option('-v', '--verbose', action='store', type='choice', choices=['0', '1', '2'], default='0',
|
||||
dest='verbose',
|
||||
help='Verbose output. Level 0 = only output lines that fail validation. '
|
||||
'Level 1 = output of lines that would have failed without an accepted usage exception. '
|
||||
'Level 2 also includes output of each filename being handled.')
|
||||
parser.add_option('-u', '--check-unused-patterns', action='store_true',
|
||||
dest='check_unused_patterns',
|
||||
help='Report on any acceptable_use patterns that are not matched.')
|
||||
parser.add_option('-l', '--list', action='store_true',
|
||||
dest='list',
|
||||
help='Only list filenames with validation errors. Useful as input to a set of files to edit or otherwise process.')
|
||||
parser.add_option('-a', '--all', action='store_true',
|
||||
dest='all',
|
||||
help='Do not skip any files or subdirectories when processing. Should be used on final clean code only. If you use this on your build tree in place lots of temp files will match.')
|
||||
parser.add_option('-i', '--ignore-file-paths', action='store_true',
|
||||
dest='ignore_file_paths',
|
||||
help='disable the filepath check for accepted_use patterns. Should only be when targeting a directory other than /dev/.')
|
||||
|
||||
(options, args) = parser.parse_args()
|
||||
|
||||
if options.verbose == '1':
|
||||
logging.basicConfig(format='%(levelname)s: %(message)s', level=logging.INFO)
|
||||
elif options.verbose == '2':
|
||||
logging.basicConfig(format='%(levelname)s: %(message)s', level=logging.DEBUG)
|
||||
else:
|
||||
logging.basicConfig(format='%(levelname)s: %(message)s')
|
||||
|
||||
if len(args) != 1:
|
||||
parser.error('no directory to scan specified and/or incorrect number of directories specified.')
|
||||
return options, args
|
||||
|
||||
def main():
|
||||
"""Main function for validator script"""
|
||||
options, args = parse_options()
|
||||
validator = Validator(options, args)
|
||||
|
||||
package_failed = 0
|
||||
package_platform = validator.options.package_platform
|
||||
package_type = validator.options.package_type
|
||||
prohibited_platforms = validator_data_LEGAL_REVIEW_REQUIRED.get_prohibited_platforms_for_package(package_platform)
|
||||
|
||||
if validator.options.exception_file != '':
|
||||
try:
|
||||
validator.exceptions_output = open(validator.options.exception_file, 'w', errors='ignore')
|
||||
except:
|
||||
logging.error("Cannot open exceptions output file '%s'", validator.options.exception_file)
|
||||
sys.exit(1)
|
||||
|
||||
for platform in prohibited_platforms:
|
||||
print('validating {} against {} for package platform {} package type {}'.format(args[0], platform, package_platform, package_type))
|
||||
platform_failed = validator.validate(platform)
|
||||
if platform_failed:
|
||||
print('{} FAILED validation against {} for package platform {} package type {}'.format(args[0], platform, package_platform, package_type))
|
||||
package_failed = platform_failed
|
||||
else:
|
||||
print('{} is VALIDATED against {} for package platform {} package type {}'.format(args[0], platform, package_platform, package_type))
|
||||
|
||||
if validator.options.exception_file != '':
|
||||
validator.exceptions_output.close()
|
||||
|
||||
return package_failed
|
||||
|
||||
if __name__ == '__main__':
|
||||
# pylint: disable-msg=C0103
|
||||
main_results = main()
|
||||
sys.exit(main_results)
|
||||
+132
@@ -0,0 +1,132 @@
|
||||
# This python file contains the data used to drive the validator.
|
||||
"""Data for Validator tool"""
|
||||
#
|
||||
# All or portions of this file Copyright (c) Amazon.com, Inc. or its affiliates or
|
||||
# its licensors.
|
||||
#
|
||||
# For complete copyright and license terms please see the LICENSE at the root of this
|
||||
# distribution (the "License"). All use of this software is governed by the License,
|
||||
# or, if provided, by the license below or the license accompanying this file. Do not
|
||||
# remove or modify any license notices. This file is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
#
|
||||
|
||||
# Anyone seeking to modify this file must follow the process, contact sig-build
|
||||
|
||||
# Notice that this is not a JSON file, even though it is almost a valid JSON
|
||||
# data file.
|
||||
# The key reason we store this in a python file instead is so that we can
|
||||
# put explanatory comments int the data, which is not supported by JSON.
|
||||
# There are other possible approaches (such as removing comments with an re after reading the string)
|
||||
# but they are more work, so for now we just do it this way.
|
||||
|
||||
# In regular expressions in this file, do not use patterns that start with ".*", as they make the entire system very slow.
|
||||
# Even patterns that start with an single "." open match will slow the system down.
|
||||
# Patterns should start with a match to a specific character, and match as many specific
|
||||
# characters as possible before using any wildcard matches.
|
||||
|
||||
import os
|
||||
import logging
|
||||
import sys
|
||||
from pathlib import Path
|
||||
|
||||
restricted_platforms_for_package = {
|
||||
'Mac': [],
|
||||
'Windows': [],
|
||||
'Provo': ['Provo'],
|
||||
'Salem': ['Salem'],
|
||||
'Jasper': ['Jasper'],
|
||||
'Linux': []
|
||||
}
|
||||
|
||||
restricted_platforms = {}
|
||||
|
||||
|
||||
def find_restricted_platforms():
|
||||
this_path = Path(__file__).resolve()
|
||||
root_folder = this_path.parents[2]
|
||||
relative_path = os.path.relpath(this_path.parent, root_folder)
|
||||
restricted_path = os.path.join(root_folder, 'restricted')
|
||||
if os.path.exists(restricted_path):
|
||||
for dir in [f.path for f in os.scandir(restricted_path) if f.is_dir()]:
|
||||
sys.path.append(os.path.join(dir, relative_path))
|
||||
try:
|
||||
module = __import__('{}_data_LEGAL_REVIEW_REQUIRED'.format(os.path.basename(dir).lower()), locals(), globals())
|
||||
module.add_restricted_platform(restricted_platforms)
|
||||
except ModuleNotFoundError:
|
||||
pass
|
||||
|
||||
|
||||
find_restricted_platforms()
|
||||
|
||||
# Order of the exceptions is important. The longer and more specific exceptions should come before the
|
||||
# short ones to improve the locality of any given exception.
|
||||
# This is based on the unix style test for binary files.
|
||||
# It provides a set of characters that occur in text files.
|
||||
# We assume that file that contain characters outside of this set are binary
|
||||
# files and skip them.
|
||||
TEXT_CHARS = bytearray({7, 8, 9, 10, 12, 13, 27} | set(range(0x20, 0x100)) - {0x7f})
|
||||
|
||||
|
||||
def skip_file(filepath):
|
||||
_ext = os.path.splitext(filepath)[1].lower()
|
||||
"""Check if a file is binary"""
|
||||
# Don't ever scan these, even thought they sometimes look like text files or are text files
|
||||
if _ext in {'.js', '.ma', '.psd', '.fbx', '.obj', '.pem', '.lyr', '.resx', '.pak', '.dat', '.dds', '.ilk', '.ppm', '.html'}:
|
||||
return True
|
||||
# the dds extension is special, there may be foo.dds, foo.dds.1, foo.dds.2, etc. In these cases, .dds is not the extension returned by splitext
|
||||
if '.dds.' in filepath:
|
||||
return True
|
||||
|
||||
# Always scan these, even if someone puts binary characters in them.
|
||||
if _ext in {'.cpp', '.c++', '.cxx', '.h', '.hpp', '.hxx', '.inl', '.c', '.cs', '.py', '.mel', '.mm', '.rc', '.ui', '.qrc', '.ts', '.ini', '.cfg', '.cfx',
|
||||
'.cfi', '.plist'}:
|
||||
return False
|
||||
|
||||
# Otherwise, open the file and see if we think it is binary by looking at the data.
|
||||
# This is based on unix style test for binary.
|
||||
try:
|
||||
with open(filepath, mode='rb') as f:
|
||||
data = f.read(1024)
|
||||
binary = bool(data.translate(None, TEXT_CHARS))
|
||||
return binary
|
||||
except:
|
||||
logging.error('Unable to load file - check this manually: %s', filepath)
|
||||
return True
|
||||
|
||||
|
||||
def get_prohibited_platforms_for_package(package):
|
||||
permitted_platforms = restricted_platforms_for_package[package]
|
||||
return [p for p in restricted_platforms if p not in permitted_platforms]
|
||||
|
||||
|
||||
def get_bypassed_directories(is_all):
|
||||
# Temporarily exempt folders to not fail validation while people is fixing validation errors, they will be removed once the errors are fixed.
|
||||
temp_bypass_directories = [
|
||||
'commit_validation',
|
||||
'LauncherTestTools',
|
||||
'AutomatedTesting',
|
||||
'Atom'
|
||||
]
|
||||
bypassed_directories = [
|
||||
'python',
|
||||
'AWSPythonSDK'
|
||||
]
|
||||
if not is_all:
|
||||
bypassed_directories.extend([
|
||||
'.git',
|
||||
'restricted',
|
||||
'Cache',
|
||||
'logs',
|
||||
'AssetProcessorTemp',
|
||||
'JenkinsScripts',
|
||||
'BuildLambdaFunctions',
|
||||
'layouts',
|
||||
'.idea',
|
||||
'user/log',
|
||||
'DirectXShaderCompiler',
|
||||
'v-hacd',
|
||||
'External'
|
||||
])
|
||||
bypassed_directories.extend(temp_bypass_directories)
|
||||
return bypassed_directories
|
||||
Reference in New Issue
Block a user